mirrors/fx
0
0 Fork 0
mirror of https://github.com/vercel-labs/fx.git synced 2026-10-28 10:15:17 +08:00
Code Activity

Merge origin/main and port the indexer to Zig 0.17

Browse Source 
main moved to Zig 0.17. The branch keeps its deletion of
file_index_cache.zig, ignored_dirs.zig and workspace_files.zig, whose
only changes on main were 0.17 renames, and keeps its own side of
grep_search.zig and glob_files.zig. AGENTS.md takes main's refresh plus
the indexer and safe git notes.

The branch's code moves to the 0.17 names: std.Io.Dir.path,
Allocator.print, std.mem.print, builtin.target and @backingInt. The
wildmatch test builds its repeated strings with a local helper because
the indexer cannot import text_utils, and two folder selection test
calls gain the try that 0.17 requires.
This commit is contained in:
Pranit committed 2026-10-12 02:20:21 -04:00
2 parents f62e85ce49 8f75c7d7c0
commit 0a76c1ee38
525 files changed
+12313 -7240
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/actions/setup-pgso/action.yml
+5 -5
View File
@@ -7,7 +7,7 @@ inputs:
required: false
default: "false"
llvm:
description: Install LLVM 21
description: Install LLVM 22
required: false
default: "false"
bun:
@@ -37,7 +37,7 @@ runs:
if: inputs.zig == 'true' || inputs.llvm == 'true'
uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
with:
version: "0.16.0"
version: "0.17.0"
cache-key: pgso-macos-arm64
- name: Install pinned LLVM
@@ -45,9 +45,9 @@ runs:
if: inputs.llvm == 'true'
shell: bash
run: |
brew install llvm@21
llvm_bin="$(brew --prefix llvm@21)/bin"
"$llvm_bin/opt" --version | grep -F "version 21.1.8"
brew install llvm@22
llvm_bin="$(brew --prefix llvm@22)/bin"
"$llvm_bin/opt" --version | grep -F "version 22.1.8"
echo "path=$llvm_bin" >> "$GITHUB_OUTPUT"
- name: Setup Bun
.github/workflows/bench.yml
+4 -4
View File
@@ -46,7 +46,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: bench-ReleaseSafe
- name: Build (ReleaseSafe)
@@ -105,7 +105,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: bench-ReleaseSafe
- name: Build (ReleaseSafe)
@@ -139,7 +139,7 @@ jobs:
- name: Setup Zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: bench-ReleaseSafe
- name: Build (ReleaseSafe)
@@ -176,7 +176,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: bench-ReleaseSafe
- name: Setup Node
.github/workflows/binary-size.yml
+25 -9
View File
@@ -54,7 +54,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"
version: "0.17.0"
use-cache: false
- name: Test size reporter
@@ -67,11 +67,10 @@ jobs:
evidence_dir="$RUNNER_TEMP/fx-binary-size-evidence"
mkdir -p "$evidence_dir"
test "$(git rev-parse HEAD)" = "$HEAD_SHA"
zig build \
ZIG_GLOBAL_CACHE_DIR="$RUNNER_TEMP/fx-global-cache" zig build \
-Dtarget=${{ matrix.target }} \
-Doptimize=ReleaseSafe \
--cache-dir "$RUNNER_TEMP/fx-head-cache" \
--global-cache-dir "$RUNNER_TEMP/fx-global-cache"
--cache-dir "$RUNNER_TEMP/fx-head-cache"
cp zig-out/bin/fx "$evidence_dir/head-fx"
file "$evidence_dir/head-fx" > "$evidence_dir/head-file.txt"
cat "$evidence_dir/head-file.txt"
@@ -91,12 +90,30 @@ jobs:
VERCEL_OIDC_TOKEN: ""
run: scripts/smoke-binary.sh "$RUNNER_TEMP/fx-binary-size-evidence/head-fx"
# The base commit builds with the Zig version it pins, which differs
# from this pull request's when the pull request changes the toolchain.
- name: Read base Zig version
id: base-zig
shell: bash
run: |
set -euo pipefail
version="$(git show "$BASE_SHA:build.zig.zon" | sed -n 's/^ *\.minimum_zig_version = "\([^"]*\)",$/\1/p')"
test -n "$version"
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Restore base binary
id: base-cache
uses: actions/cache/restore@v4
with:
path: ${{ runner.temp }}/fx-binary-size-base-bin
key: binary-size-base-v1-${{ matrix.target }}-zig-0.16.0-${{ env.BASE_SHA }}
key: binary-size-base-v1-${{ matrix.target }}-zig-${{ steps.base-zig.outputs.version }}-${{ env.BASE_SHA }}
- name: Setup base Zig
if: steps.base-cache.outputs.cache-hit != 'true'
uses: mlugg/setup-zig@v2
with:
version: ${{ steps.base-zig.outputs.version }}
use-cache: false
- name: Build base binary
if: steps.base-cache.outputs.cache-hit != 'true'
@@ -112,11 +129,10 @@ jobs:
test "$(git -C "$base_worktree" rev-parse HEAD)" = "$BASE_SHA"
(
cd "$base_worktree"
zig build \
ZIG_GLOBAL_CACHE_DIR="$RUNNER_TEMP/fx-base-global-cache" zig build \
-Dtarget=${{ matrix.target }} \
-Doptimize=ReleaseSafe \
--cache-dir "$RUNNER_TEMP/fx-base-cache" \
--global-cache-dir "$RUNNER_TEMP/fx-global-cache"
--cache-dir "$RUNNER_TEMP/fx-base-cache"
)
mkdir -p "$RUNNER_TEMP/fx-binary-size-base-bin"
cp "$base_worktree/zig-out/bin/fx" "$RUNNER_TEMP/fx-binary-size-base-bin/fx"
@@ -126,7 +142,7 @@ jobs:
uses: actions/cache/save@v4
with:
path: ${{ runner.temp }}/fx-binary-size-base-bin
key: binary-size-base-v1-${{ matrix.target }}-zig-0.16.0-${{ env.BASE_SHA }}
key: binary-size-base-v1-${{ matrix.target }}-zig-${{ steps.base-zig.outputs.version }}-${{ env.BASE_SHA }}
- name: Measure base binary
shell: bash
.github/workflows/ci.yml
+9 -9
View File
@@ -37,7 +37,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: ci-unit-${{ matrix.optimize }}
- name: Check formatting
@@ -104,7 +104,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: ci-build-ReleaseSafe
# Other runners execute this binary, and their CPUs can differ from this
@@ -141,7 +141,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: ci-sdk-wasm
- name: Build WASM surfaces (ReleaseSmall)
@@ -194,7 +194,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: ci-sdk-napi-${{ matrix.platform }}
- name: Build N-API addon (ReleaseSafe)
@@ -255,7 +255,7 @@ jobs:
run: |
node --experimental-wasm-jspi sdk/tests/test-durable.mjs memory wasm
node --experimental-wasm-jspi sdk/tests/test-durable.mjs local wasm
node --experimental-wasm-jspi sdk/tests/test-durable-codex-crash.mjs wasm
node --experimental-wasm-jspi sdk/tests/test-durable-provider-crash.mjs wasm
- name: Type-check the TypeScript declarations
run: node sdk/tests/test-types.mjs
@@ -302,7 +302,7 @@ jobs:
node sdk/tests/test-durable.mjs memory native
node sdk/tests/test-durable.mjs local native
node sdk/tests/test-durable.mjs world native
node sdk/tests/test-durable-codex-crash.mjs native
node sdk/tests/test-durable-provider-crash.mjs native
node sdk/tests/test-durable-core.mjs memory
node sdk/tests/test-durable-core.mjs local
node sdk/tests/test-durable-core.mjs world
@@ -533,7 +533,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: ci-e2e-fixtures
- name: Setup Bun
@@ -636,7 +636,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: ci-mcp-fixtures
- name: Setup Bun
@@ -715,7 +715,7 @@ jobs:
id: zig
uses: ./.github/actions/setup-zig
with:
version: "0.16.0"
version: "0.17.0"
cache-key: ci-unit-ReleaseSafe
- name: Build
.github/workflows/dev-release.yml
+2 -2
View File
@@ -41,7 +41,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"
version: "0.17.0"
- name: Build dev binary
run: zig build -Doptimize=ReleaseSafe -Dtarget=${{ matrix.target }} -Dupdate-channel=dev
@@ -77,7 +77,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"
version: "0.17.0"
- name: Build terminal WASM
run: zig build -Doptimize=ReleaseSafe -Dwasm-surface=term
.github/workflows/macos.yml
+3 -3
View File
@@ -67,7 +67,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"
version: "0.17.0"
use-cache: false
- name: Test signing and release routing
@@ -91,7 +91,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"
version: "0.17.0"
use-cache: false
# Both E2E shards run this binary on other runners, so build for the
@@ -147,7 +147,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"
version: "0.17.0"
use-cache: false
- name: Setup Bun
.github/workflows/publish-libfx.yml
+4 -4
View File
@@ -134,7 +134,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
with:
version: "0.16.0"
version: "0.17.0"
- name: Setup Node
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
@@ -191,7 +191,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
with:
version: "0.16.0"
version: "0.17.0"
- name: Setup Node
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
@@ -284,9 +284,9 @@ jobs:
const names = new Set(report.files.map(({ path }) => path));
const required = [
"package.json", "README.md", "LICENSE", "browser.js", "node.js", "node.cjs", "fx-sdk.js", "wasm-module.js",
"core-output.js", "fetch-cleanup.js", "mcp.js", "skills.js", "skills-node.js", "chatgpt.js", "chatgpt-sign-in.js",
"core-output.js", "fetch-cleanup.js", "mcp.js", "skills.js", "skills-node.js", "subscription.js", "oauth.js", "chatgpt.js", "chatgpt-sign-in.js", "grok.js", "grok-sign-in.js", "custom-provider.js",
"durable.js", "fx-harness.js", "durable/local.mjs", "durable/vercel.mjs", "durable/world.mjs",
"types/browser.d.ts", "types/chatgpt.d.ts", "types/durable-local.d.ts", "types/durable-vercel.d.ts", "types/durable-world.d.ts", "types/libfx.d.cts", "types/mcp.d.ts", "types/node.d.cts", "types/node.d.ts", "types/skills-node.d.ts", "types/skills.d.ts", "types/wasm.d.ts",
"types/browser.d.ts", "types/chatgpt.d.ts", "types/durable-local.d.ts", "types/durable-vercel.d.ts", "types/durable-world.d.ts", "types/grok.d.ts", "types/libfx.d.cts", "types/mcp.d.ts", "types/node.d.cts", "types/node.d.ts", "types/skills-node.d.ts", "types/skills.d.ts", "types/wasm.d.ts",
"fx-core.wasm", "fx-term.wasm",
"libfx.linux-x64.node", "libfx.linux-arm64.node",
"libfx.darwin-x64.node", "libfx.darwin-arm64.node",
.github/workflows/release.yml
+2 -2
View File
@@ -75,7 +75,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
with:
version: "0.16.0"
version: "0.17.0"
- name: Build
run: zig build -Doptimize=ReleaseSafe -Dtarget=${{ matrix.target }}
@@ -109,7 +109,7 @@ jobs:
- name: Setup Zig
uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
with:
version: "0.16.0"
version: "0.17.0"
- name: Build
run: zig build -Doptimize=ReleaseSafe -Dtarget=x86_64-macos
AGENTS.md
+80 -83
View File
@@ -18,34 +18,33 @@ If you cannot run the binary in your environment, say so explicitly and ask the
### Always use the built binary in this repo
When running fx for verification, **always use the freshly-built binary at** **`./zig-out/bin/fx`** from this checkout. Never run `fx` from `PATH`, never rely on whatever is at `~/.fx/bin/fx`, and never assume an installed copy reflects your change.
When running fx for verification, **always use the freshly-built binary at** **`./zig-out/bin/fx`** from this checkout. Never run `fx` from `PATH`, never rely on the installed copy at `~/.local/bin/fx`, and never assume an installed copy reflects your change.
* The user may have an older `fx` on their PATH (e.g. installed via `fx upgrade` or the CDN install script). Running that one will not exercise your edits.
* The user may have an older `fx` on their PATH (e.g. installed via `fx upgrade` or the install script). Running that one will not exercise your edits.
* `zig build` writes to `zig-out/bin/fx`. That is the only binary that contains your latest change.
* When a user reports "still not working" after you believe you fixed something, do not assume they are running the wrong binary. Assume your fix is incomplete and investigate further. If you genuinely suspect a PATH mismatch, ask — do not silently copy binaries into `~/.fx/bin/`.
* When a user reports "still not working" after you believe you fixed something, do not assume they are running the wrong binary. Assume your fix is incomplete and investigate further. If you genuinely suspect a PATH mismatch, ask — do not silently copy binaries onto their `PATH`.
* In any shell invocation — tmux, direct run, scripts — reference fx as `/Users/<you>/path/to/repo/zig-out/bin/fx` (absolute) or `./zig-out/bin/fx` (when cwd is the repo root). Bare `fx` is always wrong for dev verification.
## Language and Toolchain
This project is written in **Zig 0.16+**. There is no Node.js runtime, no `package.json` at the root, and no JavaScript build step for the main binary.
This project is written in **Zig 0.17+**. There is no Node.js runtime, no `package.json` at the root, and no JavaScript build step for the main binary.
Build and test commands:
```bash
zig build # build the binary
zig build test # run all unit tests
zig build run # build and run
zig fmt src/ # format all source files
zig build # build the binary
zig build test # run all unit tests
zig build run # build and run
zig fmt src/ # format all source files
zig fmt --check src/ # the formatting check CI runs
```
The test suites under `tests/` use Bun but are separate from the Zig codebase. See **Testing** below.
## Code Style
* Format all Zig source with `zig fmt` before committing. The canonical check is `zig fmt src/`.
* Format all Zig source with `zig fmt src/` before committing.
* Do not use emojis in code, output, or documentation. Unicode symbols (e.g. checkmark, arrow) are acceptable.
@@ -55,10 +54,12 @@ The test suites under `tests/` use Bun but are separate from the Zig codebase. S
* CLI flags use kebab-case (e.g. `--no-save`, `--json`). Never use camelCase for flags.
* Prefer `snake_case` for all Zig identifiers. Types use `PascalCase` per Zig convention.
* Follow Zig naming: `camelCase` for functions, `snake_case` for variables, fields, and file names, and `PascalCase` for types.
* Keep `pub` surface area minimal. Only mark declarations `pub` when they are used outside the file.
* Never commit personal home-directory paths, real team IDs, internal team slugs, or personal project slugs. `scripts/check-public-surface.sh` rejects them in CI. Use `/Users/example/` in fixtures.
## Architecture
Key rules:
@@ -73,6 +74,8 @@ Key rules:
* `src/core/shell_command/safe_git.zig` runs git for the `fx pr` and `fx issue` snapshot and for the agent's direct read-only `git status`, `git diff`, and `git log` when the command does not use the user's own shell environment. It turns off every program a repository's own config can name and refuses to run when git would use a different repository than the one fx read. In `auto` mode a git command it refuses is returned to the agent without running. Commands that run in a shell use git with the repository's full config: git in the shell tool's default `user` environment, and the routine commands `auto` mode runs without review, such as `git status` with other flags, `git fetch`, or `npm test`.
* `src/mcp/` is the MCP-v2 engine. `FX_MCP_ENGINE=v2` selects it in place of the default v1 client in `src/core/mcp/`. Its files import only `std` and each other, and the host passes in `std.Io` and every other service through `src/mcp/host.zig`. fx connects to it through `src/core/mcp_host/`.
* `src/tools/` owns built-in tool implementations. Generic tool contracts and dispatch live in `src/core/tooling/`. Default tool specs are centralized in `src/core/tooling/tool_specs.zig` or `src/builtins/tools.zig`, not in individual tool files.
* `src/ui/` owns terminal rendering, event loop, input, transcript. It must not own product state.
@@ -81,6 +84,8 @@ Key rules:
* `src/acp/` owns the ACP (Agent Client Protocol) JSON-RPC 2.0 server.
* `src/wasm_core_main.zig`, `src/wasm_term_main.zig`, and `src/napi_core_main.zig` are the libfx SDK entry points. Read `sdk/AGENTS.md` before changing them or anything under `sdk/`.
### Adding a Feature
Before implementing, answer in order:
@@ -113,9 +118,12 @@ its stale corpus entry. Normal PR CI loads the corpus and rejects missing,
duplicate, stale, or unclassified files without running the expensive PGSO
qualification.
Every root E2E file also needs a weight in `tests/e2e/ci-shard-weights.json`.
CI fails on a missing or stale entry.
### Adding a Command
1. Add the spec to `src/core/slash_commands/command_specs.zig`
1. Add the command kind in `src/core/slash_commands/command_specs.zig` and its spec, including usage, summary, and options, in `src/builtins/commands.zig`
2. Add dispatch wiring in `src/core/cli/cli_surface.zig`
3. Add a snapshot type if it has structured output
4. Render text and JSON from the same snapshot via `src/core/output/output_contracts.zig`
@@ -134,15 +142,15 @@ Config precedence (highest wins):
4. `<workspace>/.fx.json` (committed project defaults)
5. Built-in defaults
Project `.fx.json` accepts only repo-safe defaults: `sandbox`, `max_agent_steps`, `max_tool_result_bytes`, `context`, `provider_order`, and `provider_strict`. Profile-owned keys such as `provider`, `providers`, `models`, `model`, `effort`, `fast_mode`, `slash_menu_categories`, `startup_scrollback`, `prompt_history`, `statusLine`, `skill_match_fuzzy`, `first_call_tool_choice`, `auto_upgrade`, `auto_compact_percent`, `permission_mode`, `credential_source`, `permission`, and `skill_symlink_authorities` are ignored from project config before their values are parsed.
Project `.fx.json` accepts only repo-safe defaults: `max_agent_steps`, `max_tool_result_bytes`, `context`, `provider_order`, and `provider_strict`, parsed by `parseProjectSafeFields` in `src/core/config/config_runtime.zig`. The project layer never parses profile-owned keys, such as provider and model selection, permissions, and interface preferences.
Runtime state lives under `~/.fx/sessions/<session-id>/` (`session.json`, `background/`, `subagent/`, `logs/`). Sessions are global and portable across workspaces. Each session tracks its `workspace_root`, which updates when resumed in a different workspace. A subagent child is an internal ordinary session with its own history. Its parent owns one bounded `subagent/children.json` registry, and the child carries only an immutable owner marker. Child sessions stay out of ordinary session discovery and cannot be resumed directly. A first `subagent.message` creates a named persistent child in that parent; later messages continue it, and optional instructions replace only its child-specific system overlay.
Runtime state lives under `~/.fx/sessions/<session-id>/` (`session.json`, `background/`, `subagent/`, `logs/`). Sessions are global and portable across workspaces. Each session tracks its `workspace_root`, which updates when resumed in a different workspace. Subagent children are internal sessions registered in their parent's `subagent/children.json`. They stay out of session discovery and cannot be resumed directly.
## Permissions
Security is permission-first. All sensitive tool behavior must integrate with `src/core/permissions/permissions.zig`.
* `permission_mode` controls baseline (`ask`, `auto`, or `full-access`; `yolo` remains an alias). Full access bypasses fx permission policy and uses an effective sandbox of `none` without rewriting saved sandbox configuration
* `permission_mode` controls baseline (`ask`, `auto`, or `full-access`; `yolo` remains an alias). Full access bypasses fx permission policy
* Configured denies are evaluated before saved-session rules; an exact saved-session deny can narrow a configured allow, while an exact saved-session allow can satisfy an unresolved configured ask
@@ -162,41 +170,33 @@ Do not bypass the permission system for new tools.
## Zig-Specific Patterns
### Memory
### Memory and Errors
* Allocators are passed explicitly. Never use a global allocator.
* Free what you allocate. Use `defer` for cleanup at the call site.
* Prefer `ArenaAllocator` for request-scoped work that can be freed in bulk.
* When a function returns allocated memory, document who owns it (caller or callee).
### Error Handling
* Return errors rather than panicking. `@panic` is for programmer bugs, not runtime conditions.
* Use `errdefer` to clean up partial state on error paths.
* Prefer specific error sets over `anyerror` when the set is bounded.
### Strings and JSON
* Zig strings are `[]const u8`. There is no implicit null termination.
* Zig 0.17 removed the `**` operator. Fill arrays with `@splat` and build repeated comptime strings with `text_utils.repeat` from `src/core/shared/text_utils.zig`.
* For JSON serialization, use `std.json.Stringify.value` with an allocating writer (`std.Io.Writer.Allocating`).
* For JSON string escaping (writing raw JSON), use the project's `writeJsonStr` helper in `src/acp/jsonrpc.zig` rather than assuming `std.json.encodeJsonString` exists.
* Zig 0.16 uses `std.Io.File.stdin()` / `.stdout()` / `.stderr()`, not `std.io.getStdIn()`.
* Zig 0.17 uses `std.Io.File.stdin()` / `.stdout()` / `.stderr()`, not `std.io.getStdIn()`.
### I/O (Zig 0.16 "Juicy Main")
### I/O ("Juicy Main")
* `main` uses `pub fn main(init: std.process.Init) !void` signature.
* All I/O goes through `std.Io`, passed explicitly or via the project's `src/core/shared/io.zig` helper (`io_mod.getIo()`).
* File operations use `std.Io.Dir` and `std.Io.File` (not `std.fs`). Most methods require an `io` parameter.
* File operations use `std.Io.Dir` and `std.Io.File`. Most methods require an `io` parameter. Everything left in `std.fs` is deprecated: use `std.Io.Dir.path` for path manipulation and `std.Io.Dir.max_path_bytes` for path buffers.
* Environment variables: use `io_mod.getenv(key)` (returns `?[]const u8`), not `std.process.getEnvVarOwned`.
@@ -212,22 +212,42 @@ Do not bypass the permission system for new tools.
* HTTP: `std.http.Client` requires `.io = io_mod.getIo()` in its initializer.
* `std.mem` renames: `trimLeft` is `trimStart`, `trimRight` is `trimEnd`, `indexOf` is `find`, `indexOfScalar` is `findScalar`.
* `ArrayList(T)` initializes with `.empty` (not `.{}`).
### Zig 0.17 Standard Library
fx does not use declarations that Zig 0.17 deprecates. They still compile, so check new code against this list:
* Format with `alloc.print(fmt, args)` and `std.mem.print(buf, fmt, args)`, not `std.fmt.allocPrint` or `std.fmt.bufPrint`.
* Search with the `std.mem.find` family (`find`, `findScalar`, `findAny`, `findLast`, `findScalarLast`, and their `Pos` variants), not `indexOf` or `lastIndexOf`. Trim with `trimStart` and `trimEnd`.
* Copy overlapping slices with `@memmove(dest, src)`, not `std.mem.copyForwards` or `copyBackwards`. Both slices must have the same length.
* Resolve paths with `std.Io.Dir.path.resolveAlloc` and `relativeAlloc`.
* Read the target from `builtin.target.os`, `builtin.target.cpu`, and `builtin.target.abi`, and the optimization mode from `builtin.optimize` (`.debug`, `.safe`, `.fast`, `.small`). Gate safety-only work on `builtin.optimize.runtimeSafety()`. Language types live in `std.lang`, not `std.builtin`.
* Use `std.ArrayList`, `std.array_hash_map.String` and `.Auto`, and `std.bit_set.Dynamic` and `.Static`, not the `Unmanaged` or `BitSet` aliases. Read the last list item with `last()`.
* Convert enums with `@backingInt` and `@fromBackingInt`. Divide rounding up with `@divCeil`.
* `@hasDecl` only finds `pub` declarations, so a declaration that a duck-typed caller probes for must be `pub`.
* Decode one UTF-8 sequence with `display_width.decodeUtf8Sequence` from `src/core/shared/display_width.zig`, not `std.unicode.utf8Decode`.
### Testing
* Zig unit tests go inside the source file they test, using `test "description" { ... }` blocks.
* Run the narrowest relevant tests while developing. The complete `zig build test` suite runs in ReleaseSafe in **CI** after the feature branch is pushed, and it must pass before the draft PR is marked ready.
* Use `std.testing.expect`, `std.testing.expectEqual`, `std.testing.expectEqualStrings` for assertions.
* In test blocks, use `std.testing.io` for the `Io` parameter. `io_mod.getIo()` automatically returns `std.testing.io` in test builds.
* Use `io_mod.dirRealpathAlloc(alloc, dir, sub_path)` to resolve paths within `std.testing.tmpDir()`.
* Back `std.testing.checkAllAllocationFailures`, hand-written `FailingAllocator` sweeps, and allocation-count comparisons with `testing_allocator.no_resize` from `src/core/shared/testing_allocator.zig`. `std.testing.allocator` may grow an allocation in place on one run and not the next, which changes the allocation sequence and makes those tests flaky.
## Testing (TypeScript)
Two test suites live under `tests/`, both using Bun:
@@ -282,13 +302,13 @@ Do not run the complete deterministic test suite locally as the default developm
After the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. Linux is the gate for every change because most fx behavior is identical on every platform:
* `.github/workflows/ci.yml` runs on Linux x86_64. It checks formatting, the PGSO corpus, the public surface, and the compactor and indexer boundaries, then runs the ReleaseSafe unit tests. It builds fx once, smoke-tests it, and shares that binary with four duration-balanced E2E shards and the MCP conformance baseline. Checked-in weights assign every E2E file to exactly one shard, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one retry after its tmux server is reset. A file that passes only on retry gets a warning annotation; investigate it as a possible race. The SDK jobs build WASM and the Node-API addons once and run the complete package qualification. `Build & Test` passes only when the unit tests and every SDK job pass, because Publish libfx relies on that result instead of qualifying the package again.
* `.github/workflows/ci.yml` runs on Linux x86_64. It checks formatting, the PGSO corpus, the public surface, and the compactor and indexer boundaries, runs the ReleaseSafe unit tests, runs the E2E files in four weighted shards plus the MCP conformance baseline, and qualifies the SDK packages. A failed E2E file is retried once. A file that passes only on retry gets a warning annotation; investigate it as a possible race. `Build & Test` fails when any SDK job fails.
* `.github/workflows/binary-size.yml` builds each release target on its native runner, reports the size change, and smoke-tests every target's binary. See **Binary Size Observability**.
* `.github/workflows/bench.yml` enforces the startup latency budget. See **Benchmarks**.
Linux jobs install Zig through `.github/actions/setup-zig`, which restores the Zig cache that runs on `main` save with `.github/actions/save-zig-cache`. Pull requests never save a Zig cache through these actions, so they cannot push `main`'s entries out of the repository's cache quota. The PGSO workflow keeps its own cache through `.github/actions/setup-pgso`.
macOS runs only when a change can behave differently there. `.github/workflows/macos.yml` uses `scripts/detect-macos-need.sh` to check the diff. The macOS arm64 unit tests and the E2E files in `tests/e2e/macos-platform-tests.json` run when the change touches a Zig file with a macOS, BSD, or Linux code path, `build.zig`, the macOS signing script, the native SDK addon, a listed E2E file, a shared E2E helper that reads the platform, or the macOS checks themselves. A Linux-only branch counts because macOS takes its other path; a Windows-only branch does not. The listed E2E files run in two weighted shards that share one macOS build. Native SDK addon changes also build and load the addon on macOS. If the detector cannot read the change, the `macOS arm64` check fails instead of passing. To request macOS for any other change, add the `ci:macos` label or run `gh workflow run macos.yml --ref <branch>`. The label needs triage access and the dispatch needs write access, so an outside contributor asks in the PR and a maintainer starts the run. When the change does not need macOS, the `macOS arm64` check passes without starting a macOS runner. Every E2E file that branches on `process.platform` or `platform()` being `darwin` or `linux` must appear in `tests/e2e/macos-platform-tests.json`, and `tests/e2e/ci-shards.test.ts` enforces that.
macOS runs only when `scripts/detect-macos-need.sh` finds a change that can behave differently there, such as a Zig file with a macOS, BSD, or Linux code path, `build.zig`, the macOS signing script, the native SDK addon, or a platform E2E file. If the detector cannot read the change, the `macOS arm64` check fails instead of passing. When the change does not need macOS, the check passes without starting a macOS runner. To request macOS for any other change, add the `ci:macos` label or run `gh workflow run macos.yml --ref <branch>`. The label needs triage access and the dispatch needs write access, so an outside contributor asks in the PR and a maintainer starts the run. Every E2E file that branches on `process.platform` or `platform()` being `darwin` or `linux` must appear in `tests/e2e/macos-platform-tests.json`, and `tests/e2e/ci-shards.test.ts` enforces that.
The macOS arm64 PGSO workflow runs on a pull request only when the PGSO pipeline changes: `scripts/pgso/` other than `corpus.json`, the `setup-pgso` action, or `.github/workflows/pgso-macos-arm64.yml`. To qualify any other change, run `gh workflow run pgso-macos-arm64.yml --ref <branch>`. The stable release always runs it.
@@ -298,7 +318,7 @@ A CI result is valid only when it belongs to the exact current commit and `Build
## Reproducing Render Bugs
fx's rendering is inline by default and deliberately emits a small ANSI subset. Three owner classes are the narrow exceptions, and each takes the alternate buffer exclusively through `AlternateScreenOwner` in `src/ui/shell_runtime.zig`: interactive permission review, the full-transcript screen, and catalog menus. Only one class may own the buffer at a time, and each must leave it and restore the main grid, composer, cursor, paste, mouse, focus, and keyboard modes when it closes. Transcript rendering, question prompts, command-output expansion, and subagent delegation remain inline. Three tools exist for reproducing and regression-proofing render bugs:
fx's rendering is inline by default and deliberately emits a small ANSI subset. Three owner classes are the narrow exceptions, and each takes the alternate buffer exclusively through `AlternateScreenOwner` in `src/ui/shell_runtime.zig`: interactive file-approval review, the full-transcript screen, and catalog menus. Only one class may own the buffer at a time, and each must leave it and restore the main grid, composer, cursor, paste, mouse, focus, and keyboard modes when it closes. Transcript rendering, question prompts, command-output expansion, and subagent delegation remain inline. Three tools exist for reproducing and regression-proofing render bugs:
### tmux (live TTY repros)
@@ -348,7 +368,7 @@ Startup latency benchmarks live in `benchmarks/` and run in CI via `.github/work
./benchmarks/startup.sh --quick # quick run (20 iterations)
```
The CI workflow builds a ReleaseSafe binary, measures six CLI paths with hyperfine, and enforces per-command latency budgets. PRs that exceed a budget fail the check. On `main`, results are uploaded to Vercel Blob for historical tracking.
The `Startup Latency` job builds a ReleaseSafe binary, measures six CLI paths with hyperfine, and enforces per-command latency budgets. The same job runs `tests/e2e/tui-performance.test.ts` as the interactive terminal performance gate. A PR that exceeds either fails the check.
The long-turn memory and libfx runtime benchmarks run only on request. Start them with `gh workflow run bench.yml --ref <branch> -f long_turn_memory=true` or `-f libfx_runtime=true`.
@@ -356,29 +376,15 @@ The startup benchmark uses `FX_BENCH=1`, an environment variable that runs throu
To measure the interactive launch up to the first frame, use `benchmarks/first_frame.py`. It drives fx on a pseudo-terminal and interleaves several `--binary` arguments for before-and-after comparisons.
Current raw wall-clock contract:
* Linux CI: 2ms for every command
* Non-Linux local runs: informational raw means
The Linux CI runner is the authoritative product budget. Local macOS process
and dynamic-loader floors vary enough to exceed 2ms independently of fx, so
local runs report raw means without assigning a substitute product budget. The
process baseline is diagnostic only and is never subtracted.
The budget is 2ms of raw wall-clock time for every command on the Linux CI runner, which is the authoritative product budget. Local macOS runs report raw means only, because process and dynamic-loader overhead there can exceed 2ms independently of fx. The process baseline is diagnostic only and is never subtracted.
When adding features, consider their impact on startup latency. The `fx help` path is the baseline cold-start benchmark.
## Binary Size Observability
Every pull request runs `.github/workflows/binary-size.yml` across Linux x86_64,
Linux arm64, macOS x86_64, and macOS arm64. Each matrix job builds the pull
request merge commit and its base commit as stripped ReleaseSafe binaries on
the same native runner, then reports the exact byte and MiB delta plus ELF or
Mach-O section changes. The base binary is cached by base commit, so later
pushes to the same pull request reuse it until the base branch moves. Every job
also smoke-tests the pull request binary with `scripts/smoke-binary.sh`. These
are release-style builds, and the Linux arm64 and macOS binaries run nowhere
else on a pull request.
Every pull request runs `.github/workflows/binary-size.yml`, which builds each
release target and its base commit on native runners, reports the size change,
and smoke-tests the pull request binary with `scripts/smoke-binary.sh`.
Each platform check is informational. An increase of at least 52,429 bytes
(0.050000 MiB) emits a warning and retains that platform's binaries for
@@ -389,11 +395,15 @@ ceiling and performance gates.
## Documentation
When adding or changing user-facing features, update **all** relevant files:
When adding or changing user-facing behavior, update the documentation this repository owns in the same change:
1. `--help` output via command specs in `src/core/slash_commands/command_specs.zig`
2. `README.md` — feature descriptions, usage examples
3. `CONTRIBUTING.md` — if build steps, config, or collaboration rules change
1. `--help` output via command specs in `src/builtins/commands.zig`
2. `sdk/README.md` if libfx's supported public behavior or setup changes, as `sdk/AGENTS.md` describes
3. `CONTRIBUTING.md` if build steps, configuration, or collaboration rules change
The user manual at fx.sh/docs lives in `vercel-labs/fx-web`, an internal Vercel repository. A page's source is `apps/marketing/app/(site)/(docs)/docs/<path>/page.mdx`, where `<path>` is the part of its URL after `fx.sh/docs/`. If a change makes a page wrong or incomplete and you have access, update it in a companion fx-web pull request and link that pull request from this one. Otherwise, name the page in the PR description so a maintainer can update it. Do not move feature descriptions, option lists, or usage examples into `README.md` in place of the manual.
`README.md` is a short introduction that links to fx.sh/docs. Change it only when one of its statements becomes false, such as a command, flag, path, or link it names, or when a change adds a capability as broad as the ones in its feature list. A single new option, command, or environment variable belongs in `--help` and the manual, not in `README.md`.
Do not document intended behavior as if it already exists.
@@ -408,8 +418,7 @@ To prepare a release:
1. Create a branch (e.g. `prepare-v0.3.0`)
2. Bump `pub const version` in `src/main.zig`
3. Write the changelog entry in `CHANGELOG.md` at the top, under a new `## <version>` heading, wrapped in `<!-- release:start -->` and `<!-- release:end -->` markers. Remove the markers from the previous release entry so only the new release has them.
4. Update `README.md` install example version
5. Open a PR and merge to `main`
4. Open a PR and merge to `main`
When the PR merges, CI compares the version tag to what exists in git. If the tag is missing, it cross-compiles all platform binaries, creates the git tag, and publishes a GitHub Release with the binaries attached. The release body is extracted from the content between the `<!-- release:start -->` and `<!-- release:end -->` markers in `CHANGELOG.md`.
@@ -420,8 +429,9 @@ The changelog is public product copy. Describe observable user behavior, not the
Public changelog entries must:
* Spell the product name `fx`. Preserve different casing only when it is part of an exact code identifier such as `FX_MODEL`.
* Open with a short bold summary of the release.
* Use only relevant sections from `### Breaking Changes`, `### New Features`, `### Improvements`, `### Bug Fixes`, and `### Security`. Omit empty sections.
* Bold a short feature or fix name, then describe the user-visible change after a colon.
* Write each bullet as one or two plain sentences on a single line. Lead with the concrete change, then say what happens now. Do not add a bold `**Label:**` prefix.
* Omit pull request numbers, issue numbers, commit hashes, contributor names, and author attribution.
* Omit internal details such as repository moves, website or marketing work, CDN layout, CI workflows, test fixtures, branch history, and implementation-only refactors. Translate relevant work into its public user outcome or leave it out.
* Avoid forcing every merged change into the notes. A change without a public user outcome does not need a bullet.
@@ -432,19 +442,24 @@ Only the current release should have markers; remove `<!-- release:start -->` an
## 0.3.0
<!-- release:start -->
**`fx sessions` can now list saved sessions from every workspace.**
### New Features
- **Interactive terminal startup:** Start an interactive shell when the `terminal` tool receives an empty command
- `fx sessions --all` lists saved sessions across every workspace in the profile.
<!-- release:end -->
## 0.2.5
**The active conversation now stays in view.**
### Improvements
- **Inline rendering:** Keep the active conversation visible in terminal scrollback
- The active conversation stays visible in terminal scrollback.
```
Do not add a `### Contributors` section or tracker references. Use descriptive section names.
Do not add a `### Contributors` section or tracker references.
Do not create version tags manually. Do not change `build.zig.zon` version (it is a placeholder).
@@ -454,8 +469,6 @@ The canonical repository is `vercel-labs/fx` on GitHub. All URLs, links, and ref
## What Not To Do
* Do not grow `main.zig` with leaf feature logic
* Do not add hidden product state that only exists in the live shell
* Do not add a second execution path for the same feature without a clear reason
@@ -463,19 +476,3 @@ The canonical repository is `vercel-labs/fx` on GitHub. All URLs, links, and ref
* Do not commit generated state from `.fx/`, `.zig-cache/`, or `zig-out/`
* Do not add dependencies outside the Zig standard library without discussion
* Do not use `@import` with runtime-computed paths — Zig imports are comptime only
* Do not ignore `zig fmt` failures
* Do not create git tags manually (the release workflow owns tag creation)
* Do not report work as ready without running the binary. See **Declaring Work Ready**.
## Before Marking a PR Ready
1. Run `zig fmt --check src/` and the focused tests for the changed path.
2. Build and exercise the change locally with `./zig-out/bin/fx`.
3. Push a clean checkpoint commit and open a draft PR immediately.
4. Require **CI**, including the **macOS arm64** check, and the final ship gate to pass on the exact current commit.
5. Update docs if behavior changed.
CONTRIBUTING.md
+2 -2
View File
@@ -20,7 +20,7 @@ Contributions should preserve that direction:
Requirements:
* Zig `0.16.0+`
* Zig `0.17.0+`
* interactive terminal for manual shell testing
@@ -655,4 +655,4 @@ Minimum checklist:
2. Run `zig build`, then exercise the change with `./zig-out/bin/fx`.
3. Push the feature branch and open a draft PR immediately.
4. Require **CI**, including the **macOS arm64** check, and the final ship gate to pass for the exact current commit before marking the PR ready.
5. Update `README.md` if user-facing behavior changed.
5. If user-facing behavior changed, update the docs as [Documentation](AGENTS.md#documentation) describes.
README.md
+3 -1
View File
@@ -68,6 +68,8 @@ Each request runs as a loop. fx sends the model your message, the conversation s
Skill instructions and MCP tool schemas don't go to the model up front. They load when the model asks for them, which keeps the context small and leaves more room for your code.
Commands run by fx receive `AI_AGENT=fx` in their environment. Development tools can use it to detect fx.
fx prints every step inline as it happens and saves the session under `~/.fx/sessions/`, so you can resume it later.
## Use fx from other programs
@@ -101,7 +103,7 @@ Run `/feedback` inside fx to open the feedback form, or [open an issue](https://
## Contributing
You need [Zig 0.16.0](https://ziglang.org/download/) to build fx:
You need [Zig 0.17.0](https://ziglang.org/download/) to build fx:
```bash
git clone https://github.com/vercel-labs/fx.git
benchmarks/approval_review.zig
+3 -2
View File
@@ -40,8 +40,9 @@ const Scenario = enum {
edge,
fn parse(value: []const u8) !Scenario {
inline for (std.meta.fields(Scenario)) |field| {
if (std.mem.eql(u8, value, field.name)) return @enumFromInt(field.value);
const scenario_info = @typeInfo(Scenario).@"enum";
inline for (scenario_info.field_names, scenario_info.field_values) |field_name, field_value| {
if (std.mem.eql(u8, value, field_name)) return @fromBackingInt(@intCast(field_value));
}
return error.InvalidScenario;
}
benchmarks/file_index_bench.zig
+3 -3
View File
@@ -151,9 +151,9 @@ fn percentile(sorted: []const u64, fraction: f64) u64 {
fn formatNs(nanoseconds: u64, buf: []u8) []const u8 {
const microseconds = @as(f64, @floatFromInt(nanoseconds)) / 1_000.0;
const milliseconds = microseconds / 1_000.0;
if (nanoseconds < 1_000) return std.fmt.bufPrint(buf, "{d}ns", .{nanoseconds}) catch "?";
if (nanoseconds < 1_000_000) return std.fmt.bufPrint(buf, "{d:.2}us", .{microseconds}) catch "?";
return std.fmt.bufPrint(buf, "{d:.2}ms", .{milliseconds}) catch "?";
if (nanoseconds < 1_000) return std.mem.print(buf, "{d}ns", .{nanoseconds}) catch "?";
if (nanoseconds < 1_000_000) return std.mem.print(buf, "{d:.2}us", .{microseconds}) catch "?";
return std.mem.print(buf, "{d:.2}ms", .{milliseconds}) catch "?";
}
pub fn main(init: std.process.Init) !void {
benchmarks/libfx/fake-inference-server.zig
+1 -1
View File
@@ -88,7 +88,7 @@ fn serveConnectionFallible(io: std.Io, stream: std.Io.net.Stream) !void {
fn writeResponse(request: *std.http.Server.Request, protocol: Protocol, keep_alive: bool) !void {
const request_id = next_request_id.fetchAdd(1, .monotonic);
var request_id_buffer: [32]u8 = undefined;
const request_id_text = try std.fmt.bufPrint(&request_id_buffer, "{d}", .{request_id});
const request_id_text = try std.mem.print(&request_id_buffer, "{d}", .{request_id});
const headers = [_]std.http.Header{
.{ .name = "content-type", .value = "text/event-stream" },
.{ .name = "cache-control", .value = "no-cache" },
benchmarks/pgso_artifacts.test.ts
+1 -1
View File
@@ -348,7 +348,7 @@ function provenanceFixture(): [Provenance, Provenance] {
head_sha: sha, head_branch: branch, event, repository: { id: 1, full_name: "vercel-labs/fx" }, head_repository: { id: 1, full_name: "vercel-labs/fx" } },
source: { sha: source, parents: source === merge ? [{ sha: base }, { sha: head }] : [] },
manifest: { stage: "complete", status: "passed", eligible: true, evidence: {
identity: { source_sha: source, target: "aarch64-macos", host_arch: "arm64", zig_version: "0.16.0", llvm_version: "21.1.8" },
identity: { source_sha: source, target: "aarch64-macos", host_arch: "arm64", zig_version: "0.17.0", llvm_version: "22.1.8" },
profile: { sha256: "d".repeat(64) },
artifacts: { candidate: { sha256: "e".repeat(64), size: { size_bytes: 6_000_000 }, architecture: "arm64", signature_valid: true, version: "0.0.9", minimum_macos: "13.0" } },
} },
build.zig
+48 -34
View File
@@ -61,7 +61,7 @@ pub fn build(b: *std.Build) void {
.omit_frame_pointer = true,
.unwind_tables = .none,
.error_tracing = false,
.strip = optimize != .Debug,
.strip = optimize != .debug,
}),
});
exe.root_module.addImport("build_options", build_options.createModule());
@@ -71,9 +71,7 @@ pub fn build(b: *std.Build) void {
const run_cmd = b.addRunArtifact(exe);
run_cmd.step.dependOn(b.getInstallStep());
if (b.args) |args| {
run_cmd.addArgs(args);
}
run_cmd.addPassthruArgs();
const run_step = b.step("run", "Run fx");
run_step.dependOn(&run_cmd.step);
@@ -83,12 +81,12 @@ pub fn build(b: *std.Build) void {
});
const run_exe_tests = b.addRunArtifact(exe_tests);
run_exe_tests.step.dependOn(b.getInstallStep());
run_exe_tests.setEnvironmentVariable(
"FX_TEST_PRODUCT_EXE",
b.getInstallPath(.bin, "fx"),
);
// The session boundary test walks the source tree from here.
run_exe_tests.setEnvironmentVariable("FX_TEST_SOURCE_ROOT", b.pathFromRoot("src"));
// Tests launch the installed fx binary, and the session boundary test
// walks the source tree.
const test_paths = b.addOptions();
test_paths.addOptionPathUntracked("product_exe", b.graph.path(.install_bin, "fx"));
test_paths.addOptionPathUntracked("source_root", b.path("src"));
exe.root_module.addImport("test_paths", test_paths.createModule());
// The session manager's own tests, as fx compiles it (no hooks).
const session_manager_tests = b.addTest(.{ .root_module = session_manager });
@@ -131,12 +129,17 @@ pub fn build(b: *std.Build) void {
mcp_test_exports,
);
const run_mcp_dispatcher_e2e = b.addRunArtifact(mcp_dispatcher_e2e);
if (b.args) |args| run_mcp_dispatcher_e2e.addArgs(args);
run_mcp_dispatcher_e2e.addPassthruArgs();
const mcp_dispatcher_e2e_step = b.step(
"run-mcp-stdio-dispatcher-e2e",
"Run the MCP stdio dispatcher E2E driver",
);
mcp_dispatcher_e2e_step.dependOn(&run_mcp_dispatcher_e2e.step);
const build_mcp_dispatcher_e2e_step = b.step(
"build-mcp-stdio-dispatcher-e2e",
"Build the MCP stdio dispatcher E2E driver",
);
build_mcp_dispatcher_e2e_step.dependOn(&mcp_dispatcher_e2e.step);
// --- file_index search benchmark ---
const benchmark_exports_mod = b.createModule(.{
@@ -161,7 +164,7 @@ pub fn build(b: *std.Build) void {
const run_bench = b.addRunArtifact(file_index_bench);
run_bench.step.dependOn(&install_bench.step);
if (b.args) |args| run_bench.addArgs(args);
run_bench.addPassthruArgs();
const run_bench_step = b.step("run-bench-file-index", "Build and run file_index search benchmark");
run_bench_step.dependOn(&run_bench.step);
@@ -239,7 +242,7 @@ pub fn build(b: *std.Build) void {
const run_approval_review_bench = b.addRunArtifact(approval_review_bench);
run_approval_review_bench.step.dependOn(&install_approval_review_bench.step);
if (b.args) |args| run_approval_review_bench.addArgs(args);
run_approval_review_bench.addPassthruArgs();
const run_approval_review_bench_step = b.step(
"run-bench-approval-review",
"Build and run the file-diff approval review benchmark",
@@ -336,7 +339,7 @@ fn addWasmArtifact(
.root_module = b.createModule(.{
.root_source_file = b.path(wasm_root),
.target = wasm_target,
.optimize = .ReleaseSmall,
.optimize = .small,
.single_threaded = true,
.link_libc = true,
.stack_check = false,
@@ -349,7 +352,7 @@ fn addWasmArtifact(
});
if (surface == .core) wasm_exe.stack_size = 1024 * 1024;
wasm_exe.root_module.addImport("build_options", wasm_options.createModule());
_ = addSessionManager(b, wasm_exe.root_module, wasm_target, .ReleaseSmall, true);
_ = addSessionManager(b, wasm_exe.root_module, wasm_target, .small, true);
const install_wasm = b.addInstallArtifact(wasm_exe, .{});
const wasm_step = b.step(name ++ "-wasm", description);
@@ -381,19 +384,25 @@ fn addNapiArtifact(
.root_module = b.createModule(.{
.root_source_file = b.path(root),
.target = target,
.optimize = .ReleaseSafe,
.optimize = .safe,
.link_libc = true,
.strip = true,
}),
});
lib.root_module.addImport("build_options", napi_options.createModule());
_ = addSessionManager(b, lib.root_module, target, .ReleaseSafe, null);
_ = addSessionManager(b, lib.root_module, target, .safe, null);
const node_include = b.option(
[]const u8,
"node-include-dir",
"Directory containing node_api.h for the N-API addon",
) orelse discoverNodeIncludeDir(b);
lib.root_module.addSystemIncludePath(.{ .cwd_relative = node_include });
const node_api = b.addTranslateC(.{
.root_source_file = .{ .cwd_relative = b.pathJoin(&.{ node_include, "node_api.h" }) },
.target = target,
.optimize = .safe,
});
node_api.addSystemIncludePath(.{ .cwd_relative = node_include });
lib.root_module.addImport("node_api", node_api.createModule());
lib.linker_allow_shlib_undefined = true;
const install = b.addInstallArtifact(lib, .{ .dest_sub_path = "libfx.node" });
@@ -403,25 +412,28 @@ fn addNapiArtifact(
}
fn discoverNodeIncludeDir(b: *std.Build) []const u8 {
var code: u8 = 0;
const out = b.runAllowFail(
const out = switch (b.runFallible(
&.{ "node", "-p", "require('node:path').join(require('node:path').dirname(process.execPath), '..', 'include', 'node')" },
&code,
.ignore,
) catch std.process.fatal("Node.js is required to locate node_api.h; pass -Dnode-include-dir=<path>", .{});
if (code != 0) std.process.fatal("could not locate node_api.h; pass -Dnode-include-dir=<path>", .{});
.{ .stderr_behavior = .ignore },
)) {
.success => |stdout| stdout,
.spawn_failed, .crashed => std.process.fatal("Node.js is required to locate node_api.h; pass -Dnode-include-dir=<path>", .{}),
.bad_exit_code => std.process.fatal("could not locate node_api.h; pass -Dnode-include-dir=<path>", .{}),
};
const trimmed = std.mem.trim(u8, out, " \t\r\n");
return b.allocator.dupe(u8, trimmed) catch std.process.fatal("could not allocate Node include path", .{});
}
fn readGitCommit(b: *std.Build) []const u8 {
var code: u8 = 0;
const out = b.runAllowFail(
// The configure cache cannot see a new commit, so rerun configure.
b.graph.poisonCache();
const out = switch (b.runFallible(
&.{ "git", "rev-parse", "--short=12", "HEAD" },
&code,
.ignore,
) catch return "unknown";
if (code != 0) return "unknown";
.{ .stderr_behavior = .ignore },
)) {
.success => |stdout| stdout,
.spawn_failed, .bad_exit_code, .crashed => return "unknown",
};
const trimmed = std.mem.trim(u8, out, " \t\r\n");
return b.allocator.dupe(u8, trimmed) catch "unknown";
}
@@ -433,13 +445,13 @@ fn addSessionManager(
b: *std.Build,
importer: *std.Build.Module,
target: std.Build.ResolvedTarget,
optimize: std.builtin.OptimizeMode,
optimize: std.lang.Optimize,
single_threaded: ?bool,
) *std.Build.Module {
const options = b.addOptions();
options.addOption(bool, "hooks", false);
options.addOption([]const u8, "trace_dir", b.getInstallPath(.prefix, "session-manager-traces"));
options.addOption([]const u8, "src_dir", b.pathFromRoot("src/core/session_manager"));
options.addOptionPathUntracked("trace_dir", b.graph.path(.install_prefix, "session-manager-traces"));
options.addOptionPathUntracked("src_dir", b.path("src/core/session_manager"));
const module = b.createModule(.{
.root_source_file = b.path("src/core/session_manager/api.zig"),
.target = target,
@@ -453,7 +465,9 @@ fn addSessionManager(
}
fn readAppVersion(b: *std.Build) []const u8 {
const bytes = std.Io.Dir.cwd().readFileAlloc(b.graph.io, "src/main.zig", b.allocator, .limited(1024 * 1024)) catch
b.dependOnFileContents(b.path("src/main.zig"));
const main_path = b.pathJoin(&.{ b.root.sub_path, "src/main.zig" });
const bytes = b.root.root_dir.handle.readFileAlloc(b.graph.io, main_path, b.allocator, .limited(1024 * 1024)) catch
@panic("could not read src/main.zig to resolve app version");
defer b.allocator.free(bytes);
build.zig.zon
+1 -1
View File
@@ -25,7 +25,7 @@
.fingerprint = 0x2ca027d00bcd652c, // Changing this has security and trust implications.
// Tracks the earliest Zig version that the package considers to be a
// supported use case.
.minimum_zig_version = "0.16.0",
.minimum_zig_version = "0.17.0",
// This field is optional.
// Each dependency must either provide a `url` and `hash`, or a `path`.
// `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
scripts/check-compactor-boundary.sh
+1 -1
View File
@@ -19,7 +19,7 @@ if [[ -n "$reaching_in" ]]; then
exit 1
fi
allowed='^\.\./(shared/(types|debug_trace|token_estimate|text_utils|io|history_range)|config/(model_capabilities|model_provider))\.zig$'
allowed='^\.\./(shared/(types|debug_trace|token_estimate|text_utils|io|history_range|testing_allocator)|config/(model_capabilities|model_provider))\.zig$'
reaching_out=""
while IFS= read -r line; do
target="${line#*@import(\"}"
scripts/pgso/README.md
+6 -6
View File
@@ -10,8 +10,8 @@ The driver requires:
- macOS on an arm64 host
- generic `aarch64-macos` output
- Zig `0.16.0`
- LLVM `21.1.8` tools and profile runtime from one configured LLVM root
- Zig `0.17.0`
- LLVM `22.1.8` tools and profile runtime from one configured LLVM root
- the selected Xcode macOS SDK and native Apple linker with arm64 support
- Bun `1.3.14`
- Hyperfine `1.20.0`
@@ -37,18 +37,18 @@ output fail closed.
Every mutating command requires a fresh or empty output directory. State from separate runs is never merged implicitly.
```bash
brew install llvm@21
brew install llvm@22
python3 -m scripts.pgso build \
--llvm-bin "$(brew --prefix llvm@21)/bin" \
--llvm-bin "$(brew --prefix llvm@22)/bin" \
--output-dir /tmp/fx-pgso-build
python3 -m scripts.pgso train \
--llvm-bin "$(brew --prefix llvm@21)/bin" \
--llvm-bin "$(brew --prefix llvm@22)/bin" \
--output-dir /tmp/fx-pgso-train
python3 -m scripts.pgso all \
--llvm-bin "$(brew --prefix llvm@21)/bin" \
--llvm-bin "$(brew --prefix llvm@22)/bin" \
--output-dir /tmp/fx-pgso-candidate \
--target aarch64-macos \
--update-channel stable \
scripts/pgso/distributed.py
+5 -8
View File
@@ -29,10 +29,8 @@ from scripts.pgso.model import (
)
from scripts.pgso.pipeline import (
GENERATION_FLAGS,
ArtifactSpec,
PipelinePaths,
apply_profile,
emit_bitcode,
link_candidate,
merge_profile_batch,
verify_candidate,
@@ -883,12 +881,11 @@ def run_candidate(arguments: argparse.Namespace) -> pathlib.Path:
paths,
)
spec = ArtifactSpec(
repo_root=REPO_ROOT,
target=identity.target,
update_channel=identity.update_channel,
)
emit_bitcode(toolchain, spec, paths, expected_sha256=identity.bitcode_sha256)
# Zig 0.17 can number anonymous symbols differently on another runner, so
# rebuilding the same commit is not byte-identical. Profile the seed's
# verified bitcode, which the training profiles were collected from.
shutil.copy2(seed_paths.bitcode, paths.bitcode)
_require_hash(paths.bitcode, identity.bitcode_sha256, "candidate bitcode")
apply_profile(toolchain, paths, identity.bitcode_sha256)
link_candidate(toolchain, paths)
linked_benchmarks = relink_profile_linked_benchmarks(
scripts/pgso/pipeline.py
+58 -10
View File
@@ -19,12 +19,23 @@ from scripts.pgso.runner import hermetic_environment, run_checked
from scripts.pgso.toolchain import SUPPORTED_TARGET, Toolchain
# Zig 0.17 emits array bitcasts as per-element loops that the optimizer later
# folds into single loads. Instrumentation runs before loop unrolling, so it
# would count every element. Unroll constant-trip loops first, in both the
# generation and use pipelines, so their CFG hashes keep matching. Unrolling
# without the surrounding instcombine and simplifycfg left the ui-activity
# benchmark about 11% slower than a plain O2 build.
PROFILE_PREPARATION_PASSES = (
"function(sroa,instcombine<no-verify-fixpoint>,loop-unroll-full,"
"instcombine<no-verify-fixpoint>,simplifycfg)"
)
GENERATION_FLAGS = (
"--disable-vp",
"--runtime-counter-relocation",
"--pgo-temporal-instrumentation",
"-pgo-kind=pgo-instr-gen-pipeline",
"-passes=default<O2>",
f"-passes={PROFILE_PREPARATION_PASSES},default<O2>",
)
# Apply the profile before partitioning so every part inherits the accepted
@@ -34,7 +45,7 @@ USE_FLAGS = (
"-pgo-kind=pgo-instr-use-pipeline",
"-pgo-cold-func-opt=minsize",
"-profile-summary-cutoff-cold=600000",
"-passes=default<O2>,mergefunc",
f"-passes={PROFILE_PREPARATION_PASSES},default<O2>,mergefunc",
)
OUTLINE_PARTITIONS = 2
@@ -53,7 +64,7 @@ BENCHMARK_USE_FLAGS = (
"-pgo-kind=pgo-instr-use-pipeline",
"-pgo-cold-func-opt=minsize",
"-profile-summary-cutoff-cold=990000",
"-passes=default<O2>,mergefunc,iroutliner",
f"-passes={PROFILE_PREPARATION_PASSES},default<O2>,mergefunc,iroutliner",
)
FX_MACHINE_OUTLINER_FLAGS = (
@@ -333,13 +344,18 @@ def zig_build_argv(
str(prefix),
"--cache-dir",
str(cache),
"--global-cache-dir",
str(paths.global_cache),
)
)
return tuple(argv)
def zig_build_env(paths: PipelinePaths) -> dict[str, str]:
# `zig build` takes its global cache only from the environment.
environment = os.environ.copy()
environment["ZIG_GLOBAL_CACHE_DIR"] = str(paths.global_cache)
return environment
def instrumentation_argv(
toolchain: Toolchain,
paths: PipelinePaths,
@@ -487,11 +503,31 @@ def candidate_runtime_probe_argv(
return (*command[:2], "-###", *command[2:])
_STUB_TARGET_LIST = re.compile(r"(targets:\s*)\[([^\]]*)\]")
def apple_ld_system_stub(text: str) -> str:
# Zig 0.17's macOS 27 stub lists arm64e.x1 targets, which the Xcode 16.4
# linker rejects as an unknown architecture. An arm64 link never uses them.
def drop_arm64e_x1(match: re.Match[str]) -> str:
targets = [target.strip() for target in match.group(2).split(",")]
kept = [t for t in targets if t and not t.startswith("arm64e.x1-")]
if not kept:
raise PgsoError("system stub section targets only arm64e.x1")
return f"{match.group(1)}[ {', '.join(kept)} ]"
result = _STUB_TARGET_LIST.sub(drop_arm64e_x1, text)
if "arm64e.x1" in result:
raise PgsoError("system stub names arm64e.x1 outside a target list")
return result
def temporal_candidate_link_argv(
toolchain: Toolchain,
paths: PipelinePaths,
compiler_runtime: pathlib.Path,
contract: MacosLinkContract,
system_stub: pathlib.Path,
) -> tuple[str, ...]:
return (
str(toolchain.apple_ld),
@@ -516,7 +552,7 @@ def temporal_candidate_link_argv(
str(paths.logs / "candidate-link.map"),
str(paths.profile_use_object),
str(compiler_runtime),
str(toolchain.zig_darwin_sdk / "libSystem.tbd"),
str(system_stub),
"-o",
str(paths.candidate_binary),
)
@@ -794,7 +830,7 @@ def build_control(
run_checked(
zig_build_argv(toolchain, spec, paths, emit_ir=False),
cwd=spec.repo_root,
env=os.environ.copy(),
env=zig_build_env(paths),
timeout_s=900,
log_path=paths.logs / "build-control.json",
)
@@ -812,7 +848,7 @@ def emit_bitcode(
run_checked(
zig_build_argv(toolchain, spec, paths, emit_ir=True),
cwd=spec.repo_root,
env=os.environ.copy(),
env=zig_build_env(paths),
timeout_s=900,
log_path=paths.logs / "emit-bitcode.json",
)
@@ -1308,6 +1344,13 @@ def _link_temporal_candidate(toolchain: Toolchain, paths: PipelinePaths) -> None
system_stub = toolchain.zig_darwin_sdk / "libSystem.tbd"
_require_nonempty_file(system_stub, "pinned Zig system library stub")
stub_hash = sha256_file(system_stub)
linked_stub = paths.candidate_binary.parent / "system-stub" / "libSystem.tbd"
linked_stub.parent.mkdir(parents=True, exist_ok=True)
linked_stub.write_text(
apple_ld_system_stub(system_stub.read_text(encoding="utf-8")),
encoding="utf-8",
)
linked_stub_hash = sha256_file(linked_stub)
order_path = paths.logs / "candidate-profile.order"
run_checked(
(str(toolchain.llvm_profdata), "order", str(paths.merged_profile), "-o", str(order_path)),
@@ -1340,7 +1383,7 @@ def _link_temporal_candidate(toolchain: Toolchain, paths: PipelinePaths) -> None
json.dumps(mapping, indent=2) + "\n", encoding="utf-8",
)
result = run_checked(
temporal_candidate_link_argv(toolchain, paths, runtime_object, contract),
temporal_candidate_link_argv(toolchain, paths, runtime_object, contract, linked_stub),
cwd=paths.root,
env=os.environ.copy(),
timeout_s=900,
@@ -1360,7 +1403,11 @@ def _link_temporal_candidate(toolchain: Toolchain, paths: PipelinePaths) -> None
)
validate_macos_link_contract(linked_contract, contract)
validate_archive_unchanged(runtime, runtime_hash)
if sha256_file(runtime_object) != runtime_object_hash or sha256_file(system_stub) != stub_hash:
if (
sha256_file(runtime_object) != runtime_object_hash
or sha256_file(system_stub) != stub_hash
or sha256_file(linked_stub) != linked_stub_hash
):
raise PgsoError("candidate runtime object or system stub changed during link")
evidence = {
"linker": "apple-ld",
@@ -1373,6 +1420,7 @@ def _link_temporal_candidate(toolchain: Toolchain, paths: PipelinePaths) -> None
"sysroot": str(toolchain.sdk),
"system_stub": str(system_stub),
"system_stub_sha256": stub_hash,
"linked_system_stub_sha256": linked_stub_hash,
"runtime_archive_sha256": runtime_hash,
"runtime_object_sha256": runtime_object_hash,
"profile_sha256": sha256_file(paths.merged_profile),
scripts/pgso/tests/test_cli.py
+2 -2
View File
@@ -158,8 +158,8 @@ class PgsoCliTests(unittest.TestCase):
class ToolchainInfo:
host_arch: str = "arm64"
target: str = "aarch64-macos"
zig_version: str = "0.16.0"
llvm_version: str = "21.1.8"
zig_version: str = "0.17.0"
llvm_version: str = "22.1.8"
@dataclasses.dataclass(frozen=True)
class CorpusInfo:
scripts/pgso/tests/test_distributed.py
+2 -2
View File
@@ -337,8 +337,8 @@ class ShardAggregationTests(unittest.TestCase):
source_sha="a" * 40,
target="aarch64-macos",
host_arch="arm64",
zig_version="0.16.0",
llvm_version="21.1.8",
zig_version="0.17.0",
llvm_version="22.1.8",
bitcode_sha256="b" * 64,
corpus_sha256="c" * 64,
update_channel="stable",
scripts/pgso/tests/test_model.py
+3 -3
View File
@@ -23,8 +23,8 @@ class PgsoModelTests(unittest.TestCase):
source_sha="a" * 40,
target="aarch64-macos",
host_arch="arm64",
zig_version="0.16.0",
llvm_version="21.1.8",
zig_version="0.17.0",
llvm_version="22.1.8",
bitcode_sha256="b" * 64,
corpus_sha256="c" * 64,
update_channel="stable",
@@ -44,7 +44,7 @@ class PgsoModelTests(unittest.TestCase):
"source_sha": "d" * 40,
"target": "x86_64-macos",
"host_arch": "x86_64",
"zig_version": "0.16.1",
"zig_version": "0.17.1",
"llvm_version": "21.1.9",
"bitcode_sha256": "e" * 64,
"corpus_sha256": "f" * 64,
scripts/pgso/tests/test_pipeline.py
+50 -8
View File
@@ -22,6 +22,7 @@ from scripts.pgso.pipeline import (
MacosLinkContract,
PipelinePaths,
apply_profile,
apple_ld_system_stub,
cleanup_outlined_ir_argv,
candidate_object_argv,
candidate_link_argv,
@@ -49,6 +50,7 @@ from scripts.pgso.pipeline import (
split_ir_argv,
verify_release_safe_ir,
zig_build_argv,
zig_build_env,
)
from scripts.pgso.toolchain import Toolchain
@@ -154,11 +156,40 @@ class PgsoPipelineTests(unittest.TestCase):
with self.assertRaises(PgsoError):
validate_temporal_link_map(link_map, self.root / "other.o", ("_alpha",))
def test_apple_ld_system_stub_drops_only_arm64e_x1_targets(self) -> None:
stub = (
"--- !tapi-tbd\n"
"targets: [ x86_64-macos, arm64e-macos, \n"
" arm64e.x1-macos, arm64e.x1-maccatalyst ]\n"
"install-name: '/usr/lib/libSystem.B.dylib'\n"
"exports:\n"
" - targets: [ arm64e.x1-macos, arm64e-macos ]\n"
" symbols: [ _a, _b ]\n"
)
self.assertEqual(
"--- !tapi-tbd\n"
"targets: [ x86_64-macos, arm64e-macos ]\n"
"install-name: '/usr/lib/libSystem.B.dylib'\n"
"exports:\n"
" - targets: [ arm64e-macos ]\n"
" symbols: [ _a, _b ]\n",
apple_ld_system_stub(stub),
)
older = "targets: [ x86_64-macos, arm64e-macos ]\n"
self.assertEqual(older, apple_ld_system_stub(older))
for invalid in (
" - targets: [ arm64e.x1-macos, arm64e.x1-maccatalyst ]\n",
" symbols: [ _arm64e.x1 ]\n",
):
with self.subTest(invalid=invalid), self.assertRaises(PgsoError):
apple_ld_system_stub(invalid)
def test_temporal_link_preserves_the_control_platform_and_original_object(self) -> None:
runtime = self.root / "libcompiler_rt_zcu.o"
contract = MacosLinkContract(1, "13.3", "26.4", 16 * 1024 * 1024, ())
toolchain = dataclasses.replace(self.toolchain, sdk_version="15.5")
command = temporal_candidate_link_argv(toolchain, self.paths, runtime, contract)
stub = self.root / "linked" / "libSystem.tbd"
command = temporal_candidate_link_argv(toolchain, self.paths, runtime, contract, stub)
self.assertEqual(str(self.toolchain.apple_ld), command[0])
platform = command.index("-platform_version")
self.assertEqual(("macos", "13.3", "26.4"), command[platform + 1:platform + 4])
@@ -167,7 +198,8 @@ class PgsoPipelineTests(unittest.TestCase):
self.assertNotIn(str(self.paths.instrumented_object), command)
self.assertNotIn(str(self.toolchain.profile_runtime), command)
self.assertEqual("1000000", command[command.index("-stack_size") + 1])
self.assertIn(str(toolchain.zig_darwin_sdk / "libSystem.tbd"), command)
self.assertIn(str(stub), command)
self.assertNotIn(str(toolchain.zig_darwin_sdk / "libSystem.tbd"), command)
self.assertIn(str(self.paths.logs / "candidate-order.txt"), command)
for flag in ("-order_file", "-no_deduplicate", "-no_function_starts", "-map"):
self.assertIn(flag, command)
@@ -224,8 +256,8 @@ class PgsoPipelineTests(unittest.TestCase):
"zig_darwin_sdk": self.root / "ZigDarwin.sdk",
"zig_sdk_version": "26.4",
"profile_runtime": self.root / "libclang_rt.profile_osx.a",
"zig_version": "0.16.0",
"llvm_version": "21.1.8",
"zig_version": "0.17.0",
"llvm_version": "22.1.8",
"target": "aarch64-macos",
"host_arch": "arm64",
}
@@ -245,7 +277,7 @@ class PgsoPipelineTests(unittest.TestCase):
"--runtime-counter-relocation",
"--pgo-temporal-instrumentation",
"-pgo-kind=pgo-instr-gen-pipeline",
"-passes=default<O2>",
"-passes=function(sroa,instcombine<no-verify-fixpoint>,loop-unroll-full,instcombine<no-verify-fixpoint>,simplifycfg),default<O2>",
),
GENERATION_FLAGS,
)
@@ -255,7 +287,7 @@ class PgsoPipelineTests(unittest.TestCase):
"-pgo-kind=pgo-instr-use-pipeline",
"-pgo-cold-func-opt=minsize",
"-profile-summary-cutoff-cold=600000",
"-passes=default<O2>,mergefunc",
"-passes=function(sroa,instcombine<no-verify-fixpoint>,loop-unroll-full,instcombine<no-verify-fixpoint>,simplifycfg),default<O2>,mergefunc",
),
USE_FLAGS,
)
@@ -274,7 +306,7 @@ class PgsoPipelineTests(unittest.TestCase):
"-pgo-kind=pgo-instr-use-pipeline",
"-pgo-cold-func-opt=minsize",
"-profile-summary-cutoff-cold=990000",
"-passes=default<O2>,mergefunc,iroutliner",
"-passes=function(sroa,instcombine<no-verify-fixpoint>,loop-unroll-full,instcombine<no-verify-fixpoint>,simplifycfg),default<O2>,mergefunc,iroutliner",
),
BENCHMARK_USE_FLAGS,
)
@@ -411,6 +443,11 @@ class PgsoPipelineTests(unittest.TestCase):
control[control.index("--cache-dir") + 1],
ir[ir.index("--cache-dir") + 1],
)
self.assertNotIn("--global-cache-dir", control)
self.assertEqual(
str(self.paths.global_cache),
zig_build_env(self.paths)["ZIG_GLOBAL_CACHE_DIR"],
)
def test_benchmark_artifacts_use_their_existing_build_owners_and_names(self) -> None:
cases = (
@@ -609,6 +646,7 @@ pathlib.Path(sys.argv[sys.argv.index('-map') + 1]).write_bytes({link_map!r}.enco
toolchain, self.paths,
self.paths.candidate_binary.parent / "compiler-runtime" / "libcompiler_rt_zcu.o",
self.good_link_contract(),
self.paths.candidate_binary.parent / "system-stub" / "libSystem.tbd",
)[1:]),
f"strip -S -x {self.paths.candidate_binary}",
"codesign --force --sign - --options linker-signed "
@@ -621,6 +659,10 @@ pathlib.Path(sys.argv[sys.argv.index('-map') + 1]).write_bytes({link_map!r}.enco
layout = json.loads((self.paths.logs / "candidate-layout.json").read_text())
self.assertEqual("26.4", layout["sdk_version"])
self.assertEqual("15.5", layout["sysroot_sdk_version"])
self.assertEqual(
sha256_file(self.paths.candidate_binary.parent / "system-stub" / "libSystem.tbd"),
layout["linked_system_stub_sha256"],
)
self.assertEqual(16777216, layout["main_stack_size"])
self.assertEqual(8, layout["ordered_bytes"])
self.assertIn(
@@ -778,7 +820,7 @@ print('_main T ---------------- 0')""",
self.assertEqual(b"bitcode", result.read_bytes())
lines = actions.read_text().splitlines()
self.assertEqual(8, len(lines))
self.assertIn("-passes=default<O2>,mergefunc", lines[0])
self.assertIn("-passes=function(sroa,instcombine<no-verify-fixpoint>,loop-unroll-full,instcombine<no-verify-fixpoint>,simplifycfg),default<O2>,mergefunc", lines[0])
self.assertNotIn("iroutliner", lines[0])
self.assertEqual("nm", lines[1])
self.assertTrue(lines[2].startswith("split -j 2 -o "))
scripts/pgso/tests/test_toolchain.py
+10 -10
View File
@@ -39,7 +39,7 @@ class PgsoToolchainTests(unittest.TestCase):
(self.zig_sdk / "SDKSettings.json").write_text('{"MinimalDisplayName":"26.4"}')
zig_env = f'.{{\n .lib_dir = {json.dumps(str(self.zig_lib))},\n}}\n'
self.write_executable(self.zig, f"""case "$1" in
version) printf '0.16.0\\n' ;;
version) printf '0.17.0\\n' ;;
env) printf '%s' {shlex.quote(zig_env)} ;;
*) exit 2 ;;
esac""")
@@ -74,13 +74,13 @@ esac""",
path.write_text(f"#!/bin/sh\n{body}\n")
path.chmod(0o755)
def write_llvm_tool(self, name: str, version: str = "21.1.8") -> None:
def write_llvm_tool(self, name: str, version: str = "22.1.8") -> None:
self.write_executable(
self.llvm_bin / name,
f"printf 'LLVM version {version}\\n'",
)
def write_clang(self, version: str = "21.1.8") -> None:
def write_clang(self, version: str = "22.1.8") -> None:
body = f"""case "$1" in
--print-resource-dir) printf '%s\\n' {shlex.quote(str(self.resource_dir))} ;;
--version) printf 'clang version {version}\\n' ;;
@@ -136,8 +136,8 @@ esac"""
self.assertEqual(self.zig_sdk.resolve(), toolchain.zig_darwin_sdk)
self.assertEqual("26.4", toolchain.zig_sdk_version)
self.assertEqual(self.profile_runtime.resolve(), toolchain.profile_runtime)
self.assertEqual("0.16.0", toolchain.zig_version)
self.assertEqual("21.1.8", toolchain.llvm_version)
self.assertEqual("0.17.0", toolchain.zig_version)
self.assertEqual("22.1.8", toolchain.llvm_version)
self.assertEqual("aarch64-macos", toolchain.target)
self.assertEqual("arm64", toolchain.host_arch)
@@ -166,15 +166,15 @@ esac"""
)
def test_discover_rejects_the_wrong_zig_version(self) -> None:
self.write_executable(self.zig, "printf '0.16.1\\n'")
self.write_executable(self.zig, "printf '0.17.1\\n'")
with self.assertRaisesRegex(PgsoError, "requires Zig 0.16.0"):
with self.assertRaisesRegex(PgsoError, "requires Zig 0.17.0"):
self.discover()
def test_discover_rejects_a_mixed_llvm_version(self) -> None:
self.write_llvm_tool("llc", version="21.1.9")
with self.assertRaisesRegex(PgsoError, "requires LLVM 21.1.8"):
with self.assertRaisesRegex(PgsoError, "requires LLVM 22.1.8"):
self.discover()
def test_discover_rejects_a_missing_executable(self) -> None:
@@ -185,7 +185,7 @@ esac"""
def test_discover_rejects_an_llvm_tool_outside_the_root(self) -> None:
outside = self.root / "outside-opt"
self.write_executable(outside, "printf 'LLVM version 21.1.8\\n'")
self.write_executable(outside, "printf 'LLVM version 22.1.8\\n'")
(self.llvm_bin / "opt").unlink()
(self.llvm_bin / "opt").symlink_to(outside)
@@ -223,7 +223,7 @@ esac"""
def test_discover_rejects_unusable_zig_library_metadata(self) -> None:
for env_text in (".{}", '.{\n.lib_dir = "relative",\n}', '.{\n.lib_dir = "a",\n.lib_dir = "b",\n}'):
self.write_executable(self.zig, f"""if [ "$1" = version ]; then printf '0.16.0\\n'; else printf '%s' {shlex.quote(env_text)}; fi""")
self.write_executable(self.zig, f"""if [ "$1" = version ]; then printf '0.17.0\\n'; else printf '%s' {shlex.quote(env_text)}; fi""")
with self.subTest(env_text=env_text), self.assertRaises(PgsoError):
self.discover()
scripts/pgso/toolchain.py
+2 -2
View File
@@ -12,8 +12,8 @@ import subprocess
from scripts.pgso.model import PgsoError
REQUIRED_ZIG_VERSION = "0.16.0"
REQUIRED_LLVM_VERSION = "21.1.8"
REQUIRED_ZIG_VERSION = "0.17.0"
REQUIRED_LLVM_VERSION = "22.1.8"
SUPPORTED_TARGET = "aarch64-macos"
sdk/README.md
+71 -4
View File
@@ -40,7 +40,7 @@ The full documentation is at [fx.sh/docs/lib](https://fx.sh/docs/lib).
- [Steers](https://fx.sh/docs/lib/steering) a turn while it runs
- Takes [images](https://fx.sh/docs/lib/images) in prompts and tool results
- Gives each conversation its own model, instructions and context
- Runs on AI Gateway, or on a user's [ChatGPT plan](#chatgpt-plans) through Codex
- Runs on AI Gateway, on a user's own [ChatGPT](#chatgpt-plans) or [Grok](#grok-plans) plan, or on [any OpenAI-compatible server](#custom-providers)
- Uses a native addon on macOS and Linux, and WebAssembly everywhere else, including browsers
- Puts the [fx terminal](https://fx.sh/docs/lib/webassembly#embed-the-terminal) in a web page
- Ships TypeScript types for every entry point, with no runtime dependencies
@@ -217,12 +217,20 @@ An agent can run on a user's ChatGPT plan through Codex instead of AI Gateway. S
const agent = createFxAgent({
provider: "codex",
model: "gpt-6.1-sol",
chatgpt: ({ sessionId }) => tokens.load(sessionId),
onChatgptCredentials: (credentials, { sessionId }) => tokens.save(sessionId, credentials),
credentials: ({ provider, sessionId }) => tokens.load(provider, sessionId),
onCredentials: (credentials, { provider, sessionId }) => tokens.save(provider, sessionId, credentials),
});
```
libfx calls `chatgpt()` before Codex requests, in whichever process runs the turn, and adds the credentials to those requests itself, so they never reach the engine or a saved session. It refreshes them shortly before they expire and once after a rejected request, then passes the new ones to `onChatgptCredentials` for you to save. Codex doesn't accept requests from web pages, so these turns run on your server.
libfx calls `credentials()` before Codex requests, in whichever process runs the turn, and adds the credentials to those requests itself, so they never reach the engine or a saved session. It refreshes them shortly before they expire and once after a rejected request, then passes the new ones to `onCredentials` for you to save. Codex doesn't accept requests from web pages, so these turns run on your server.
Sessions in one process share each refresh. If your app runs on several servers, two of them can notice the same expiring credentials at once, and the provider accepts only the first refresh. Pass `refreshLock` so they take turns:
```js
refreshLock: ({ provider, sessionId }, refresh) => locks.withLock(`${provider}:${accountOf(sessionId)}`, refresh),
```
libfx calls it with a function to run while you hold a lock on that user's account, for example in Redis. Inside it, libfx asks `credentials()` again and uses what another server already saved. This works for Grok too.
To sign a user in, use `libfx/chatgpt`:
@@ -240,6 +248,64 @@ After the user signs in, OpenAI opens a page at `localhost:1455` that doesn't lo
The sign-in uses the same OpenAI client as the Codex CLI, and the user's ChatGPT plan and OpenAI's terms apply to the turns.
### Grok plans
An agent can also run on a user's Grok plan. Set `provider: "grok"` and pass the same `credentials` callback, which libfx calls with `provider: "grok"`:
```js
const agent = createFxAgent({
provider: "grok",
model: "grok-4.5",
credentials: ({ provider, sessionId }) => tokens.load(provider, sessionId),
onCredentials: (credentials, { provider, sessionId }) => tokens.save(provider, sessionId, credentials),
});
```
libfx handles Grok credentials the same way as Codex's: it adds them to Grok requests itself, refreshes them, and checks that a refreshed token still belongs to the same xAI account. Grok doesn't accept requests from web pages either, so these turns run on your server.
To sign a user in, use `libfx/grok`. Device code is the simplest:
```js
import { startDeviceSignIn, completeDeviceSignIn } from "libfx/grok";
// Show device.userCode and a link to device.verificationUrl.
const device = await startDeviceSignIn();
// Wait until the user enters the code. pollDeviceSignIn() checks once, for a later request.
const credentials = await completeDeviceSignIn(device);
```
`startSignIn()` and `finishSignIn()` work like ChatGPT's: after the user signs in, xAI opens a page at `127.0.0.1` that doesn't load, and the user pastes its address back. The sign-in uses the same xAI client as the fx CLI, and the user's Grok plan and xAI's terms apply to the turns.
### Custom providers
An agent can also run on any server that speaks the OpenAI Chat Completions API, such as a local model server or a router. Pass the provider as an object and name the model it serves:
```js
const agent = createFxAgent({
provider: { name: "local", baseUrl: "http://localhost:11434/v1" },
model: "qwen3:8b",
});
```
Requests go to `{baseUrl}/chat/completions`, which must use HTTPS, or HTTP on `localhost`. A server that needs a key gets it from the same `credentials` callback, which libfx calls with the provider's `name`:
```js
const agent = createFxAgent({
provider: {
name: "router",
baseUrl: "https://openrouter.ai/api/v1",
models: { "qwen/qwen3-coder": { contextWindow: 128000, supportsToolUse: true } },
},
model: "qwen/qwen3-coder",
credentials: async ({ provider, sessionId }) => ({ apiKey: await keys.load(provider, sessionId) }),
});
```
libfx sends the key as a bearer token to that chat URL only, and the key never reaches the engine or a saved session. A key isn't refreshed, so a rejected one ends the turn with the server's error. `models` tells libfx what it can't ask the server: each model's context window and output limit, and whether it takes tools and images. Tool calls work when the model and server support them. In a browser, the server must accept cross-origin requests, and the page can read the key.
Saved sessions follow the agent's provider: point an agent at another base URL and its sessions continue there, history included.
## One conversation, no sessions
`createFxEngine()` is the engine under the agent: one conversation in the memory of the process that runs it, with no queue and no session store. Use it for a single request, or to keep the conversation in storage of your own.
@@ -320,6 +386,7 @@ This is the fx CLI's interface compiled to WebAssembly. Your page supplies stora
| `libfx/wasm` | The WebAssembly SDK without packaged asset paths, so you pass `wasm` yourself |
| `libfx/durable-local`, `libfx/durable-vercel`, `libfx/durable-world` | `local()`, `vercel()` and `world()` |
| `libfx/chatgpt` | `startSignIn`, `finishSignIn`, `startDeviceSignIn`, `pollDeviceSignIn`, `completeDeviceSignIn`, `refreshCredentials` |
| `libfx/grok` | `startSignIn`, `finishSignIn`, `startDeviceSignIn`, `pollDeviceSignIn`, `completeDeviceSignIn`, `refreshCredentials` |
| `libfx/mcp` | `createMcpAdapter` |
| `libfx/skills`, `libfx/skills/node` | `createSkillsAdapter`, and `loadSkillFile` for Node.js and Bun |
sdk/chatgpt-sign-in.js
+4 -49
View File
@@ -7,29 +7,18 @@
// - Device code: the user enters a short code on OpenAI's page. ChatGPT
// workspaces and accounts must turn device code sign-in on first.
import { chatgptClientId, chatgptIssuer, normalizeChatgptCredentials, refreshChatgptCredentials } from "./chatgpt.js";
import { hostFetch, pkceChallenge, randomSecret, readSignInAddress, signInError, wait } from "./oauth.js";
const redirectUri = "http://localhost:1455/auth/callback";
const deviceRedirectUri = `${chatgptIssuer}/deviceauth/callback`;
const scope = "openid profile email offline_access api.connectors.read api.connectors.invoke";
const deviceCodeLifetimeMs = 15 * 60 * 1000;
const maxAddressBytes = 16 * 1024;
const encoder = new TextEncoder();
function signInError(code, message, fields = {}) {
const error = new Error(message);
error.code = code;
return Object.assign(error, fields);
}
const base64url = (bytes) => btoa(String.fromCharCode(...bytes)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
const randomSecret = () => base64url(crypto.getRandomValues(new Uint8Array(32)));
const hostFetch = (fetch) => fetch ?? globalThis.fetch?.bind(globalThis);
/** Starts a sign-in. Keep the result on the server: `verifier` is secret. */
export async function startSignIn() {
const verifier = randomSecret();
const state = randomSecret();
const challenge = base64url(new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(verifier))));
const challenge = await pkceChallenge(verifier);
const url = new URL(`${chatgptIssuer}/oauth/authorize`);
for (const [key, value] of Object.entries({
response_type: "code",
@@ -46,35 +35,6 @@ export async function startSignIn() {
return { url: url.href, state, verifier };
}
/**
* Reads the sign-in result from a pasted address: the whole address, its query,
* or either wrapped in quotes, backticks or angle brackets.
*/
function parseSignInAddress(text) {
if (typeof text !== "string" || encoder.encode(text).length > maxAddressBytes) {
throw signInError("LIBFX_CHATGPT_SIGN_IN_INVALID_ADDRESS", "Paste the address of the page OpenAI opened after you signed in.");
}
const trimmed = text.trim().replace(/^[`'"<\s]+|[`'">\s]+$/g, "");
let params;
try {
params = new URL(trimmed).searchParams;
} catch {
params = new URLSearchParams(trimmed.includes("?") ? trimmed.slice(trimmed.indexOf("?") + 1) : trimmed);
}
const error = params.get("error");
if (error) {
throw signInError("LIBFX_CHATGPT_SIGN_IN_DENIED", `OpenAI did not sign you in: ${params.get("error_description") || error}`, { reason: error });
}
const code = params.get("code");
if (!code) {
throw signInError(
"LIBFX_CHATGPT_SIGN_IN_INVALID_ADDRESS",
`That address has no sign-in code. Copy the whole address of the page OpenAI opened after you signed in; it starts with ${redirectUri}?code=`,
);
}
return { code, state: params.get("state") };
}
async function exchangeCode({ code, verifier, redirect, fetch, signal }) {
const response = await hostFetch(fetch)(`${chatgptIssuer}/oauth/token`, {
method: "POST",
@@ -103,7 +63,7 @@ export async function finishSignIn(signIn, address, { fetch, signal } = {}) {
if (!signIn || typeof signIn.state !== "string" || typeof signIn.verifier !== "string") {
throw new TypeError("finishSignIn() needs the object startSignIn() returned");
}
const { code, state } = parseSignInAddress(address);
const { code, state } = readSignInAddress(address, { prefix: "LIBFX_CHATGPT", issuer: "OpenAI", redirectUri });
if (state !== signIn.state) {
throw signInError("LIBFX_CHATGPT_SIGN_IN_STATE_MISMATCH", "That address is from a different sign-in. Start again and paste the address from this one.");
}
@@ -171,12 +131,7 @@ export async function completeDeviceSignIn(device, { fetch, signal } = {}) {
for (;;) {
const credentials = await pollDeviceSignIn(device, { fetch, signal });
if (credentials) return credentials;
await new Promise((resolve, reject) => {
if (signal?.aborted) return reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
const timer = setTimeout(() => { signal?.removeEventListener("abort", onAbort); resolve(); }, device.intervalMs);
const onAbort = () => { clearTimeout(timer); reject(signal.reason ?? new DOMException("Aborted", "AbortError")); };
signal?.addEventListener("abort", onAbort, { once: true });
});
await wait(device.intervalMs, signal);
}
}
sdk/chatgpt.js
+15 -119
View File
@@ -1,7 +1,7 @@
// ChatGPT (Codex) credentials for libfx agents. The engine sends Codex requests
// without credentials; this module adds them at the fetch edge, from the host's
// `chatgpt` callback, refreshing near expiry and once after a rejection.
// Credentials never enter the engine or a session's saved records.
// ChatGPT (Codex) credentials for libfx agents: the Codex endpoints, how a
// ChatGPT plan's credentials are checked and refreshed, and the headers that
// carry them. `subscription.js` adds them at the fetch edge.
import { normalizeCredentials } from "./subscription.js";
export const chatgptIssuer = "https://auth.openai.com";
export const chatgptClientId = "app_EMoamEEZ73f0CkXaXp7hrann";
@@ -9,10 +9,6 @@ export const chatgptClientId = "app_EMoamEEZ73f0CkXaXp7hrann";
const codexOrigin = "https://chatgpt.com";
const codexPaths = new Set(["/backend-api/codex/responses", "/backend-api/codex/models"]);
const authClaim = "https://api.openai.com/auth";
const refreshSkewMs = 5 * 60 * 1000;
const maxRememberedCredentials = 256;
const maxTokenBytes = 64 * 1024;
const encoder = new TextEncoder();
/** True only for the exact Codex endpoints the engine calls. */
export function isCodexRequest(url) {
@@ -21,62 +17,12 @@ export function isCodexRequest(url) {
return parsed.origin === codexOrigin && codexPaths.has(parsed.pathname);
}
function tokenString(value, name, required) {
if (value === undefined && !required) return undefined;
if (typeof value !== "string" || value.length === 0) {
throw new TypeError(`ChatGPT credentials ${name} must be a non-empty string`);
}
if (encoder.encode(value).length > maxTokenBytes) {
throw new RangeError(`ChatGPT credentials ${name} exceeds ${maxTokenBytes} bytes`);
}
return value;
}
/** The claims of a JWT, or null when it is not one. Does not verify it. */
export function jwtClaims(token) {
const part = typeof token === "string" ? token.split(".")[1] : undefined;
if (!part) return null;
try {
const base64 = part.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(part.length / 4) * 4, "=");
const binary = atob(base64);
const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0));
const claims = JSON.parse(new TextDecoder().decode(bytes));
return claims && typeof claims === "object" ? claims : null;
} catch {
return null;
}
}
/**
* Validates credentials from the host. The account id and expiry default to
* the access token's own claims. Returns a new object.
*/
export function normalizeChatgptCredentials(value) {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new TypeError("chatgpt() must return { accessToken, refreshToken?, accountId?, expiresAt? }");
}
const accessToken = tokenString(value.accessToken, "accessToken", true);
const refreshToken = tokenString(value.refreshToken, "refreshToken", false);
const claims = jwtClaims(accessToken);
const accountId = value.accountId === undefined
? claims?.[authClaim]?.chatgpt_account_id
: tokenString(value.accountId, "accountId", true);
if (typeof accountId !== "string" || accountId.length === 0) {
throw new TypeError("ChatGPT credentials need an accountId; it was not in the access token");
}
let expiresAt = value.expiresAt;
if (expiresAt === undefined && Number.isFinite(claims?.exp)) expiresAt = claims.exp * 1000;
if (expiresAt !== undefined && !Number.isFinite(expiresAt)) {
throw new TypeError("ChatGPT credentials expiresAt must be milliseconds since the epoch");
}
return { accessToken, ...(refreshToken === undefined ? {} : { refreshToken }), accountId, ...(expiresAt === undefined ? {} : { expiresAt }) };
}
/** True when the credentials can and should be refreshed before use. */
export function needsRefresh(credentials, nowMs) {
return credentials.refreshToken !== undefined &&
credentials.expiresAt !== undefined &&
credentials.expiresAt - refreshSkewMs <= nowMs;
return normalizeCredentials(value, { label: "ChatGPT", accountClaim: (claims) => claims?.[authClaim]?.chatgpt_account_id });
}
/**
@@ -108,65 +54,15 @@ export async function refreshChatgptCredentials(credentials, { fetch, signal } =
return next.accountId === credentials.accountId ? next : { ...next, accountId: credentials.accountId };
}
/**
* The fetch edge for Codex requests. `chatgpt({ sessionId })` supplies
* credentials; `onCredentials(next, { sessionId })` receives refreshed ones,
* with the session whose request needed them, so the host can save them.
*/
export function createChatgptAuthorizer({ chatgpt, onCredentials, fetch, now = Date.now }) {
if (typeof chatgpt !== "function") throw new TypeError("provider \"codex\" needs a chatgpt() credentials function");
if (onCredentials !== undefined && typeof onCredentials !== "function") {
throw new TypeError("onChatgptCredentials must be a function");
}
// Each access token the host supplied maps to the newest credentials refreshed
// from it, so sessions on one account share a refresh and a host that has not
// saved a rotation yet never spends the old refresh token twice.
const descendants = new Map();
const refreshing = new Map();
const remember = (supplied, credentials) => {
descendants.delete(supplied.accessToken);
descendants.set(supplied.accessToken, credentials);
if (descendants.size > maxRememberedCredentials) descendants.delete(descendants.keys().next().value);
};
const refresh = (supplied, credentials, sessionId, signal) => {
const pending = refreshing.get(credentials.refreshToken);
if (pending) return pending;
const task = (async () => {
const next = await refreshChatgptCredentials(credentials, { fetch, signal });
remember(supplied, next);
await onCredentials?.(next, { sessionId });
return next;
})().finally(() => refreshing.delete(credentials.refreshToken));
refreshing.set(credentials.refreshToken, task);
return task;
};
const current = async (sessionId, signal) => {
const supplied = normalizeChatgptCredentials(await chatgpt({ sessionId }));
const best = descendants.get(supplied.accessToken) ?? supplied;
const used = needsRefresh(best, now()) ? await refresh(supplied, best, sessionId, signal) : best;
return { supplied, used };
};
const withCredentials = (init, credentials) => {
const headers = new Headers(init.headers);
/** Codex on a ChatGPT plan, for `createSubscriptionAuthorizer()`. */
export const codexSubscription = {
provider: "codex",
matches: isCodexRequest,
sessionId: (headers) => headers.get("session-id"),
normalize: normalizeChatgptCredentials,
refresh: refreshChatgptCredentials,
authorize(headers, _endpoint, credentials) {
headers.set("authorization", `Bearer ${credentials.accessToken}`);
headers.set("chatgpt-account-id", credentials.accountId);
return { ...init, headers };
};
return {
/** Sends one Codex request with credentials, refreshing once after a 401. */
async fetch(hostFetch, input, init = {}) {
const sessionId = new Headers(init.headers).get("session-id") || undefined;
const { supplied, used } = await current(sessionId, init.signal);
const response = await hostFetch(input, withCredentials(init, used));
if (response.status !== 401 || used.refreshToken === undefined) return response;
await response.body?.cancel().catch(() => {});
const next = await refresh(supplied, used, sessionId, init.signal);
return hostFetch(input, withCredentials(init, next));
},
};
}
},
};
sdk/custom-provider.js
+123
View File
@@ -0,0 +1,123 @@
// A custom provider for libfx agents: any server that speaks OpenAI Chat
// Completions, such as a local model server or a router. The engine sends its
// requests through the host's fetch without credentials; when the host has an
// API key, this module adds it at the fetch edge, to that provider's chat URL
// only. The key never enters the engine or a session's saved records.
import { tokenString } from "./subscription.js";
const encoder = new TextEncoder();
const reservedNames = new Set(["gateway", "codex", "grok"]);
const namePattern = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/;
const loopbackHosts = new Set(["localhost", "127.0.0.1", "[::1]"]);
const providerFields = new Set(["name", "baseUrl", "models"]);
// libfx option names to the CLI's settings names, which the engine parses.
const modelFields = {
contextWindow: "context_window",
maxOutputTokens: "max_output_tokens",
supportsToolUse: "supports_tool_use",
supportsVision: "supports_vision",
};
function isPlainObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function chatUrlOf(baseUrl) {
if (typeof baseUrl !== "string") throw new TypeError("provider.baseUrl must be a string");
let url;
try {
url = new URL(`${baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl}/chat/completions`);
} catch {
throw new TypeError("provider.baseUrl must be an absolute URL");
}
if (url.username || url.password || url.search || url.hash) {
throw new TypeError("provider.baseUrl must not carry credentials, a query or a fragment");
}
if (url.protocol !== "https:" && !(url.protocol === "http:" && loopbackHosts.has(url.hostname))) {
throw new TypeError("provider.baseUrl must use https, or http on localhost");
}
// The same form the fetch edge compares requests in.
return `${url.origin}${url.pathname}`;
}
// The engine's limits (configured_provider.zig), checked here so both engines
// report a mistake the same way.
const maxModels = 256;
const maxModelIdBytes = 1024;
const maxLimit = 2 ** 32 - 1;
function modelMetadata(models) {
if (!isPlainObject(models)) throw new TypeError("provider.models must be an object keyed by model id");
const entries = Object.entries(models);
if (entries.length > maxModels) throw new RangeError(`provider.models lists more than ${maxModels} models`);
const metadata = {};
for (const [id, model] of entries) {
if (id.length === 0 || id.trim() !== id || /[\u0000-\u001f\u007f]/.test(id) || encoder.encode(id).length > maxModelIdBytes) {
throw new TypeError(`provider.models has an invalid model id "${id}"`);
}
if (!isPlainObject(model)) throw new TypeError(`provider.models["${id}"] must be an object`);
const entry = {};
for (const [field, value] of Object.entries(model)) {
const name = modelFields[field];
if (name === undefined) throw new TypeError(`provider.models["${id}"] has unknown field ${field}`);
const limit = field === "contextWindow" || field === "maxOutputTokens";
if (limit ? !(Number.isInteger(value) && value > 0 && value <= maxLimit) : typeof value !== "boolean") {
throw new TypeError(`provider.models["${id}"].${field} must be ${limit ? "a positive integer" : "a boolean"}`);
}
entry[name] = value;
}
if (model.contextWindow !== undefined && model.maxOutputTokens !== undefined && model.maxOutputTokens >= model.contextWindow) {
throw new RangeError(`provider.models["${id}"].maxOutputTokens must be less than contextWindow`);
}
metadata[id] = entry;
}
return metadata;
}
/**
* Checks a custom provider `{ name, baseUrl, models? }` and returns its name,
* the chat URL its key may go to, and the settings-style JSON the engine
* parses. The engine applies the CLI's complete rules; these checks only catch
* mistakes early with a clearer message.
*/
export function customProviderDefinition(provider) {
if (!isPlainObject(provider)) throw new TypeError("a custom provider must be an object { name, baseUrl, models? }");
for (const field of Object.keys(provider)) {
if (!providerFields.has(field)) throw new TypeError(`a custom provider has unknown field ${field}`);
}
const { name, baseUrl, models } = provider;
if (typeof name !== "string" || !namePattern.test(name) || reservedNames.has(name.toLowerCase())) {
throw new TypeError("provider.name must be a letter followed by letters, digits, _ or -, at most 64 characters, and not gateway, codex or grok");
}
const chatUrl = chatUrlOf(baseUrl);
const definition = {
protocol: "openai-chat-completions",
base_url: baseUrl,
// The host adds the key, so the engine's copy has none.
auth: { type: "none" },
...(models === undefined ? {} : { model_metadata: modelMetadata(models) }),
};
return { name, chatUrl, json: JSON.stringify({ [name]: definition }) };
}
/**
* The custom provider's spec for `createSubscriptionAuthorizer()`: the host's
* API key as a bearer token, on the provider's chat URL only. `sessionId()`
* names the agent's session. A key has no refresh token, so it is never
* refreshed, and a rejected request returns as it is.
*/
export function customProviderSpec({ name, chatUrl }, sessionId) {
return {
provider: name,
matches: (endpoint) => endpoint === chatUrl,
sessionId: () => sessionId(),
normalize(value) {
if (!isPlainObject(value)) throw new TypeError("credentials() must return { apiKey } for a custom provider");
return { accessToken: tokenString(value.apiKey, "apiKey", true, `provider "${name}"`) };
},
authorize(headers, _endpoint, credentials) {
headers.set("authorization", `Bearer ${credentials.accessToken}`);
},
};
}
sdk/fx-harness.js
+4
View File
@@ -2,6 +2,7 @@
// core running fx engines. Everything the core needs to know about fx is
// here; the core itself knows only the harness contract.
import { coreAnswered, engineInternals, journalMarksWanted, normalizeInstructions, normalizeModelChoice } from "./fx-sdk.js";
import { createCredentialMemory, credentialMemory } from "./subscription.js";
const encoder = new TextEncoder();
@@ -19,6 +20,8 @@ export function fxHarness({ createEngine, defaultApiKey = async () => undefined
if (Object.hasOwn(engineOptions, name)) throw new TypeError(`createFxAgent() does not accept ${name}`);
}
const idempotent = idempotentTools(engineOptions.tools);
// One for all of this agent's sessions, so they refresh an account once.
const credentials = createCredentialMemory();
return {
...(checkpoint === undefined ? {} : { seed: snapshotOf(checkpointBytesOf(checkpoint)) }),
settings: sessionSettings,
@@ -29,6 +32,7 @@ export function fxHarness({ createEngine, defaultApiKey = async () => undefined
...(apiKey === undefined ? {} : { apiKey }),
...(ready === undefined || engineOptions.tools === undefined ? {} : { tools: claimedTools(engineOptions.tools, ready) }),
sessionId,
[credentialMemory]: credentials,
// The core folds the session from the marks on each record.
persistence: { ...store, [journalMarksWanted]: true },
inputsDurable: true,
sdk/fx-sdk.js
+42 -17
View File
@@ -1,7 +1,13 @@
import { CoreOutput, maxCoreMessageBytes } from "./core-output.js";
import { createChatgptAuthorizer, isCodexRequest } from "./chatgpt.js";
import { codexSubscription } from "./chatgpt.js";
import { customProviderDefinition, customProviderSpec } from "./custom-provider.js";
import { grokSubscription } from "./grok.js";
import { createSubscriptionAuthorizer, credentialMemory } from "./subscription.js";
import { loadModule } from "./wasm-module.js";
// Providers that run on a user's plan, authorized by the host's credentials.
const subscriptionProviders = new Set(["codex", "grok"]);
const encoder = new TextEncoder();
const decoder = new TextDecoder();
const strictDecoder = new TextDecoder("utf-8", { fatal: true });
@@ -144,20 +150,34 @@ export function normalizeAgentOptions(value) {
if (Object.hasOwn(options, "env")) {
throw new TypeError("createFxEngine() does not accept env; pass apiKey and model directly");
}
if (options.provider !== undefined && options.provider !== "gateway" && options.provider !== "codex") {
throw new TypeError("provider must be \"gateway\" or \"codex\"");
// Derived from the caller's provider every time, so normalizing again agrees.
const custom = options.provider !== null && typeof options.provider === "object"
? customProviderDefinition(options.provider)
: undefined;
if (!custom && options.provider !== undefined && options.provider !== "gateway" && !subscriptionProviders.has(options.provider)) {
throw new TypeError("provider must be \"gateway\", \"codex\", \"grok\" or a custom provider { name, baseUrl }");
}
const codex = options.provider === "codex";
// A Codex agent runs on the host's ChatGPT credentials; Gateway needs its key.
options.apiKey = boundedString(options.apiKey, "apiKey", maxApiKeyBytes, !codex);
if (codex && typeof options.chatgpt !== "function") {
throw new TypeError("provider \"codex\" needs a chatgpt() credentials function");
options.customProvider = custom;
options.providerName = custom ? custom.name : options.provider;
const subscription = subscriptionProviders.has(options.provider);
// Gateway needs its key; every other provider runs on the host's credentials.
options.apiKey = boundedString(options.apiKey, "apiKey", maxApiKeyBytes, !subscription && !custom);
if (subscription && typeof options.credentials !== "function") {
throw new TypeError(`provider "${options.provider}" needs a credentials() function`);
}
if (!codex && options.chatgpt !== undefined) throw new TypeError("chatgpt needs provider \"codex\"");
if (options.onChatgptCredentials !== undefined && typeof options.onChatgptCredentials !== "function") {
throw new TypeError("onChatgptCredentials must be a function");
if (!subscription && !custom && options.credentials !== undefined) {
throw new TypeError("credentials needs provider \"codex\", \"grok\" or a custom provider");
}
// A custom provider's key is never refreshed, so it has no onCredentials or refreshLock.
for (const name of ["onCredentials", "refreshLock"]) {
if (!subscription && options[name] !== undefined) throw new TypeError(`${name} needs provider "codex" or "grok"`);
}
for (const name of ["credentials", "onCredentials", "refreshLock"]) {
if (options[name] !== undefined && typeof options[name] !== "function") throw new TypeError(`${name} must be a function`);
}
Object.assign(options, normalizeModelChoice(options));
// fx's default model is a Gateway id; a custom provider names its own.
if (custom && options.model === undefined) throw new TypeError("a custom provider needs a model");
validateGatewayChatUrl(options.gatewayChatUrl);
if (options.resizeImage !== undefined && typeof options.resizeImage !== "function") {
throw new TypeError("resizeImage must be a function");
@@ -436,7 +456,8 @@ function journalMarks(events) {
function agentEnvironment(options) {
return {
...(options.apiKey === undefined ? {} : { AI_GATEWAY_API_KEY: options.apiKey }),
...(options.provider === undefined ? {} : { FX_PROVIDER: options.provider }),
...(options.providerName === undefined ? {} : { FX_PROVIDER: options.providerName }),
...(options.customProvider === undefined ? {} : { FX_CONFIGURED_PROVIDERS: options.customProvider.json }),
...(options.model === undefined ? {} : { FX_MODEL: options.model }),
...(options.effort === undefined ? {} : { FX_EFFORT: options.effort }),
...(options.fast === undefined ? {} : { FX_FAST: options.fast ? "true" : "false" }),
@@ -2409,9 +2430,12 @@ export async function createFxEngine(options = {}) {
return checkpointing;
};
const hostFetch = options.fetch ?? globalThis.fetch?.bind(globalThis);
const codexAuth = options.provider === "codex"
? createChatgptAuthorizer({ chatgpt: options.chatgpt, onCredentials: options.onChatgptCredentials, fetch: hostFetch })
: null;
// Adds the host's credentials to the provider's own requests.
const providerAuth = subscriptionProviders.has(options.provider)
? createSubscriptionAuthorizer([codexSubscription, grokSubscription], { credentials: options.credentials, onCredentials: options.onCredentials, refreshLock: options.refreshLock, fetch: hostFetch, memory: options[credentialMemory] })
: options.customProvider && options.credentials
? createSubscriptionAuthorizer([customProviderSpec(options.customProvider, () => sessionId)], { credentials: options.credentials, fetch: hostFetch })
: null;
const transportFetch = async (input, init = {}) => {
const method = String(init.method ?? input?.method ?? "GET").toUpperCase();
let endpoint = String(input?.url ?? input);
@@ -2439,8 +2463,9 @@ export async function createFxEngine(options = {}) {
throw new DOMException("Aborted", "AbortError");
}
if (!hostFetch) throw new TypeError("fetch is unavailable");
const response = codexAuth && isCodexRequest(endpoint)
? await codexAuth.fetch(hostFetch, input, init)
const spec = providerAuth?.specFor(endpoint);
const response = spec
? await providerAuth.fetch(spec, endpoint, hostFetch, input, init)
: await hostFetch(input, init);
const headers = response.headers;
emit("transport.response", {
sdk/grok-sign-in.js
+145
View File
@@ -0,0 +1,145 @@
// `libfx/grok`: sign in with a Grok plan for a Grok agent. Two flows, both
// plain data between steps so a website can start in one request and finish in
// another:
//
// - Device code: the user enters a short code on xAI's page.
// - Paste the address: the user signs in on xAI, lands on a page at
// 127.0.0.1 that does not load, and pastes its address back.
import { grokAccountId, grokClientId, grokIssuer, normalizeGrokCredentials, refreshGrokCredentials } from "./grok.js";
import { hostFetch, pkceChallenge, randomSecret, readSignInAddress, signInError, wait } from "./oauth.js";
const scope = "openid profile email offline_access grok-cli:access api:access";
const deviceGrant = "urn:ietf:params:oauth:grant-type:device_code";
const slowDownMs = 5000;
const form = (fields) => ({
method: "POST",
headers: { "content-type": "application/x-www-form-urlencoded", accept: "application/json" },
body: new URLSearchParams(fields).toString(),
});
/** Credentials for a token response, with the account it belongs to. */
async function credentialsFor(tokens, { fetch, signal }) {
const accountId = await grokAccountId(tokens.access_token, { fetch, signal });
return normalizeGrokCredentials({
accessToken: tokens.access_token,
refreshToken: tokens.refresh_token,
accountId,
...(Number.isFinite(tokens.expires_in) ? { expiresAt: Date.now() + tokens.expires_in * 1000 } : {}),
});
}
/** Starts a sign-in. Keep the result on the server: `verifier` is secret. */
export async function startSignIn() {
const verifier = randomSecret();
const state = randomSecret();
// xAI accepts any loopback port. Nothing listens there: the user copies the address.
const port = 1024 + (crypto.getRandomValues(new Uint16Array(1))[0] % (65536 - 1024));
const redirectUri = `http://127.0.0.1:${port}/callback`;
const url = new URL(`${grokIssuer}/oauth2/authorize`);
for (const [key, value] of Object.entries({
response_type: "code",
client_id: grokClientId,
redirect_uri: redirectUri,
scope,
code_challenge: await pkceChallenge(verifier),
code_challenge_method: "S256",
state,
referrer: "fx",
})) url.searchParams.set(key, value);
return { url: url.href, state, verifier, redirectUri };
}
/** Finishes a sign-in from the address the user pasted. */
export async function finishSignIn(signIn, address, { fetch, signal } = {}) {
if (!signIn || typeof signIn.state !== "string" || typeof signIn.verifier !== "string" || typeof signIn.redirectUri !== "string") {
throw new TypeError("finishSignIn() needs the object startSignIn() returned");
}
const { code, state } = readSignInAddress(address, { prefix: "LIBFX_GROK", issuer: "xAI", redirectUri: signIn.redirectUri });
if (state !== signIn.state) {
throw signInError("LIBFX_GROK_SIGN_IN_STATE_MISMATCH", "That address is from a different sign-in. Start again and paste the address from this one.");
}
const response = await hostFetch(fetch)(`${grokIssuer}/oauth2/token`, {
...form({ grant_type: "authorization_code", client_id: grokClientId, code, code_verifier: signIn.verifier, redirect_uri: signIn.redirectUri }),
signal,
});
if (response.status === 400 || response.status === 401) {
await response.body?.cancel().catch(() => {});
throw signInError("LIBFX_GROK_SIGN_IN_EXPIRED", "This sign-in expired or was already used. Start again.", { status: response.status });
}
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw signInError("LIBFX_GROK_SIGN_IN_FAILED", `xAI could not finish the sign-in (HTTP ${response.status}).`, { status: response.status });
}
return credentialsFor(await response.json(), { fetch, signal });
}
/** Starts a device code sign-in. Show `userCode` and `verificationUrl`. */
export async function startDeviceSignIn({ fetch, signal } = {}) {
const response = await hostFetch(fetch)(`${grokIssuer}/oauth2/device/code`, { ...form({ client_id: grokClientId, scope }), signal });
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw signInError("LIBFX_GROK_DEVICE_CODE_UNAVAILABLE", `xAI did not issue a device code (HTTP ${response.status}).`, { status: response.status });
}
const body = await response.json();
const expiresIn = Number(body.expires_in);
if (typeof body.user_code !== "string" || typeof body.device_code !== "string" || typeof body.verification_uri !== "string" || !Number.isFinite(expiresIn)) {
throw signInError("LIBFX_GROK_DEVICE_CODE_UNAVAILABLE", "xAI returned a device code response libfx does not understand.");
}
const interval = Number(body.interval);
return {
verificationUrl: typeof body.verification_uri_complete === "string" ? body.verification_uri_complete : body.verification_uri,
userCode: body.user_code,
deviceCode: body.device_code,
intervalMs: Math.max(Number.isFinite(interval) ? interval : 5, 1) * 1000,
expiresAt: Date.now() + expiresIn * 1000,
};
}
/** One poll: `{ credentials }`, or `{ pending: true, slowDown }` until the user enters the code. */
async function poll(device, { fetch, signal }) {
if (!device || typeof device.deviceCode !== "string" || typeof device.userCode !== "string") {
throw new TypeError("pollDeviceSignIn() needs the object startDeviceSignIn() returned");
}
if (Date.now() > device.expiresAt) {
throw signInError("LIBFX_GROK_DEVICE_CODE_EXPIRED", "The code expired before it was entered. Start again.");
}
const response = await hostFetch(fetch)(`${grokIssuer}/oauth2/token`, {
...form({ grant_type: deviceGrant, device_code: device.deviceCode, client_id: grokClientId }),
signal,
});
if (response.ok) return { credentials: await credentialsFor(await response.json(), { fetch, signal }) };
const body = await response.json().catch(() => null);
switch (body?.error) {
case "authorization_pending": return { pending: true, slowDown: false };
case "slow_down": return { pending: true, slowDown: true };
case "expired_token":
throw signInError("LIBFX_GROK_DEVICE_CODE_EXPIRED", "The code expired before it was entered. Start again.");
case "access_denied":
throw signInError("LIBFX_GROK_SIGN_IN_DENIED", "The sign-in was declined on xAI's page.", { reason: body.error });
default:
throw signInError("LIBFX_GROK_SIGN_IN_FAILED", `xAI could not finish the sign-in (HTTP ${response.status}).`, { status: response.status });
}
}
/** Checks a device sign-in once: credentials when the user has entered the code, otherwise null. */
export async function pollDeviceSignIn(device, { fetch, signal } = {}) {
const result = await poll(device, { fetch, signal });
return result.credentials ?? null;
}
/** Waits until the user enters the code, polling at xAI's interval and slowing down when asked. */
export async function completeDeviceSignIn(device, { fetch, signal } = {}) {
let intervalMs = device?.intervalMs;
for (;;) {
const result = await poll(device, { fetch, signal });
if (result.credentials) return result.credentials;
if (result.slowDown) intervalMs += slowDownMs;
await wait(intervalMs, signal);
}
}
/** Refreshes credentials. Grok agents refresh on their own; this is for hosts. */
export function refreshCredentials(credentials, { fetch, signal } = {}) {
return refreshGrokCredentials(normalizeGrokCredentials(credentials), { fetch, signal });
}
sdk/grok.js
+108
View File
@@ -0,0 +1,108 @@
// Grok (xAI subscription) credentials for libfx agents: the Grok endpoints,
// how a Grok plan's credentials are checked and refreshed, and the headers
// that carry them. `subscription.js` adds them at the fetch edge.
import { normalizeCredentials } from "./subscription.js";
export const grokIssuer = "https://auth.x.ai";
export const grokClientId = "b1a00492-073a-47ea-816f-4c329264a828";
const proxyOrigin = "https://cli-chat-proxy.grok.com";
const responsesPath = "/v1/responses";
const modelsPath = "/v1/models";
const languageModelsEndpoint = "https://api.x.ai/v1/language-models";
function endpointOf(url) {
try {
const parsed = new URL(String(url));
return `${parsed.origin}${parsed.pathname}`;
} catch {
return null;
}
}
/** True only for the exact Grok endpoints the engine calls. */
export function isGrokRequest(url) {
const endpoint = endpointOf(url);
return endpoint === `${proxyOrigin}${responsesPath}` ||
endpoint === `${proxyOrigin}${modelsPath}` ||
endpoint === languageModelsEndpoint;
}
/**
* Validates credentials from the host. The account id defaults to the access
* token's `sub` claim when the token carries one; sign-in from `libfx/grok`
* always returns it. Returns a new object.
*/
export function normalizeGrokCredentials(value) {
return normalizeCredentials(value, { label: "Grok", accountClaim: (claims) => claims?.sub });
}
function grokError(code, message, status) {
const error = new Error(message);
error.code = code;
if (status !== undefined) error.status = status;
return error;
}
/** The xAI account an access token belongs to: the `sub` of its userinfo. */
export async function grokAccountId(accessToken, { fetch, signal } = {}) {
const doFetch = fetch ?? globalThis.fetch?.bind(globalThis);
const response = await doFetch(`${grokIssuer}/oauth2/userinfo`, {
headers: { authorization: `Bearer ${accessToken}`, accept: "application/json" },
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw grokError("LIBFX_GROK_ACCOUNT_UNAVAILABLE", `xAI did not identify the account (HTTP ${response.status})`, response.status);
}
const info = await response.json().catch(() => null);
if (typeof info?.sub !== "string" || info.sub.length === 0) {
throw grokError("LIBFX_GROK_ACCOUNT_UNAVAILABLE", "xAI's account response had no account id");
}
return info.sub;
}
/**
* Exchanges a refresh token at xAI, keeping the old refresh token when the
* response does not rotate it, and confirms the new token is for the same
* account.
*/
export async function refreshGrokCredentials(credentials, { fetch, signal } = {}) {
if (credentials.refreshToken === undefined) throw new TypeError("Grok credentials have no refreshToken");
const doFetch = fetch ?? globalThis.fetch?.bind(globalThis);
const response = await doFetch(`${grokIssuer}/oauth2/token`, {
method: "POST",
headers: { "content-type": "application/x-www-form-urlencoded", accept: "application/json" },
body: new URLSearchParams({ grant_type: "refresh_token", client_id: grokClientId, refresh_token: credentials.refreshToken }).toString(),
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw grokError("LIBFX_GROK_REFRESH_FAILED", `Grok token refresh failed with HTTP ${response.status}`, response.status);
}
const tokens = await response.json();
const next = normalizeGrokCredentials({
accessToken: tokens.access_token,
refreshToken: tokens.refresh_token ?? credentials.refreshToken,
accountId: credentials.accountId,
...(Number.isFinite(tokens.expires_in) ? { expiresAt: Date.now() + tokens.expires_in * 1000 } : {}),
});
if (await grokAccountId(next.accessToken, { fetch, signal }) !== credentials.accountId) {
throw grokError("LIBFX_GROK_ACCOUNT_CHANGED", "The refreshed Grok token belongs to a different account");
}
return next;
}
/** Grok on a Grok plan, for `createSubscriptionAuthorizer()`. */
export const grokSubscription = {
provider: "grok",
matches: isGrokRequest,
sessionId: (headers) => headers.get("x-grok-conv-id"),
normalize: normalizeGrokCredentials,
refresh: refreshGrokCredentials,
authorize(headers, endpoint, credentials) {
headers.set("authorization", `Bearer ${credentials.accessToken}`);
if (endpoint === `${proxyOrigin}${responsesPath}`) headers.set("x-grok-user-id", credentials.accountId);
if (endpoint === `${proxyOrigin}${modelsPath}`) headers.set("x-userid", credentials.accountId);
},
};
sdk/node.js
+5 -3
View File
@@ -352,20 +352,22 @@ export async function getBackendInfo(value = {}) {
}
function createNativeCoreRuntime(addon, options) {
const { apiKey, provider, model, effort, fast, ultrafast, gatewayChatUrl } = options;
const { apiKey, providerName: provider, customProvider, model, effort, fast, ultrafast, gatewayChatUrl } = options;
if (ultrafast !== undefined && addon.supportsUltrafast !== true) {
const error = new Error("native addon does not support the ultrafast option");
error.code = "LIBFX_NATIVE_CAPABILITY_UNAVAILABLE";
throw error;
}
if (provider === "codex" && addon.supportsCodex !== true) {
const error = new Error("native addon does not support the codex provider");
const capability = customProvider ? "supportsConfiguredProvider" : { codex: "supportsCodex", grok: "supportsGrok" }[provider];
if (capability && addon[capability] !== true) {
const error = new Error(`native addon does not support the ${provider} provider`);
error.code = "LIBFX_NATIVE_CAPABILITY_UNAVAILABLE";
throw error;
}
const core = addon.createCore({
...(apiKey === undefined ? {} : { apiKey }),
...(provider === undefined ? {} : { provider }),
...(customProvider === undefined ? {} : { configuredProviders: customProvider.json }),
home: options.home ?? homedir(),
workspaceRoot: options.workspaceRoot ?? process.cwd(),
...(model === undefined ? {} : { model }),
sdk/oauth.js
+64
View File
@@ -0,0 +1,64 @@
// Pieces the subscription sign-ins share: PKCE secrets, reading a pasted
// redirect address, and waiting between device code polls.
const maxAddressBytes = 16 * 1024;
const encoder = new TextEncoder();
export function signInError(code, message, fields = {}) {
const error = new Error(message);
error.code = code;
return Object.assign(error, fields);
}
const base64url = (bytes) => btoa(String.fromCharCode(...bytes)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
/** A random URL-safe secret, for a PKCE verifier or an OAuth state. */
export const randomSecret = () => base64url(crypto.getRandomValues(new Uint8Array(32)));
/** The S256 challenge for a PKCE verifier. */
export async function pkceChallenge(verifier) {
return base64url(new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(verifier))));
}
export const hostFetch = (fetch) => fetch ?? globalThis.fetch?.bind(globalThis);
/**
* Reads the sign-in result from a pasted address: the whole address, its query,
* or either wrapped in quotes, backticks or angle brackets. `prefix` names the
* error codes, `issuer` the sign-in page, and `redirectUri` the address the
* user should have copied.
*/
export function readSignInAddress(text, { prefix, issuer, redirectUri }) {
if (typeof text !== "string" || encoder.encode(text).length > maxAddressBytes) {
throw signInError(`${prefix}_SIGN_IN_INVALID_ADDRESS`, `Paste the address of the page ${issuer} opened after you signed in.`);
}
const trimmed = text.trim().replace(/^[`'"<\s]+|[`'">\s]+$/g, "");
let params;
try {
params = new URL(trimmed).searchParams;
} catch {
params = new URLSearchParams(trimmed.includes("?") ? trimmed.slice(trimmed.indexOf("?") + 1) : trimmed);
}
const error = params.get("error");
if (error) {
throw signInError(`${prefix}_SIGN_IN_DENIED`, `${issuer} did not sign you in: ${params.get("error_description") || error}`, { reason: error });
}
const code = params.get("code");
if (!code) {
throw signInError(
`${prefix}_SIGN_IN_INVALID_ADDRESS`,
`That address has no sign-in code. Copy the whole address of the page ${issuer} opened after you signed in; it starts with ${redirectUri}?code=`,
);
}
return { code, state: params.get("state") };
}
/** Resolves after `ms`, or rejects when `signal` aborts first. */
export function wait(ms, signal) {
return new Promise((resolve, reject) => {
if (signal?.aborted) return reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
const timer = setTimeout(() => { signal?.removeEventListener("abort", onAbort); resolve(); }, ms);
const onAbort = () => { clearTimeout(timer); reject(signal.reason ?? new DOMException("Aborted", "AbortError")); };
signal?.addEventListener("abort", onAbort, { once: true });
});
}
sdk/package.json
+9
View File
@@ -70,6 +70,10 @@
"types": "./types/chatgpt.d.ts",
"default": "./chatgpt-sign-in.js"
},
"./grok": {
"types": "./types/grok.d.ts",
"default": "./grok-sign-in.js"
},
"./mcp": {
"types": "./types/mcp.d.ts",
"default": "./mcp.js"
@@ -97,8 +101,13 @@
"types/*.d.cts",
"wasm-module.js",
"core-output.js",
"subscription.js",
"oauth.js",
"chatgpt.js",
"chatgpt-sign-in.js",
"grok.js",
"grok-sign-in.js",
"custom-provider.js",
"fetch-cleanup.js",
"mcp.js",
"skills.js",
sdk/scripts/package-libfx.mjs
+5
View File
@@ -34,8 +34,13 @@ const files = [
["sdk/fx-harness.js", "fx-harness.js"],
["sdk/wasm-module.js", "wasm-module.js"],
["sdk/core-output.js", "core-output.js"],
["sdk/subscription.js", "subscription.js"],
["sdk/oauth.js", "oauth.js"],
["sdk/chatgpt.js", "chatgpt.js"],
["sdk/chatgpt-sign-in.js", "chatgpt-sign-in.js"],
["sdk/grok.js", "grok.js"],
["sdk/grok-sign-in.js", "grok-sign-in.js"],
["sdk/custom-provider.js", "custom-provider.js"],
["sdk/fetch-cleanup.js", "fetch-cleanup.js"],
["sdk/mcp.js", "mcp.js"],
["sdk/skills.js", "skills.js"],
sdk/subscription.js
+251
View File
@@ -0,0 +1,251 @@
// Subscription credentials for libfx agents: Codex on a ChatGPT plan, Grok on
// a Grok plan. The engine sends these providers' requests without credentials;
// this module adds them at the fetch edge, from the host's `credentials`
// callback, refreshing near expiry and once after a rejection. Credentials
// never enter the engine or a session's saved records, and are masked in the
// body of a failed response.
//
// A provider spec supplies what differs: which requests it authorizes, how its
// credentials are checked and refreshed, and which headers carry them.
const refreshSkewMs = 5 * 60 * 1000;
const maxRememberedCredentials = 256;
const maxTokenBytes = 64 * 1024;
const maxErrorBodyBytes = 64 * 1024;
const encoder = new TextEncoder();
/** A non-empty, bounded credential field, or undefined when optional and absent. */
export function tokenString(value, name, required, label) {
if (value === undefined && !required) return undefined;
if (typeof value !== "string" || value.length === 0) {
throw new TypeError(`${label} credentials ${name} must be a non-empty string`);
}
if (encoder.encode(value).length > maxTokenBytes) {
throw new RangeError(`${label} credentials ${name} exceeds ${maxTokenBytes} bytes`);
}
return value;
}
/** The claims of a JWT, or null when it is not one. Does not verify it. */
export function jwtClaims(token) {
const part = typeof token === "string" ? token.split(".")[1] : undefined;
if (!part) return null;
try {
const base64 = part.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(part.length / 4) * 4, "=");
const binary = atob(base64);
const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0));
const claims = JSON.parse(new TextDecoder().decode(bytes));
return claims && typeof claims === "object" ? claims : null;
} catch {
return null;
}
}
/**
* Validates credentials from the host. `accountId` and `expiresAt` default to
* the access token's claims, `accountClaim(claims)` picking the account.
* Returns a new object.
*/
export function normalizeCredentials(value, { label, accountClaim }) {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new TypeError("credentials() must return { accessToken, refreshToken?, accountId?, expiresAt? }");
}
const accessToken = tokenString(value.accessToken, "accessToken", true, label);
const refreshToken = tokenString(value.refreshToken, "refreshToken", false, label);
const claims = jwtClaims(accessToken);
const accountId = value.accountId === undefined ? accountClaim(claims) : tokenString(value.accountId, "accountId", true, label);
if (typeof accountId !== "string" || accountId.length === 0) {
throw new TypeError(`${label} credentials need an accountId; it was not in the access token`);
}
let expiresAt = value.expiresAt;
if (expiresAt === undefined && Number.isFinite(claims?.exp)) expiresAt = claims.exp * 1000;
if (expiresAt !== undefined && !Number.isFinite(expiresAt)) {
throw new TypeError(`${label} credentials expiresAt must be milliseconds since the epoch`);
}
return { accessToken, ...(refreshToken === undefined ? {} : { refreshToken }), accountId, ...(expiresAt === undefined ? {} : { expiresAt }) };
}
/**
* A failed response whose body cannot repeat the credentials its request
* carried. Some servers quote the bearer token back in an error, and the
* engine shows and saves error details, so this edge, the only code that knows
* the token, masks it the way fx masks error details from its own providers.
* JSON is normalized first, so an escaped copy cannot survive.
*/
async function withoutCredentials(response, credentials) {
if (response.status < 400) return response;
const raw = await boundedText(response, maxErrorBodyBytes);
let body = raw ?? "Provider error details exceeded the local limit";
if (raw !== null && /^\s*[[{"]/.test(raw)) {
try {
body = JSON.stringify(JSON.parse(raw));
} catch {
body = "Provider error details could not be decoded";
}
}
for (const secret of [credentials.accessToken, credentials.refreshToken]) {
if (!secret) continue;
for (const form of new Set([secret, JSON.stringify(secret).slice(1, -1)])) body = body.split(form).join("*".repeat(form.length));
}
const headers = new Headers(response.headers);
headers.delete("content-length");
headers.delete("content-encoding");
return new Response(body, { status: response.status, statusText: response.statusText, headers });
}
/** The body as text, or null once it exceeds `limit` bytes. */
async function boundedText(response, limit) {
if (!response.body) return "";
const reader = response.body.getReader();
const decoder = new TextDecoder();
let size = 0;
let text = "";
for (;;) {
const { done, value } = await reader.read();
if (done) return text + decoder.decode();
size += value.byteLength;
if (size > limit) {
await reader.cancel().catch(() => {});
return null;
}
text += decoder.decode(value, { stream: true });
}
}
/** True when the credentials can and should be refreshed before use. */
export function needsRefresh(credentials, nowMs) {
return credentials.refreshToken !== undefined &&
credentials.expiresAt !== undefined &&
credentials.expiresAt - refreshSkewMs <= nowMs;
}
/** The engine option that carries an agent's `createCredentialMemory()`. */
export const credentialMemory = Symbol.for("libfx.credentialMemory");
/**
* Refresh state for `createSubscriptionAuthorizer()`. An agent gives the same
* memory to every session's engine, so its sessions refresh an account once.
*/
export function createCredentialMemory() {
return { descendants: new Map(), refreshing: new Map(), saved: new WeakMap() };
}
/**
* The fetch edge for subscription requests. `credentials({ provider, sessionId })`
* supplies credentials; `onCredentials(next, { provider, sessionId })` receives
* refreshed ones once for each session that uses them, so the host can save them.
* `refreshLock({ provider, sessionId }, refresh)` runs `refresh` while the host
* holds a lock on the account, for a host with several servers.
*
* Each spec: `{ provider, matches(endpoint), sessionId(headers), normalize(value),
* refresh(credentials, { fetch, signal }), authorize(headers, endpoint, credentials) }`.
*/
export function createSubscriptionAuthorizer(specs, { credentials, onCredentials, refreshLock, fetch, now = Date.now, memory = createCredentialMemory() }) {
if (typeof credentials !== "function") throw new TypeError("a subscription provider needs a credentials() function");
if (onCredentials !== undefined && typeof onCredentials !== "function") {
throw new TypeError("onCredentials must be a function");
}
if (refreshLock !== undefined && typeof refreshLock !== "function") {
throw new TypeError("refreshLock must be a function");
}
// Each access token the host supplied maps to the newest credentials refreshed
// from it, so a host that has not saved a rotation yet never spends the old
// refresh token twice. Engines that share `memory` share this.
const { descendants, refreshing, saved } = memory;
const keyOf = (spec, token) => `${spec.provider}\u0000${token}`;
const remember = (spec, supplied, next) => {
const key = keyOf(spec, supplied.accessToken);
descendants.delete(key);
descendants.set(key, next);
if (descendants.size > maxRememberedCredentials) descendants.delete(descendants.keys().next().value);
};
// A host that keeps credentials per session must also save a rotation that
// another session's request made.
const savedBy = (next) => {
let sessions = saved.get(next);
if (!sessions) saved.set(next, (sessions = new Set()));
return sessions;
};
const save = async (spec, next, sessionId) => {
if (!onCredentials) return;
const sessions = savedBy(next);
if (sessions.has(sessionId)) return;
sessions.add(sessionId);
await onCredentials(next, { provider: spec.provider, sessionId });
};
const redeem = async (spec, supplied, current, sessionId, signal) => {
if (refreshLock) {
// Another server may have refreshed and saved them while this one waited.
const latest = spec.normalize(await credentials({ provider: spec.provider, sessionId }));
if (latest.accessToken !== current.accessToken && latest.accessToken !== supplied.accessToken && !needsRefresh(latest, now())) {
remember(spec, supplied, latest);
savedBy(latest).add(sessionId);
return latest;
}
}
const next = await spec.refresh(current, { fetch, signal });
remember(spec, supplied, next);
await save(spec, next, sessionId);
return next;
};
// With `refreshLock`, the host holds a lock on the account while the refresh
// runs, so its servers refresh one at a time.
const locked = async (spec, supplied, current, sessionId, signal) => {
let next;
await refreshLock({ provider: spec.provider, sessionId }, async () => {
next = await redeem(spec, supplied, current, sessionId, signal);
});
if (next === undefined) throw new Error("refreshLock must run and await the refresh it is given");
return next;
};
const refresh = async (spec, supplied, current, sessionId, signal) => {
const key = keyOf(spec, current.refreshToken);
let task = refreshing.get(key);
if (!task) {
task = (refreshLock ? locked : redeem)(spec, supplied, current, sessionId, signal).finally(() => refreshing.delete(key));
refreshing.set(key, task);
}
const next = await task;
if (sessionId !== undefined) await save(spec, next, sessionId);
return next;
};
const current = async (spec, sessionId, signal) => {
const supplied = spec.normalize(await credentials({ provider: spec.provider, sessionId }));
const best = descendants.get(keyOf(spec, supplied.accessToken)) ?? supplied;
const used = needsRefresh(best, now()) ? await refresh(spec, supplied, best, sessionId, signal) : best;
if (used !== supplied && sessionId !== undefined) await save(spec, used, sessionId);
return { supplied, used };
};
const withCredentials = (spec, endpoint, init, used) => {
const headers = new Headers(init.headers);
spec.authorize(headers, endpoint, used);
return { ...init, headers };
};
return {
/** The spec that authorizes requests to `endpoint` (origin and path), if any. */
specFor: (endpoint) => specs.find((spec) => spec.matches(endpoint)),
/** Sends one request with credentials, refreshing once after a 401. */
async fetch(spec, endpoint, hostFetch, input, init = {}) {
const sessionId = spec.sessionId(new Headers(init.headers)) || undefined;
const { used } = await current(spec, sessionId, init.signal);
const response = await hostFetch(input, withCredentials(spec, endpoint, init, used));
if (response.status !== 401 || used.refreshToken === undefined) return withoutCredentials(response, used);
await response.body?.cancel().catch(() => {});
// Another request may have replaced these credentials while this one was
// out. Only a rejection of the newest ones spends a refresh token.
const latest = await current(spec, sessionId, init.signal);
const next = latest.used.accessToken !== used.accessToken
? latest.used
: await refresh(spec, latest.supplied, used, sessionId, init.signal);
return withoutCredentials(await hostFetch(input, withCredentials(spec, endpoint, init, next)), next);
},
};
}
sdk/tests/test-agent-codex.mjs
+13 -10
View File
@@ -95,7 +95,7 @@ const assertNoSecrets = (value, secrets) => {
const asked = [];
const { result, text, events } = await runTurn({
fetch: world.fetch,
chatgpt: (request) => { asked.push(request); return { accessToken: access, refreshToken: "rt-1" }; },
credentials: (request) => { asked.push(request); return { accessToken: access, refreshToken: "rt-1" }; },
});
assert.equal(result.stopReason, "end_turn");
assert.equal(text, "libfx ok");
@@ -120,7 +120,8 @@ const assertNoSecrets = (value, secrets) => {
assert.equal(models.headers.get("authorization"), `Bearer ${access}`, "the model list uses the same credentials");
assert.equal(models.headers.get("chatgpt-account-id"), accountId);
}
assert.ok(asked.some((request) => request.sessionId === sessionId), "chatgpt() learns which session the request is for");
assert.ok(asked.some((request) => request.sessionId === sessionId), "credentials() learns which session the request is for");
assert.ok(asked.length > 0 && asked.every((request) => request.provider === "codex"), "credentials() learns the provider");
assertNoSecrets(events, [access, "rt-1"]);
}
@@ -132,8 +133,8 @@ const assertNoSecrets = (value, secrets) => {
const saved = [];
const { result, text, events } = await runTurn({
fetch: world.fetch,
chatgpt: () => ({ accessToken: first, refreshToken: "rt-1" }),
onChatgptCredentials: (credentials, info) => { saved.push({ credentials, info }); },
credentials: () => ({ accessToken: first, refreshToken: "rt-1" }),
onCredentials: (credentials, info) => { saved.push({ credentials, info }); },
});
assert.equal(result.stopReason, "end_turn");
assert.equal(text, "libfx ok");
@@ -147,6 +148,7 @@ const assertNoSecrets = (value, secrets) => {
assert.equal(replies[1].body, replies[0].body, "the resent request is unchanged");
assert.equal(saved.length, 1, "the host receives the refreshed credentials once");
assert.equal(saved[0].credentials.accessToken, second);
assert.equal(saved[0].info.provider, "codex", "onCredentials learns the provider");
assert.equal(saved[0].credentials.refreshToken, "rt-2");
assert.equal(saved[0].credentials.accountId, accountId);
assert.ok(saved[0].credentials.expiresAt > Date.now() + 9 * 24 * 3600 * 1000);
@@ -158,7 +160,7 @@ const assertNoSecrets = (value, secrets) => {
const expiring = jwt("expiring", 60);
const fresh = jwt("fresh", tenDays);
const world = codexWorld({ refreshedToken: fresh });
const { result } = await runTurn({ fetch: world.fetch, chatgpt: () => ({ accessToken: expiring, refreshToken: "rt-1" }) });
const { result } = await runTurn({ fetch: world.fetch, credentials: () => ({ accessToken: expiring, refreshToken: "rt-1" }) });
assert.equal(result.stopReason, "end_turn");
assert.equal(world.log.filter((entry) => entry.url === "https://auth.openai.com/oauth/token").length, 1);
for (const entry of world.log.filter((entry) => entry.url.startsWith("https://chatgpt.com/"))) {
@@ -177,7 +179,7 @@ const assertNoSecrets = (value, secrets) => {
provider: "codex",
model: "gpt-6.1-sol",
fetch: world.fetch,
chatgpt: (request) => { asked.push(request.sessionId); return { accessToken: access, refreshToken: "rt-1" }; },
credentials: (request) => { asked.push(request.sessionId); return { accessToken: access, refreshToken: "rt-1" }; },
});
try {
const turn = agent.session().prompt("say ok");
@@ -186,7 +188,7 @@ const assertNoSecrets = (value, secrets) => {
for await (const event of turn) if (event.type === "text_delta") text += event.delta;
assert.equal((await turn.result).stopReason, "end_turn");
assert.equal(text, "libfx ok");
assert.ok(asked.includes(sessionId), "chatgpt() is asked for the durable session's own credentials");
assert.ok(asked.includes(sessionId), "credentials() is asked for the durable session's own credentials");
assert.equal(replyRequests(world.log)[0].headers.get("session-id"), sessionId);
assert.equal(world.log.some((entry) => entry.url.includes("ai-gateway.vercel.sh")), false);
} finally {
@@ -195,9 +197,10 @@ const assertNoSecrets = (value, secrets) => {
}
// 4. Options are checked before anything starts.
await assert.rejects(createFxEngine({ ...backendOptions, provider: "codex", model: "gpt-6.1-sol" }), /needs a chatgpt\(\) credentials function/);
await assert.rejects(createFxEngine({ ...backendOptions, provider: "grok", apiKey: "key" }), /provider must be "gateway" or "codex"/);
await assert.rejects(createFxEngine({ ...backendOptions, apiKey: "key", chatgpt: () => ({}) }), /chatgpt needs provider "codex"/);
await assert.rejects(createFxEngine({ ...backendOptions, provider: "codex", model: "gpt-6.1-sol" }), /needs a credentials\(\) function/);
await assert.rejects(createFxEngine({ ...backendOptions, provider: "custom", apiKey: "key" }), /provider must be "gateway", "codex", "grok" or a custom provider/);
await assert.rejects(createFxEngine({ ...backendOptions, apiKey: "key", credentials: () => ({}) }), /credentials needs provider "codex"/);
await assert.rejects(createFxEngine({ ...backendOptions, apiKey: "key", onCredentials: () => {} }), /onCredentials needs provider "codex"/);
await assert.rejects(createFxEngine({ ...backendOptions }), /apiKey is required/);
console.log(`${process.versions.bun ? "Bun" : "Node"} ${backend} Codex agent passed`);
sdk/tests/test-agent-custom.mjs
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env node
// A custom provider agent: the engine's requests reach an OpenAI-compatible
// server through the host fetch, with the host's API key on that server's chat
// URL only, and never touch AI Gateway. No network: every request is answered
// here.
import { strict as assert } from "node:assert";
import { readFile } from "node:fs/promises";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { customProviderDefinition, customProviderSpec } from "../custom-provider.js";
import { createFxAgent, createFxEngine, memory, supportsJspi } from "../node.js";
import { createSubscriptionAuthorizer } from "../subscription.js";
const backend = process.argv[2] || "native";
if (!new Set(["native", "wasm"]).has(backend)) throw new Error("usage: test-agent-custom.mjs [native|wasm]");
if (backend === "wasm" && !supportsJspi()) {
console.error("Node JSPI is disabled. Run with --experimental-wasm-jspi");
process.exit(2);
}
const scriptDir = fileURLToPath(new URL(".", import.meta.url));
const backendOptions = {
backend,
nativeAddon: resolve(scriptDir, "../../zig-out/lib/libfx.node"),
...(backend === "wasm" ? { wasm: await readFile(resolve(scriptDir, "../../zig-out/bin/fx-core.wasm")) } : {}),
};
const baseUrl = "https://models.example/v1/";
const chatUrl = "https://models.example/v1/chat/completions";
const provider = { name: "local", baseUrl, models: { "local-model": { contextWindow: 32768, maxOutputTokens: 4096, supportsToolUse: true } } };
const sse = (...chunks) => chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n";
const reply = (text) => sse(
{ id: "chat-1", model: "local-model", choices: [{ index: 0, delta: { role: "assistant", content: text }, finish_reason: null }] },
{ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] },
);
const toolCall = sse(
{ choices: [{ index: 0, delta: { role: "assistant", tool_calls: [{ index: 0, id: "call-1", type: "function", function: { name: "lookup", arguments: "{\"key\":\"alpha\"}" } }] }, finish_reason: null }] },
{ choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] },
);
/** A host fetch that plays an OpenAI-compatible server; `answer(index, request)` picks each reply. */
function server({ answer = () => reply("libfx on a custom provider") } = {}) {
const log = [];
const fetch = async (input, init = {}) => {
const url = String(input?.url ?? input);
const method = (init.method ?? "GET").toUpperCase();
const headers = new Headers(init.headers);
const body = init.body === undefined ? undefined : typeof init.body === "string" ? init.body : new TextDecoder().decode(init.body);
const entry = { method, url, headers, body };
log.push(entry);
if (url === chatUrl && method === "POST") {
const answered = answer(log.filter((logged) => logged.url === chatUrl).length - 1, entry);
if (answered instanceof Response) return answered;
return new Response(answered, { status: 200, headers: { "content-type": "text/event-stream" } });
}
return new Response(`unexpected ${method} ${url}`, { status: 599 });
};
return { fetch, log };
}
async function runTurn(options, prompt = "say ok") {
const events = [];
const engine = await createFxEngine({ ...backendOptions, provider, model: "local-model", onEvent: (event) => events.push(event), ...options });
try {
const turn = engine.prompt(prompt);
let text = "";
for await (const event of turn) if (event.type === "text_delta") text += event.delta;
return { result: await turn.result, text, events, sessionId: engine.sessionId };
} finally {
await engine.close();
}
}
const chatRequests = (log) => log.filter((entry) => entry.url === chatUrl);
const assertNoGateway = (log) => assert.equal(log.some((entry) => entry.url.includes("ai-gateway.vercel.sh")), false, "a custom provider agent must not call AI Gateway");
// 1. A turn on the host's API key reaches only the provider's chat URL.
{
const world = server();
const asked = [];
const { result, text, events, sessionId } = await runTurn({
fetch: world.fetch,
credentials: (request) => { asked.push(request); return { apiKey: "sk-provider" }; },
});
assert.equal(result.stopReason, "end_turn");
assert.equal(text, "libfx on a custom provider");
assertNoGateway(world.log);
assert.deepEqual(world.log.map((entry) => entry.url), [chatUrl], "the only request is the chat request");
const [request] = chatRequests(world.log);
assert.equal(request.headers.get("authorization"), "Bearer sk-provider");
assert.equal(request.headers.get("accept"), "text/event-stream");
const payload = JSON.parse(request.body);
assert.equal(payload.model, "local-model");
assert.equal(payload.stream, true);
assert.match(JSON.stringify(payload.messages), /say ok/);
assert.ok(asked.length > 0 && asked.every((entry) => entry.provider === "local"), "credentials() learns the provider's name");
assert.ok(asked.some((entry) => entry.sessionId === sessionId), "credentials() learns which session the request is for");
assert.equal(JSON.stringify(events).includes("sk-provider"), false, "libfx diagnostics must not expose the API key");
}
// 2. Without credentials(), requests carry no key, as a local server expects.
{
const world = server();
const { result, text } = await runTurn({ fetch: world.fetch });
assert.equal(result.stopReason, "end_turn");
assert.equal(text, "libfx on a custom provider");
assert.equal(chatRequests(world.log)[0].headers.get("authorization"), null);
}
// 3. A tool call round-trips through Chat Completions.
{
const world = server({ answer: (index) => index === 0 ? toolCall : reply("alpha is beta") });
const lookup = {
name: "lookup",
description: "Looks up a key",
inputSchema: { type: "object", properties: { key: { type: "string" } }, required: ["key"] },
execute: async ({ key }) => `value of ${key} is beta`,
};
const { result, text } = await runTurn({ fetch: world.fetch, credentials: () => ({ apiKey: "sk-provider" }), tools: [lookup] }, "look up alpha");
assert.equal(result.stopReason, "end_turn");
assert.equal(text, "alpha is beta");
const [first, second] = chatRequests(world.log);
assert.ok(JSON.parse(first.body).tools.some((tool) => tool.function?.name === "lookup"), "the model is offered the host tool");
const messages = JSON.parse(second.body).messages;
const call = messages.find((message) => message.role === "assistant" && message.tool_calls);
assert.equal(call.tool_calls[0].id, "call-1");
const answer = messages.find((message) => message.role === "tool");
assert.equal(answer.tool_call_id, "call-1");
assert.match(JSON.stringify(answer.content), /value of alpha is beta/);
assert.equal(second.headers.get("authorization"), "Bearer sk-provider");
}
// 4. A rejected key is not refreshed or resent; the turn reports the failure.
{
const world = server({ answer: () => Response.json({ error: { message: "invalid api key" } }, { status: 401 }) });
let asked = 0;
const outcome = await runTurn({ fetch: world.fetch, credentials: () => { asked += 1; return { apiKey: "sk-wrong" }; } }).catch((error) => ({ error }));
assert.ok(outcome.error || outcome.result.stopReason !== "end_turn", "the turn does not finish on a rejected key");
assert.equal(chatRequests(world.log).length, 1, "a rejected request is not resent");
assert.ok(asked >= 1);
}
// 5. A durable session continues in a new agent, and follows the agent's
// provider to another base URL with its history, like a change of model.
{
const durability = memory();
const world = server({ answer: (index) => reply(index === 0 ? "first reply" : "second reply") });
const agentOptions = (overrides = {}) => ({
...backendOptions,
durability,
provider,
model: "local-model",
fetch: world.fetch,
credentials: () => ({ apiKey: "sk-provider" }),
...overrides,
});
const first = createFxAgent(agentOptions());
let id;
try {
const session = first.session();
assert.equal((await session.prompt("first").result).stopReason, "end_turn");
id = session.id;
} finally {
await first.close();
}
const second = createFxAgent(agentOptions());
try {
const turn = second.session(id).prompt("second");
let text = "";
for await (const event of turn) if (event.type === "text_delta") text += event.delta;
assert.equal((await turn.result).stopReason, "end_turn");
assert.equal(text, "second reply");
const resumed = JSON.parse(chatRequests(world.log)[1].body).messages;
assert.match(JSON.stringify(resumed), /first reply/, "the resumed session keeps its history");
} finally {
await second.close();
}
const elsewhereUrl = "https://elsewhere.example/v1/chat/completions";
const elsewhere = [];
const moved = createFxAgent(agentOptions({
provider: { ...provider, baseUrl: "https://elsewhere.example/v1" },
fetch: async (input, init = {}) => {
const url = String(input?.url ?? input);
const body = typeof init.body === "string" ? init.body : new TextDecoder().decode(init.body);
elsewhere.push({ url, headers: new Headers(init.headers), body });
if (url !== elsewhereUrl) return new Response("unexpected", { status: 599 });
return new Response(reply("third reply"), { status: 200, headers: { "content-type": "text/event-stream" } });
},
}));
const before = world.log.length;
try {
assert.equal((await moved.session(id).prompt("third").result).stopReason, "end_turn");
} finally {
await moved.close();
}
assert.deepEqual(elsewhere.map((entry) => entry.url), [elsewhereUrl]);
assert.equal(elsewhere[0].headers.get("authorization"), "Bearer sk-provider");
assert.match(elsewhere[0].body, /second reply/, "the session's history goes with it");
assert.equal(world.log.length, before, "the first server hears nothing more");
assertNoGateway(world.log);
}
// 6. A server that quotes the key back in an error never gets it into the
// turn, its events, or the saved session, even escaped.
{
const key = "sk-quoted-back";
const escaped = [...key].map((char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`).join("");
const world = server({
answer: (_index, request) => new Response(
`{"detail":"bad credential ${request.headers.get("authorization")}","escaped":"${escaped}"}`,
{ status: 403, headers: { "content-type": "application/json" } },
),
});
const durability = memory();
const events = [];
const streamed = [];
const agent = createFxAgent({
...backendOptions,
durability,
provider,
model: "local-model",
fetch: world.fetch,
credentials: () => ({ apiKey: key }),
onEvent: (event) => events.push(event),
});
let sessionId;
try {
const turn = agent.session().prompt("say ok");
({ sessionId } = await turn.accepted);
for await (const event of turn) streamed.push(event);
assert.notEqual((await turn.result).stopReason, "end_turn");
} finally {
await agent.close();
}
assert.equal(chatRequests(world.log)[0].headers.get("authorization"), `Bearer ${key}`, "the key reached the server");
assert.match(JSON.stringify(streamed), /bad credential Bearer \*{14}/, "the error reaches the turn with the key masked");
// Everything the session saved: its log, with byte payloads as text, and its UI stream.
const stored = await durability.create().session(sessionId);
const log = JSON.stringify(await stored.readSince(0), (_, value) => value instanceof Uint8Array ? Buffer.from(value).toString("latin1") : value);
const lines = await stored.ui.length();
assert.ok(lines > 0, "the session saved its UI stream");
const reader = stored.ui.read(0).getReader();
let ui = "";
for (let line = 0; line < lines; line += 1) ui += new TextDecoder().decode((await reader.read()).value);
await reader.cancel();
// Undoes \u escapes at any depth of JSON nesting, so an escaped copy counts.
const decoded = (text) => text.replace(/\\+u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
for (const [where, text] of [["the stream", JSON.stringify(streamed)], ["the events", JSON.stringify(events)], ["the saved log", log], ["the saved UI stream", ui]]) {
assert.equal(decoded(text).includes(key), false, `${where} must not repeat the key`);
}
}
// 7. The key is added to the provider's chat URL and nowhere else, as the
// engine names a request: origin and path.
{
const definition = customProviderDefinition(provider);
const auth = createSubscriptionAuthorizer([customProviderSpec(definition, () => undefined)], { credentials: () => ({ apiKey: "sk-provider" }) });
assert.ok(auth.specFor(chatUrl), "the chat URL carries the key");
for (const endpoint of [
"https://models.example/v1/models",
"https://models.example/v1/chat/completions/extra",
"https://models.example/chat/completions",
"http://models.example/v1/chat/completions",
"https://models.example.evil/v1/chat/completions",
"https://ai-gateway.vercel.sh/v1/chat/completions",
]) {
assert.equal(auth.specFor(endpoint), undefined, `${endpoint} must not carry the key`);
}
}
// 8. Options are checked before anything starts.
const base = { ...backendOptions, model: "local-model" };
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, name: "codex" } }), /provider.name must be/);
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, name: "1local" } }), /provider.name must be/);
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, baseUrl: "http://models.example/v1" } }), /https, or http on localhost/);
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, baseUrl: "https://models.example/v1?key=1" } }), /credentials, a query or a fragment/);
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, apiKey: "sk" } }), /unknown field apiKey/);
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, models: { "local-model": { context: 1 } } } }), /unknown field context/);
await assert.rejects(createFxEngine({ ...backendOptions, provider }), /a custom provider needs a model/);
await assert.rejects(createFxEngine({ ...base, provider, credentials: "sk" }), /credentials must be a function/);
await assert.rejects(createFxEngine({ ...base, provider, onCredentials: () => {} }), /onCredentials needs provider "codex" or "grok"/);
await assert.rejects(createFxEngine({ ...base, provider, refreshLock: async (_info, refresh) => refresh() }), /refreshLock needs provider "codex" or "grok"/);
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, models: { "local-model": { contextWindow: 100, maxOutputTokens: 100 } } } }), /maxOutputTokens must be less than contextWindow/);
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, models: { "local-model": { supportsVision: "yes" } } } }), /supportsVision must be a boolean/);
// The engine applies the CLI's complete rules to what passes the quick checks.
await assert.rejects(createFxEngine({ ...base, provider: { ...provider, baseUrl: "https://models_example.test/v1" } }));
console.log(`${process.versions.bun ? "Bun" : "Node"} ${backend} custom provider agent passed`);
sdk/tests/test-agent-grok.mjs
+380
View File
@@ -0,0 +1,380 @@
#!/usr/bin/env node
// A Grok (Grok plan) agent: the engine's requests reach xAI's subscription
// proxy through the host fetch with the host's Grok credentials, refreshed
// when needed, and never touch AI Gateway. No network: every request is
// answered here.
import { strict as assert } from "node:assert";
import { readFile } from "node:fs/promises";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { createFxAgent, createFxEngine, memory, supportsJspi } from "../node.js";
const backend = process.argv[2] || "native";
if (!new Set(["native", "wasm"]).has(backend)) throw new Error("usage: test-agent-grok.mjs [native|wasm]");
if (backend === "wasm" && !supportsJspi()) {
console.error("Node JSPI is disabled. Run with --experimental-wasm-jspi");
process.exit(2);
}
const scriptDir = fileURLToPath(new URL(".", import.meta.url));
const backendOptions = {
backend,
nativeAddon: resolve(scriptDir, "../../zig-out/lib/libfx.node"),
...(backend === "wasm" ? { wasm: await readFile(resolve(scriptDir, "../../zig-out/bin/fx-core.wasm")) } : {}),
};
const clientId = "b1a00492-073a-47ea-816f-4c329264a828";
const accountId = "xai-account-fixture";
const sixHours = 6 * 3600 * 1000;
const responsesUrl = "https://cli-chat-proxy.grok.com/v1/responses";
const modelsUrl = "https://cli-chat-proxy.grok.com/v1/models";
const languageModelsUrl = "https://api.x.ai/v1/language-models";
const tokenUrl = "https://auth.x.ai/oauth2/token";
const userinfoUrl = "https://auth.x.ai/oauth2/userinfo";
const models = {
data: [
{ id: "grok-4.5", model: "grok-4.5", api_backend: "responses", context_window: 500000, supports_reasoning_effort: true, reasoning_efforts: [{ value: "low" }, { value: "high" }] },
{ id: "grok-4.7", model: "grok-4.7", api_backend: "responses", context_window: 500000, supports_reasoning_effort: false, reasoning_efforts: [] },
],
};
const languageModels = { models: [{ id: "grok-4.5", input_modalities: ["text", "image"], output_modalities: ["text"] }] };
const replySse =
'data: {"type":"response.output_item.added","output_index":0,"item":{"type":"message"}}\n\n' +
'data: {"type":"response.output_text.delta","output_index":0,"delta":"libfx on grok"}\n\n' +
'data: {"type":"response.completed","response":{"status":"completed","usage":{"input_tokens":23,"output_tokens":7}}}\n\n';
/** A host fetch that plays x.ai's version file, the subscription proxy, and xAI's OAuth endpoints. */
function grokWorld({ rejectFirstReply = false, rejectToken, refreshedToken = "unused", refreshedAccount = accountId }) {
const log = [];
let replies = 0;
const fetch = async (input, init = {}) => {
const url = String(input?.url ?? input);
const method = (init.method ?? "GET").toUpperCase();
const headers = new Headers(init.headers);
const body = init.body === undefined ? undefined : typeof init.body === "string" ? init.body : new TextDecoder().decode(init.body);
log.push({ method, url, headers, body });
if (url === "https://x.ai/cli/stable") return new Response("1.0.50\n");
if (url === modelsUrl) return Response.json(models);
if (url === languageModelsUrl) return Response.json(languageModels);
if (url === tokenUrl && method === "POST") {
return Response.json({ access_token: refreshedToken, refresh_token: "rt-2", expires_in: 21600, token_type: "Bearer" });
}
if (url === userinfoUrl) {
const token = headers.get("authorization")?.replace(/^Bearer /, "");
return Response.json({ sub: token === refreshedToken ? refreshedAccount : accountId, email: "user@example.com" });
}
if (url === responsesUrl && method === "POST") {
replies += 1;
if (rejectFirstReply && replies === 1) return Response.json({ error: "token expired" }, { status: 401 });
if (rejectToken && headers.get("authorization") === `Bearer ${rejectToken}`) return Response.json({ error: "token expired" }, { status: 401 });
return new Response(replySse, { status: 200, headers: { "content-type": "text/event-stream" } });
}
return new Response(`unexpected ${method} ${url}`, { status: 599 });
};
return { fetch, log };
}
async function runTurn(options) {
const events = [];
const engine = await createFxEngine({ ...backendOptions, provider: "grok", model: "grok-4.5", onEvent: (event) => events.push(event), ...options });
try {
const turn = engine.prompt("say ok");
let text = "";
for await (const event of turn) if (event.type === "text_delta") text += event.delta;
return { result: await turn.result, text, events };
} finally {
await engine.close();
}
}
const credentialsFor = (accessToken, extra = {}) => ({ accessToken, refreshToken: "rt-1", accountId, expiresAt: Date.now() + sixHours, ...extra });
const replyRequests = (log) => log.filter((entry) => entry.url === responsesUrl);
const assertNoSecrets = (value, secrets) => {
const text = JSON.stringify(value);
for (const secret of secrets) assert.equal(text.includes(secret), false, "libfx diagnostics must not expose Grok credentials");
};
// 1. A Grok turn on the host's credentials, with no Gateway request.
{
const world = grokWorld({});
const asked = [];
const { result, text, events } = await runTurn({
fetch: world.fetch,
credentials: (request) => { asked.push(request); return credentialsFor("at-first"); },
});
assert.equal(result.stopReason, "end_turn");
assert.equal(text, "libfx on grok");
assert.equal(world.log.some((entry) => entry.url.includes("ai-gateway.vercel.sh")), false, "a Grok agent must not call AI Gateway");
assert.equal(world.log.some((entry) => entry.url.startsWith("https://auth.x.ai/")), false, "fresh credentials need no refresh");
const [reply] = replyRequests(world.log);
assert.ok(reply, "the turn must reach Grok");
assert.equal(reply.headers.get("authorization"), "Bearer at-first");
assert.equal(reply.headers.get("x-grok-user-id"), accountId);
assert.equal(reply.headers.get("X-XAI-Token-Auth"), "xai-grok-cli");
assert.equal(reply.headers.get("x-authenticateresponse"), "authenticate-response");
assert.equal(reply.headers.get("x-grok-client-version"), "1.0.50");
assert.equal(reply.headers.get("x-grok-client-identifier"), "fx");
assert.equal(reply.headers.get("x-grok-model-override"), "grok-4.5");
const sessionId = reply.headers.get("x-grok-conv-id");
assert.ok(sessionId, "Grok requests carry the session id");
const payload = JSON.parse(reply.body);
assert.equal(payload.model, "grok-4.5");
assert.equal(payload.store, false);
assert.equal(payload.stream, true);
assert.match(JSON.stringify(payload.input), /say ok/);
const list = world.log.find((entry) => entry.url === modelsUrl);
if (list) {
assert.equal(list.headers.get("authorization"), "Bearer at-first", "the model list uses the same credentials");
assert.equal(list.headers.get("x-userid"), accountId);
assert.equal(list.headers.get("x-grok-user-id"), null);
}
const enrichment = world.log.find((entry) => entry.url === languageModelsUrl);
if (enrichment) {
assert.equal(enrichment.headers.get("authorization"), "Bearer at-first");
assert.equal(enrichment.headers.get("x-userid"), null, "only the subscription proxy hears the account id");
}
assert.ok(asked.some((request) => request.sessionId === sessionId), "credentials() learns which session the request is for");
assert.ok(asked.length > 0 && asked.every((request) => request.provider === "grok"), "credentials() learns the provider");
assertNoSecrets(events, ["at-first", "rt-1"]);
}
// 2. A rejected token is refreshed once, checked against the account, and the request sent again.
{
const world = grokWorld({ rejectFirstReply: true, refreshedToken: "at-second" });
const saved = [];
const { result, text, events } = await runTurn({
fetch: world.fetch,
credentials: () => credentialsFor("at-rejected"),
onCredentials: (credentials, info) => { saved.push({ credentials, info }); },
});
assert.equal(result.stopReason, "end_turn");
assert.equal(text, "libfx on grok");
const refreshes = world.log.filter((entry) => entry.url === tokenUrl);
assert.equal(refreshes.length, 1, "one rejection causes one refresh");
assert.equal(refreshes[0].headers.get("content-type"), "application/x-www-form-urlencoded");
assert.deepEqual(Object.fromEntries(new URLSearchParams(refreshes[0].body)), { grant_type: "refresh_token", client_id: clientId, refresh_token: "rt-1" });
const checks = world.log.filter((entry) => entry.url === userinfoUrl);
assert.equal(checks.length, 1, "the refreshed token's account is checked");
assert.equal(checks[0].headers.get("authorization"), "Bearer at-second");
const replies = replyRequests(world.log);
assert.equal(replies.length, 2);
assert.equal(replies[0].headers.get("authorization"), "Bearer at-rejected");
assert.equal(replies[1].headers.get("authorization"), "Bearer at-second");
assert.equal(replies[1].body, replies[0].body, "the resent request is unchanged");
assert.equal(saved.length, 1, "the host receives the refreshed credentials once");
assert.equal(saved[0].info.provider, "grok", "onCredentials learns the provider");
assert.equal(saved[0].credentials.accessToken, "at-second");
assert.equal(saved[0].credentials.refreshToken, "rt-2");
assert.equal(saved[0].credentials.accountId, accountId);
assert.ok(saved[0].credentials.expiresAt > Date.now() + 5 * 3600 * 1000);
assertNoSecrets(events, ["at-rejected", "at-second", "rt-1", "rt-2"]);
}
// 3. Credentials about to expire are refreshed before the request.
{
const world = grokWorld({ refreshedToken: "at-fresh" });
const { result } = await runTurn({ fetch: world.fetch, credentials: () => credentialsFor("at-expiring", { expiresAt: Date.now() + 60_000 }) });
assert.equal(result.stopReason, "end_turn");
assert.equal(world.log.filter((entry) => entry.url === tokenUrl).length, 1);
for (const entry of world.log.filter((entry) => entry.url.startsWith("https://cli-chat-proxy.grok.com/"))) {
assert.equal(entry.headers.get("authorization"), "Bearer at-fresh", "no Grok request uses the expiring token");
}
}
// 4. A refresh that comes back for another account is never used or saved.
{
// Every request on the user's own token is rejected, so only a refresh could finish the turn.
const world = grokWorld({ rejectToken: "at-mine", refreshedToken: "at-other", refreshedAccount: "someone-else" });
const saved = [];
const outcome = await runTurn({
fetch: world.fetch,
credentials: () => credentialsFor("at-mine"),
onCredentials: (credentials) => { saved.push(credentials); },
}).catch((error) => ({ error }));
assert.ok(outcome.error || outcome.result.stopReason !== "end_turn", "the turn does not finish on another account's token");
assert.equal(saved.length, 0, "the host never receives another account's credentials");
assert.ok(world.log.some((entry) => entry.url === userinfoUrl), "the refreshed token was checked");
for (const entry of world.log.filter((entry) => entry.url.startsWith("https://cli-chat-proxy.grok.com/"))) {
assert.notEqual(entry.headers.get("authorization"), "Bearer at-other", "no Grok request uses another account's token");
}
}
// 5. A durable session asks for its own session's credentials.
{
const world = grokWorld({});
const asked = [];
const agent = createFxAgent({
...backendOptions,
durability: memory(),
provider: "grok",
model: "grok-4.5",
fetch: world.fetch,
credentials: (request) => { asked.push(request.sessionId); return credentialsFor("at-durable"); },
});
try {
const turn = agent.session().prompt("say ok");
const { sessionId } = await turn.accepted;
let text = "";
for await (const event of turn) if (event.type === "text_delta") text += event.delta;
assert.equal((await turn.result).stopReason, "end_turn");
assert.equal(text, "libfx on grok");
assert.ok(asked.includes(sessionId), "credentials() is asked for the durable session's own credentials");
assert.equal(replyRequests(world.log)[0].headers.get("x-grok-conv-id"), sessionId);
assert.equal(world.log.some((entry) => entry.url.includes("ai-gateway.vercel.sh")), false);
} finally {
await agent.close();
}
}
// 6. One agent's sessions refresh an account once, and each session hears the
// refreshed credentials, for a host that keeps them per session.
{
const world = grokWorld({ refreshedToken: "at-shared" });
const saved = [];
const agent = createFxAgent({
...backendOptions,
durability: memory(),
provider: "grok",
model: "grok-4.5",
fetch: world.fetch,
credentials: () => credentialsFor("at-expiring", { expiresAt: Date.now() + 60_000 }),
onCredentials: (credentials, { sessionId }) => { saved.push({ accessToken: credentials.accessToken, sessionId }); },
});
try {
const turns = [agent.session().prompt("say ok"), agent.session().prompt("say ok")];
const sessionIds = await Promise.all(turns.map(async (turn) => (await turn.accepted).sessionId));
for (const turn of turns) {
for await (const event of turn) void event;
assert.equal((await turn.result).stopReason, "end_turn");
}
assert.equal(world.log.filter((entry) => entry.url === tokenUrl).length, 1, "the sessions share one refresh");
for (const sessionId of sessionIds) {
assert.ok(saved.some((entry) => entry.sessionId === sessionId && entry.accessToken === "at-shared"), "each session hears the refreshed credentials");
}
} finally {
await agent.close();
}
}
// 7. A rejection that arrives after another session refreshed retries on the
// new token instead of spending the old refresh token again.
{
const world = grokWorld({ refreshedToken: "at-new" });
let rejected = 0;
let secondSent;
const bothSent = new Promise((resolve) => { secondSent = resolve; });
let releaseLate;
const late = new Promise((resolve) => { releaseLate = resolve; });
const fetch = async (input, init = {}) => {
const url = String(input?.url ?? input);
// The refresh finishes only once both sessions have sent the old token.
if (url === tokenUrl) await bothSent;
if (url === responsesUrl) {
const authorization = new Headers(init.headers).get("authorization");
if (authorization === "Bearer at-old") {
rejected += 1;
if (rejected === 2) {
secondSent();
// Rejected only after the first session's refresh has finished.
await late;
}
return Response.json({ error: "token expired" }, { status: 401 });
}
if (authorization === "Bearer at-new") releaseLate();
}
return world.fetch(input, init);
};
const agent = createFxAgent({
...backendOptions,
durability: memory(),
provider: "grok",
model: "grok-4.5",
fetch,
credentials: () => credentialsFor("at-old"),
});
try {
const turns = [agent.session().prompt("say ok"), agent.session().prompt("say ok")];
for (const turn of turns) {
for await (const event of turn) void event;
assert.equal((await turn.result).stopReason, "end_turn");
}
assert.equal(rejected, 2);
assert.equal(world.log.filter((entry) => entry.url === tokenUrl).length, 1, "a late rejection does not refresh again");
assert.equal(replyRequests(world.log).filter((entry) => entry.headers.get("authorization") === "Bearer at-new").length, 2);
} finally {
await agent.close();
}
}
// 8. Two servers that share a lock refresh an account once: the second waits,
// then uses the credentials the first saved.
{
const world = grokWorld({ refreshedToken: "at-renewed" });
let stored = credentialsFor("at-expiring", { expiresAt: Date.now() + 60_000 });
// The refresh finishes only once both servers have asked for credentials, so
// without the lock each would spend the refresh token.
const asked = new Set();
let bothAsked;
const ready = new Promise((resolve) => { bothAsked = resolve; });
const fetch = async (input, init = {}) => {
if (String(input?.url ?? input) === tokenUrl) await ready;
return world.fetch(input, init);
};
let held = Promise.resolve();
const refreshLock = (_info, refresh) => {
const run = held.then(refresh);
held = run.catch(() => {});
return run;
};
const server = (name) => createFxAgent({
...backendOptions,
durability: memory(),
provider: "grok",
model: "grok-4.5",
fetch,
credentials: () => {
asked.add(name);
if (asked.size === 2) bothAsked();
return stored;
},
onCredentials: (credentials) => { stored = credentials; },
refreshLock,
});
const servers = [server("a"), server("b")];
try {
const turns = servers.map((agent) => agent.session().prompt("say ok"));
for (const turn of turns) {
for await (const event of turn) void event;
assert.equal((await turn.result).stopReason, "end_turn");
}
assert.equal(world.log.filter((entry) => entry.url === tokenUrl).length, 1, "servers that share a lock refresh an account once");
assert.equal(replyRequests(world.log).filter((entry) => entry.headers.get("authorization") === "Bearer at-renewed").length, 2);
} finally {
await Promise.all(servers.map((agent) => agent.close()));
}
}
// A lock that never runs the refresh fails the request instead of sending the
// expiring credentials.
{
const world = grokWorld({ refreshedToken: "at-unused" });
const outcome = await runTurn({
fetch: world.fetch,
credentials: () => credentialsFor("at-expiring", { expiresAt: Date.now() + 60_000 }),
refreshLock: async () => {},
}).catch((error) => ({ error }));
assert.ok(outcome.error || outcome.result.stopReason !== "end_turn", "the turn does not finish without a refresh");
assert.equal(world.log.filter((entry) => entry.url === tokenUrl).length, 0);
assert.equal(replyRequests(world.log).length, 0, "nothing is sent on credentials the lock never refreshed");
}
// 9. Options are checked before anything starts.
await assert.rejects(createFxEngine({ ...backendOptions, provider: "grok", model: "grok-4.5" }), /provider "grok" needs a credentials\(\) function/);
await assert.rejects(
createFxEngine({ ...backendOptions, provider: "grok", model: "grok-4.5", credentials: () => credentialsFor("at-first"), refreshLock: "lock" }),
/refreshLock must be a function/,
);
await assert.rejects(createFxEngine({ ...backendOptions, apiKey: "key", refreshLock: async (_info, refresh) => refresh() }), /refreshLock needs provider "codex" or "grok"/);
console.log(`${process.versions.bun ? "Bun" : "Node"} ${backend} Grok agent passed`);
sdk/tests/test-durable-codex-crash.mjs
-243
View File
@@ -1,243 +0,0 @@
#!/usr/bin/env node
// A durable Codex session survives a SIGKILL. A child process runs a turn on a
// local() durability and is killed during the model's reply or during a tool
// call; a fresh agent here resumes the turn on the credentials its own
// chatgpt() supplies. No ChatGPT token reaches the saved session. No network.
//
// node test-durable-codex-crash.mjs [native|wasm]
import { strict as assert } from "node:assert";
import { spawn } from "node:child_process";
import { mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { createFxAgent, supportsJspi } from "../node.js";
import { local } from "../durable/local.mjs";
const args = process.argv.slice(2);
const childAt = args.indexOf("--child");
const backend = args[0] && args[0] !== "--child" ? args[0] : "native";
if (!new Set(["native", "wasm"]).has(backend)) throw new Error("usage: test-durable-codex-crash.mjs [native|wasm]");
if (backend === "wasm" && !supportsJspi()) {
console.error("Node JSPI is disabled. Run with --experimental-wasm-jspi");
process.exit(2);
}
const scriptDir = fileURLToPath(new URL(".", import.meta.url));
const backendOptions = {
backend,
nativeAddon: resolve(scriptDir, "../../zig-out/lib/libfx.node"),
...(backend === "wasm" ? { wasm: await readFile(resolve(scriptDir, "../../zig-out/bin/fx-core.wasm")) } : {}),
};
const accountId = "acct-crash";
function jwt(label) {
const part = (value) => Buffer.from(JSON.stringify(value)).toString("base64url");
return `${part({ alg: "none" })}.${part({ exp: Math.floor(Date.now() / 1000) + 864000, "https://api.openai.com/auth": { chatgpt_account_id: accountId }, label })}.sig`;
}
// The dying process and the resuming one hold different tokens, so a request
// can show which process authorized it.
const childToken = jwt("child-process");
const parentToken = jwt("resuming-process");
const models = {
models: [
{ slug: "gpt-6.1-sol", visibility: "list", supported_in_api: true, priority: 1, supported_reasoning_levels: [{ effort: "low" }], additional_speed_tiers: [], input_modalities: ["text"], context_window: 272000 },
{ slug: "gpt-5.6-luna", visibility: "list", supported_in_api: true, priority: 2, supported_reasoning_levels: [{ effort: "low" }], additional_speed_tiers: [], input_modalities: ["text"], context_window: 272000 },
],
};
const event = (value) => `data: ${JSON.stringify(value)}\n\n`;
const textReply = (text) =>
event({ type: "response.output_item.added", output_index: 0, item: { type: "message" } }) +
event({ type: "response.output_text.delta", output_index: 0, delta: text }) +
event({ type: "response.completed", response: { status: "completed", usage: { input_tokens: 5, output_tokens: 2 } } });
const lookupCall =
event({ type: "response.output_item.added", output_index: 0, item: { type: "function_call", call_id: "call_lookup", name: "lookup" } }) +
event({ type: "response.function_call_arguments.delta", output_index: 0, delta: JSON.stringify({ key: "order-7" }) }) +
event({ type: "response.completed", response: { status: "completed", usage: { input_tokens: 5, output_tokens: 2 } } });
/** A fake Codex. `holdReply` streams the first event of a reply and then never finishes it. */
function codex({ holdReply = false, onReply } = {}) {
const log = [];
const fetch = async (input, init = {}) => {
const url = String(input?.url ?? input);
const method = (init.method ?? "GET").toUpperCase();
const body = init.body === undefined ? undefined : typeof init.body === "string" ? init.body : new TextDecoder().decode(init.body);
log.push({ method, url, headers: new Headers(init.headers), body });
if (url === "https://registry.npmjs.org/@openai/codex/latest") return Response.json({ version: "0.162.1" });
if (url.startsWith("https://chatgpt.com/backend-api/codex/models")) return Response.json(models);
if (url === "https://chatgpt.com/backend-api/codex/responses" && method === "POST") {
const payload = JSON.parse(body);
const sawLookup = JSON.stringify(payload.input).includes("function_call_output");
const wantsLookup = JSON.stringify(payload.input).includes("use lookup");
// A request that allows no tool calls gets text, as a real model gives.
const reply = wantsLookup && !sawLookup && payload.tool_choice !== "none"
? lookupCall
: textReply(sawLookup ? "found order-7" : wantsLookup ? "could not confirm order-7" : "hello again");
onReply?.(payload);
if (!holdReply) return new Response(reply, { status: 200, headers: { "content-type": "text/event-stream" } });
const firstEvent = reply.slice(0, reply.indexOf("\n\n") + 2);
return new Response(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(firstEvent)); } }), {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response(`unexpected ${method} ${url}`, { status: 599 });
};
return { fetch, log };
}
function agentOptions(dir, { fetch, token, asked, lookupRuns, lookupStarted, events }) {
return {
...backendOptions,
...(events ? { onEvent: (event) => events.push(event) } : {}),
durability: local({ dir }),
provider: "codex",
model: "gpt-6.1-sol",
fetch,
chatgpt: (request) => { asked?.push(request.sessionId); return { accessToken: token, refreshToken: "rt-crash" }; },
tools: [{
name: "lookup",
description: "Looks up an order",
idempotent: true,
inputSchema: { type: "object", properties: { key: { type: "string" } } },
execute: async ({ key }, { sessionId }) => {
lookupRuns?.push(sessionId);
if (lookupStarted) {
// A call starts only after its intent is stored, so the crash
// comes the moment the tool runs.
lookupStarted();
await new Promise(() => {}); // the process dies here
}
return { content: `${key} shipped` };
},
}],
};
}
// The child: start a turn, report the session once the crash point is reached, wait to be killed.
if (childAt !== -1) {
const [dir, phase] = args.slice(childAt + 1);
let reached;
const crashPoint = new Promise((resolveReached) => { reached = resolveReached; });
const fake = codex({ holdReply: phase === "reply", onReply: () => { if (phase === "reply") reached(); } });
const agent = createFxAgent(agentOptions(dir, {
fetch: fake.fetch,
token: childToken,
...(phase === "tool" ? { lookupStarted: () => reached() } : {}),
}));
const turn = agent.session().prompt(phase === "tool" ? "use lookup for order-7" : "say hello");
const { sessionId } = await turn.accepted;
await crashPoint;
process.stdout.write(`${JSON.stringify({ sessionId })}\n`);
await new Promise(() => {});
}
// True when the text, or base64 found in it decoded up to two levels deep,
// contains any of the secrets. The local World stores entries as base64.
function holdsSecret(text, secrets, depth = 2) {
if (secrets.some((secret) => text.includes(secret))) return true;
if (depth === 0) return false;
for (const [encoded] of text.matchAll(/[A-Za-z0-9+/_-]{24,}={0,2}/g)) {
const decoded = Buffer.from(encoded, "base64").toString("latin1");
if (holdsSecret(decoded, secrets, depth - 1)) return true;
}
return false;
}
async function filesUnder(dir) {
const out = [];
for (const entry of await readdir(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
if (entry.isDirectory()) out.push(...await filesUnder(path));
else if ((await stat(path)).size < 16 * 1024 * 1024) out.push(path);
}
return out;
}
async function crashAndResume(phase) {
const dir = await mkdtemp(join(tmpdir(), "libfx-codex-crash-"));
try {
const child = spawn(process.execPath, [...process.execArgv, fileURLToPath(import.meta.url), backend, "--child", dir, phase], {
stdio: ["ignore", "pipe", "inherit"],
env: { ...process.env },
});
let output = "";
child.stdout.on("data", (chunk) => { output += chunk; });
const started = Date.now();
while (!output.includes("\n")) {
if (child.exitCode !== null) throw new Error(`the ${phase} child exited early with ${child.exitCode}`);
if (Date.now() - started > 30_000) throw new Error(`the ${phase} child never reached its crash point`);
await new Promise((wait) => setTimeout(wait, 20));
}
const { sessionId } = JSON.parse(output.trim().split("\n")[0]);
child.kill("SIGKILL");
const [, signal] = await new Promise((exited) => child.once("exit", (code, sig) => exited([code, sig])));
assert.equal(signal, "SIGKILL", "the child died by SIGKILL, mid-turn");
const fake = codex();
const asked = [];
const lookupRuns = [];
const events = [];
const agent = createFxAgent(agentOptions(dir, { fetch: fake.fetch, token: parentToken, asked, lookupRuns, events }));
let text = "";
const lines = [];
try {
const resumed = agent.session(sessionId).resume();
for await (const line of resumed) {
lines.push(line);
if (line.type === "text_delta") text += line.delta;
}
const result = await resumed.result;
const seen = () => JSON.stringify({
lines: lines.map(({ type, name, isError, content, text: lineText }) => ({ type, name, isError, content, text: lineText })),
requests: fake.log.filter((entry) => entry.method === "POST").map((entry) => JSON.parse(entry.body).input),
events: events.map(({ type, error, message, status, reason }) => ({ type, error, message, status, reason })),
result,
}).slice(0, 6000);
assert.equal(result.stopReason, "end_turn", `the ${phase} crash leaves a turn that resumes to its end; saw ${seen()}`);
} finally {
await agent.close();
}
const replies = fake.log.filter((entry) => entry.url === "https://chatgpt.com/backend-api/codex/responses");
assert.ok(replies.length >= 1, "the resumed turn called Codex");
for (const reply of replies) {
assert.equal(reply.headers.get("authorization"), `Bearer ${parentToken}`, "the resuming process authorizes with its own credentials");
assert.equal(reply.headers.get("chatgpt-account-id"), accountId);
assert.equal(reply.headers.get("session-id"), sessionId, "the resumed requests belong to the crashed session");
}
assert.ok(asked.includes(sessionId), "chatgpt() is asked for the crashed session's credentials");
assert.equal(fake.log.some((entry) => entry.url.includes("ai-gateway.vercel.sh")), false);
const history = JSON.stringify(JSON.parse(replies.at(-1).body).input);
if (phase === "reply") {
assert.match(history, /say hello/, "the resumed turn keeps the user's prompt");
assert.equal(text, "hello again");
} else {
assert.match(history, /use lookup for order-7/);
assert.deepEqual(lookupRuns, [sessionId], "the idempotent lookup ran again, once, in the resuming process");
assert.match(history, /order-7 shipped/, "the model received the rerun lookup's result");
assert.equal(text, "found order-7");
}
// The saved session holds no ChatGPT credentials from either process.
const files = await filesUnder(dir);
assert.ok(files.length > 0, "the session was saved");
for (const path of files) {
const bytes = await readFile(path, "latin1");
assert.equal(holdsSecret(bytes, [childToken, parentToken, "rt-crash"]), false, `${path} must not contain ChatGPT credentials`);
}
return { phase, sessionId, replies: replies.length, resumeMs: Date.now() - started };
} catch (error) {
if (process.env.CODEX_CRASH_KEEP) console.error(`kept ${dir}`);
throw error;
} finally {
if (!process.env.CODEX_CRASH_KEEP) await rm(dir, { recursive: true, force: true });
}
}
for (const phase of ["reply", "tool"]) {
const result = await crashAndResume(phase);
console.log(` ${backend} crash during the ${phase === "reply" ? "Codex reply" : "lookup tool"}: resumed with ${result.replies} Codex request(s), ${result.resumeMs} ms from start to resumed end`);
}
console.log(`${process.versions.bun ? "Bun" : "Node"} ${backend} durable Codex crash passed`);
sdk/tests/test-durable-provider-crash.mjs
+319
View File
@@ -0,0 +1,319 @@
#!/usr/bin/env node
// A durable session on a provider the host authorizes (Codex on a ChatGPT plan,
// Grok on a Grok plan, or a custom provider with an API key) survives a
// SIGKILL. A child process runs a turn on a local() durability and is killed
// during the model's reply or during a tool call; a fresh agent here resumes
// the turn on the credentials its own credentials() supplies. No credential
// reaches the saved session. No network.
//
// node test-durable-provider-crash.mjs [native|wasm]
import { strict as assert } from "node:assert";
import { spawn } from "node:child_process";
import { mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { createFxAgent, supportsJspi } from "../node.js";
import { local } from "../durable/local.mjs";
const args = process.argv.slice(2);
const childAt = args.indexOf("--child");
const backend = args[0] && args[0] !== "--child" ? args[0] : "native";
if (!new Set(["native", "wasm"]).has(backend)) throw new Error("usage: test-durable-provider-crash.mjs [native|wasm]");
if (backend === "wasm" && !supportsJspi()) {
console.error("Node JSPI is disabled. Run with --experimental-wasm-jspi");
process.exit(2);
}
const scriptDir = fileURLToPath(new URL(".", import.meta.url));
const backendOptions = {
backend,
nativeAddon: resolve(scriptDir, "../../zig-out/lib/libfx.node"),
...(backend === "wasm" ? { wasm: await readFile(resolve(scriptDir, "../../zig-out/bin/fx-core.wasm")) } : {}),
};
const accountId = "acct-crash";
function codexJwt(label) {
const part = (value) => Buffer.from(JSON.stringify(value)).toString("base64url");
return `${part({ alg: "none" })}.${part({ exp: Math.floor(Date.now() / 1000) + 864000, "https://api.openai.com/auth": { chatgpt_account_id: accountId }, label })}.sig`;
}
const event = (value) => `data: ${JSON.stringify(value)}\n\n`;
// The wire formats: how a text reply and a lookup call stream, and where a
// request carries the conversation.
const responsesProtocol = {
history: (payload) => payload.input,
sawToolOutput: (payload) => JSON.stringify(payload.input).includes("function_call_output"),
textReply: (text) =>
event({ type: "response.output_item.added", output_index: 0, item: { type: "message" } }) +
event({ type: "response.output_text.delta", output_index: 0, delta: text }) +
event({ type: "response.completed", response: { status: "completed", usage: { input_tokens: 5, output_tokens: 2 } } }),
lookupCall:
event({ type: "response.output_item.added", output_index: 0, item: { type: "function_call", call_id: "call_lookup", name: "lookup" } }) +
event({ type: "response.function_call_arguments.delta", output_index: 0, delta: JSON.stringify({ key: "order-7" }) }) +
event({ type: "response.completed", response: { status: "completed", usage: { input_tokens: 5, output_tokens: 2 } } }),
};
const chatProtocol = {
history: (payload) => payload.messages,
sawToolOutput: (payload) => payload.messages.some((message) => message.role === "tool"),
textReply: (text) =>
event({ choices: [{ index: 0, delta: { role: "assistant", content: text }, finish_reason: null }] }) +
event({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }) +
"data: [DONE]\n\n",
lookupCall:
event({ choices: [{ index: 0, delta: { role: "assistant", tool_calls: [{ index: 0, id: "call_lookup", type: "function", function: { name: "lookup", arguments: JSON.stringify({ key: "order-7" }) } }] }, finish_reason: null }] }) +
event({ choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] }) +
"data: [DONE]\n\n",
};
// What differs between the providers: the wire format, the fake endpoints
// besides the reply, the credentials, and the headers that carry session and
// account, if any.
const providers = {
codex: {
label: "Codex",
provider: "codex",
protocol: responsesProtocol,
model: "gpt-6.1-sol",
token: codexJwt,
credentials: (token) => ({ accessToken: token, refreshToken: "rt-crash" }),
replyUrl: "https://chatgpt.com/backend-api/codex/responses",
sessionHeader: "session-id",
accountHeader: "chatgpt-account-id",
serve(url) {
if (url === "https://registry.npmjs.org/@openai/codex/latest") return Response.json({ version: "0.162.1" });
if (url.startsWith("https://chatgpt.com/backend-api/codex/models")) {
return Response.json({ models: [
{ slug: "gpt-6.1-sol", visibility: "list", supported_in_api: true, priority: 1, supported_reasoning_levels: [{ effort: "low" }], additional_speed_tiers: [], input_modalities: ["text"], context_window: 272000 },
{ slug: "gpt-5.6-luna", visibility: "list", supported_in_api: true, priority: 2, supported_reasoning_levels: [{ effort: "low" }], additional_speed_tiers: [], input_modalities: ["text"], context_window: 272000 },
] });
}
return null;
},
},
grok: {
label: "Grok",
provider: "grok",
protocol: responsesProtocol,
model: "grok-4.5",
token: (label) => `at-${label}`,
credentials: (token) => ({ accessToken: token, refreshToken: "rt-crash", accountId, expiresAt: Date.now() + 6 * 3600 * 1000 }),
replyUrl: "https://cli-chat-proxy.grok.com/v1/responses",
sessionHeader: "x-grok-conv-id",
accountHeader: "x-grok-user-id",
serve(url) {
if (url === "https://x.ai/cli/stable") return new Response("1.0.50\n");
if (url === "https://cli-chat-proxy.grok.com/v1/models") {
return Response.json({ data: [{ id: "grok-4.5", model: "grok-4.5", api_backend: "responses", context_window: 500000, supports_reasoning_effort: true, reasoning_efforts: [{ value: "low" }] }] });
}
if (url === "https://api.x.ai/v1/language-models") return Response.json({ models: [] });
return null;
},
},
// The session reaches credentials() from the agent, not from a header.
custom: {
label: "custom provider",
provider: { name: "local", baseUrl: "https://models.example/v1", models: { "local-model": { supportsToolUse: true } } },
protocol: chatProtocol,
model: "local-model",
token: (label) => `sk-${label}`,
credentials: (token) => ({ apiKey: token }),
replyUrl: "https://models.example/v1/chat/completions",
serve: () => null,
},
};
// The dying process and the resuming one hold different tokens, so a request
// can show which process authorized it.
const tokensFor = (spec) => ({ child: spec.token("child-process"), parent: spec.token("resuming-process") });
/** A fake provider. `holdReply` streams the first event of a reply and then never finishes it. */
function fakeProvider(spec, { holdReply = false, onReply } = {}) {
const log = [];
const fetch = async (input, init = {}) => {
const url = String(input?.url ?? input);
const method = (init.method ?? "GET").toUpperCase();
const body = init.body === undefined ? undefined : typeof init.body === "string" ? init.body : new TextDecoder().decode(init.body);
log.push({ method, url, headers: new Headers(init.headers), body });
const served = spec.serve(url);
if (served) return served;
if (url === spec.replyUrl && method === "POST") {
const payload = JSON.parse(body);
const sawLookup = spec.protocol.sawToolOutput(payload);
const wantsLookup = JSON.stringify(spec.protocol.history(payload)).includes("use lookup");
// A request that allows no tool calls gets text, as a real model gives.
const reply = wantsLookup && !sawLookup && payload.tool_choice !== "none"
? spec.protocol.lookupCall
: spec.protocol.textReply(sawLookup ? "found order-7" : wantsLookup ? "could not confirm order-7" : "hello again");
onReply?.(payload);
if (!holdReply) return new Response(reply, { status: 200, headers: { "content-type": "text/event-stream" } });
const firstEvent = reply.slice(0, reply.indexOf("\n\n") + 2);
return new Response(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(firstEvent)); } }), {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response(`unexpected ${method} ${url}`, { status: 599 });
};
return { fetch, log };
}
function agentOptions(dir, spec, { fetch, token, asked, lookupRuns, lookupStarted, events }) {
return {
...backendOptions,
...(events ? { onEvent: (event) => events.push(event) } : {}),
durability: local({ dir }),
provider: spec.provider,
model: spec.model,
fetch,
credentials: (request) => { asked?.push(request.sessionId); return spec.credentials(token); },
tools: [{
name: "lookup",
description: "Looks up an order",
idempotent: true,
inputSchema: { type: "object", properties: { key: { type: "string" } } },
execute: async ({ key }, { sessionId }) => {
lookupRuns?.push(sessionId);
if (lookupStarted) {
// A call starts only after its intent is stored, so the crash
// comes the moment the tool runs.
lookupStarted();
await new Promise(() => {}); // the process dies here
}
return { content: `${key} shipped` };
},
}],
};
}
// The child: start a turn, report the session once the crash point is reached, wait to be killed.
if (childAt !== -1) {
const [dir, phase, provider] = args.slice(childAt + 1);
const spec = providers[provider];
let reached;
const crashPoint = new Promise((resolveReached) => { reached = resolveReached; });
const fake = fakeProvider(spec, { holdReply: phase === "reply", onReply: () => { if (phase === "reply") reached(); } });
const agent = createFxAgent(agentOptions(dir, spec, {
fetch: fake.fetch,
token: tokensFor(spec).child,
...(phase === "tool" ? { lookupStarted: () => reached() } : {}),
}));
const turn = agent.session().prompt(phase === "tool" ? "use lookup for order-7" : "say hello");
const { sessionId } = await turn.accepted;
await crashPoint;
process.stdout.write(`${JSON.stringify({ sessionId })}\n`);
await new Promise(() => {});
}
// True when the text, or base64 found in it decoded up to two levels deep,
// contains any of the secrets. The local World stores entries as base64.
function holdsSecret(text, secrets, depth = 2) {
if (secrets.some((secret) => text.includes(secret))) return true;
if (depth === 0) return false;
for (const [encoded] of text.matchAll(/[A-Za-z0-9+/_-]{24,}={0,2}/g)) {
const decoded = Buffer.from(encoded, "base64").toString("latin1");
if (holdsSecret(decoded, secrets, depth - 1)) return true;
}
return false;
}
async function filesUnder(dir) {
const out = [];
for (const entry of await readdir(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
if (entry.isDirectory()) out.push(...await filesUnder(path));
else if ((await stat(path)).size < 16 * 1024 * 1024) out.push(path);
}
return out;
}
async function crashAndResume(provider, phase) {
const spec = providers[provider];
const tokens = tokensFor(spec);
const dir = await mkdtemp(join(tmpdir(), `libfx-${provider}-crash-`));
try {
const child = spawn(process.execPath, [...process.execArgv, fileURLToPath(import.meta.url), backend, "--child", dir, phase, provider], {
stdio: ["ignore", "pipe", "inherit"],
env: { ...process.env },
});
let output = "";
child.stdout.on("data", (chunk) => { output += chunk; });
const started = Date.now();
while (!output.includes("\n")) {
if (child.exitCode !== null) throw new Error(`the ${phase} child exited early with ${child.exitCode}`);
if (Date.now() - started > 30_000) throw new Error(`the ${phase} child never reached its crash point`);
await new Promise((wait) => setTimeout(wait, 20));
}
const { sessionId } = JSON.parse(output.trim().split("\n")[0]);
child.kill("SIGKILL");
const [, signal] = await new Promise((exited) => child.once("exit", (code, sig) => exited([code, sig])));
assert.equal(signal, "SIGKILL", "the child died by SIGKILL, mid-turn");
const fake = fakeProvider(spec);
const asked = [];
const lookupRuns = [];
const events = [];
const agent = createFxAgent(agentOptions(dir, spec, { fetch: fake.fetch, token: tokens.parent, asked, lookupRuns, events }));
let text = "";
const lines = [];
try {
const resumed = agent.session(sessionId).resume();
for await (const line of resumed) {
lines.push(line);
if (line.type === "text_delta") text += line.delta;
}
const result = await resumed.result;
const seen = () => JSON.stringify({
lines: lines.map(({ type, name, isError, content, text: lineText }) => ({ type, name, isError, content, text: lineText })),
requests: fake.log.filter((entry) => entry.method === "POST").map((entry) => spec.protocol.history(JSON.parse(entry.body))),
events: events.map(({ type, error, message, status, reason }) => ({ type, error, message, status, reason })),
result,
}).slice(0, 6000);
assert.equal(result.stopReason, "end_turn", `the ${spec.label} ${phase} crash leaves a turn that resumes to its end; saw ${seen()}`);
} finally {
await agent.close();
}
const replies = fake.log.filter((entry) => entry.url === spec.replyUrl);
assert.ok(replies.length >= 1, `the resumed turn called ${spec.label}`);
for (const reply of replies) {
assert.equal(reply.headers.get("authorization"), `Bearer ${tokens.parent}`, "the resuming process authorizes with its own credentials");
if (spec.accountHeader) assert.equal(reply.headers.get(spec.accountHeader), accountId);
if (spec.sessionHeader) assert.equal(reply.headers.get(spec.sessionHeader), sessionId, "the resumed requests belong to the crashed session");
}
assert.ok(asked.includes(sessionId), "credentials() is asked for the crashed session's credentials");
assert.equal(fake.log.some((entry) => entry.url.includes("ai-gateway.vercel.sh")), false);
const history = JSON.stringify(spec.protocol.history(JSON.parse(replies.at(-1).body)));
if (phase === "reply") {
assert.match(history, /say hello/, "the resumed turn keeps the user's prompt");
assert.equal(text, "hello again");
} else {
assert.match(history, /use lookup for order-7/);
assert.deepEqual(lookupRuns, [sessionId], "the idempotent lookup ran again, once, in the resuming process");
assert.match(history, /order-7 shipped/, "the model received the rerun lookup's result");
assert.equal(text, "found order-7");
}
// The saved session holds no credentials from either process.
const files = await filesUnder(dir);
assert.ok(files.length > 0, "the session was saved");
for (const path of files) {
const bytes = await readFile(path, "latin1");
assert.equal(holdsSecret(bytes, [tokens.child, tokens.parent, "rt-crash"]), false, `${path} must not contain ${spec.label} credentials`);
}
return { sessionId, replies: replies.length, resumeMs: Date.now() - started };
} catch (error) {
if (process.env.PROVIDER_CRASH_KEEP) console.error(`kept ${dir}`);
throw error;
} finally {
if (!process.env.PROVIDER_CRASH_KEEP) await rm(dir, { recursive: true, force: true });
}
}
for (const provider of Object.keys(providers)) {
for (const phase of ["reply", "tool"]) {
const result = await crashAndResume(provider, phase);
const label = providers[provider].label;
console.log(` ${backend} crash during the ${phase === "reply" ? `${label} reply` : `${label} lookup tool`}: resumed with ${result.replies} request(s), ${result.resumeMs} ms from start to resumed end`);
}
}
console.log(`${process.versions.bun ? "Bun" : "Node"} ${backend} durable provider crash passed`);
sdk/tests/test-grok-sign-in.mjs
+164
View File
@@ -0,0 +1,164 @@
#!/usr/bin/env node
// `libfx/grok` against a fake xAI: device code sign-in, the paste-the-address
// sign-in and refresh, with the error each mistake produces. No network.
import { strict as assert } from "node:assert";
import { createHash } from "node:crypto";
import {
completeDeviceSignIn,
finishSignIn,
pollDeviceSignIn,
refreshCredentials,
startDeviceSignIn,
startSignIn,
} from "../grok-sign-in.js";
import { normalizeGrokCredentials } from "../grok.js";
const clientId = "b1a00492-073a-47ea-816f-4c329264a828";
const scope = "openid profile email offline_access grok-cli:access api:access";
const accountId = "xai-account-fixture";
/**
* A fake xAI auth server. `tokenStatus` answers code exchanges, `pollAnswers`
* answers device polls in order, and `account` is who userinfo says the token is.
*/
function xai({ tokenStatus = 200, pollAnswers = [], account = accountId, userinfoStatus = 200 } = {}) {
const log = [];
const fetch = async (url, init = {}) => {
const body = typeof init.body === "string" ? init.body : init.body === undefined ? undefined : String(init.body);
log.push({ url: String(url), method: init.method ?? "GET", headers: new Headers(init.headers), body });
if (url === "https://auth.x.ai/oauth2/device/code") {
return Response.json({ device_code: "dc-1", user_code: "ABCD-EFGH", verification_uri: "https://accounts.x.ai/oauth2/device", verification_uri_complete: "https://accounts.x.ai/oauth2/device?user_code=ABCD-EFGH", expires_in: 1800, interval: 0 });
}
if (url === "https://auth.x.ai/oauth2/token") {
const grant = new URLSearchParams(body).get("grant_type");
if (grant === "urn:ietf:params:oauth:grant-type:device_code") {
const answer = pollAnswers.shift() ?? "authorization_pending";
if (answer !== 200) return Response.json({ error: answer }, { status: 400 });
} else if (tokenStatus !== 200) {
return Response.json({ error: "invalid_grant" }, { status: tokenStatus });
}
return Response.json({ access_token: `at-${grant}`, refresh_token: grant === "refresh_token" ? "rt-rotated" : "rt-1", expires_in: 21600, token_type: "Bearer" });
}
if (url === "https://auth.x.ai/oauth2/userinfo") {
if (userinfoStatus !== 200) return new Response("no", { status: userinfoStatus });
return Response.json({ sub: account, email: "user@example.com" });
}
return new Response("unexpected", { status: 599 });
};
return { fetch, log };
}
const rejectsWith = (promise, code) => assert.rejects(promise, (error) => {
assert.equal(error.code, code);
return true;
});
const formOf = (entry) => Object.fromEntries(new URLSearchParams(entry.body));
// startSignIn: the CLI's own authorize request, with an S256 challenge of the secret verifier.
const signIn = await startSignIn();
{
const url = new URL(signIn.url);
assert.equal(`${url.origin}${url.pathname}`, "https://auth.x.ai/oauth2/authorize");
const params = Object.fromEntries(url.searchParams);
assert.equal(params.client_id, clientId);
assert.equal(params.response_type, "code");
assert.equal(params.scope, scope);
assert.equal(params.code_challenge_method, "S256");
assert.equal(params.code_challenge, createHash("sha256").update(signIn.verifier).digest("base64url"));
assert.equal(params.state, signIn.state);
assert.equal(params.referrer, "fx");
assert.equal(params.redirect_uri, signIn.redirectUri);
const redirect = new URL(signIn.redirectUri);
assert.equal(`${redirect.protocol}//${redirect.hostname}${redirect.pathname}`, "http://127.0.0.1/callback");
assert.ok(Number(redirect.port) >= 1024 && Number(redirect.port) <= 65535, "a loopback port xAI accepts");
assert.notEqual((await startSignIn()).verifier, signIn.verifier, "every sign-in has its own secret");
}
// finishSignIn: every paste style yields the account's credentials.
for (const pasted of [
`${signIn.redirectUri}?code=auth-code&state=${signIn.state}`,
`code=auth-code&state=${signIn.state}`,
`\`${signIn.redirectUri}?code=auth-code&state=${signIn.state}\` `,
]) {
const server = xai();
const credentials = await finishSignIn(signIn, pasted, { fetch: server.fetch });
assert.equal(credentials.accessToken, "at-authorization_code");
assert.equal(credentials.refreshToken, "rt-1");
assert.equal(credentials.accountId, accountId, "the account comes from userinfo");
assert.ok(credentials.expiresAt > Date.now() + 5 * 3600 * 1000);
const exchange = server.log.find((entry) => entry.url === "https://auth.x.ai/oauth2/token");
assert.equal(exchange.headers.get("content-type"), "application/x-www-form-urlencoded");
assert.deepEqual(formOf(exchange), { grant_type: "authorization_code", client_id: clientId, code: "auth-code", code_verifier: signIn.verifier, redirect_uri: signIn.redirectUri });
const userinfo = server.log.find((entry) => entry.url === "https://auth.x.ai/oauth2/userinfo");
assert.equal(userinfo.headers.get("authorization"), "Bearer at-authorization_code");
}
// finishSignIn: each mistake has its own error.
await rejectsWith(finishSignIn(signIn, `code=auth-code&state=other`, { fetch: xai().fetch }), "LIBFX_GROK_SIGN_IN_STATE_MISMATCH");
await rejectsWith(finishSignIn(signIn, "localhost", { fetch: xai().fetch }), "LIBFX_GROK_SIGN_IN_INVALID_ADDRESS");
await rejectsWith(finishSignIn(signIn, `${signIn.redirectUri}?error=access_denied&state=${signIn.state}`, { fetch: xai().fetch }), "LIBFX_GROK_SIGN_IN_DENIED");
await rejectsWith(finishSignIn(signIn, `code=used&state=${signIn.state}`, { fetch: xai({ tokenStatus: 400 }).fetch }), "LIBFX_GROK_SIGN_IN_EXPIRED");
await rejectsWith(finishSignIn(signIn, `code=c&state=${signIn.state}`, { fetch: xai({ tokenStatus: 503 }).fetch }), "LIBFX_GROK_SIGN_IN_FAILED");
await rejectsWith(finishSignIn(signIn, `code=c&state=${signIn.state}`, { fetch: xai({ userinfoStatus: 401 }).fetch }), "LIBFX_GROK_ACCOUNT_UNAVAILABLE");
await assert.rejects(finishSignIn({ url: signIn.url }, `code=c&state=${signIn.state}`), TypeError);
// Device code: start, pending, slow down, then signed in.
{
const server = xai({ pollAnswers: ["authorization_pending", "slow_down", 200] });
const device = await startDeviceSignIn({ fetch: server.fetch });
assert.equal(device.userCode, "ABCD-EFGH");
assert.equal(device.verificationUrl, "https://accounts.x.ai/oauth2/device?user_code=ABCD-EFGH");
assert.equal(device.intervalMs, 1000, "the interval has a floor of one second");
assert.ok(device.expiresAt > Date.now() + 29 * 60 * 1000);
assert.deepEqual(formOf(server.log[0]), { client_id: clientId, scope });
assert.equal(await pollDeviceSignIn(device, { fetch: server.fetch }), null, "pending");
assert.equal(await pollDeviceSignIn(device, { fetch: server.fetch }), null, "slow down");
const credentials = await pollDeviceSignIn(device, { fetch: server.fetch });
assert.equal(credentials.accessToken, "at-urn:ietf:params:oauth:grant-type:device_code");
assert.equal(credentials.accountId, accountId);
const polls = server.log.filter((entry) => entry.url === "https://auth.x.ai/oauth2/token");
assert.deepEqual(formOf(polls[0]), { grant_type: "urn:ietf:params:oauth:grant-type:device_code", device_code: "dc-1", client_id: clientId });
}
{
const server = xai({ pollAnswers: ["authorization_pending", 200] });
const device = await startDeviceSignIn({ fetch: server.fetch });
const started = Date.now();
const credentials = await completeDeviceSignIn(device, { fetch: server.fetch });
assert.equal(credentials.accountId, accountId);
assert.ok(Date.now() - started >= 900, "waits the interval between polls");
}
{
const server = xai({ pollAnswers: ["slow_down", "authorization_pending", "authorization_pending"] });
const device = await startDeviceSignIn({ fetch: server.fetch });
const controller = new AbortController();
setTimeout(() => controller.abort(new Error("stopped")), 300);
await assert.rejects(completeDeviceSignIn(device, { fetch: server.fetch, signal: controller.signal }), /stopped/);
}
await rejectsWith(pollDeviceSignIn(await startDeviceSignIn({ fetch: xai().fetch }), { fetch: xai({ pollAnswers: ["expired_token"] }).fetch }), "LIBFX_GROK_DEVICE_CODE_EXPIRED");
await rejectsWith(pollDeviceSignIn(await startDeviceSignIn({ fetch: xai().fetch }), { fetch: xai({ pollAnswers: ["access_denied"] }).fetch }), "LIBFX_GROK_SIGN_IN_DENIED");
await rejectsWith(pollDeviceSignIn({ ...(await startDeviceSignIn({ fetch: xai().fetch })), expiresAt: Date.now() - 1 }, { fetch: xai().fetch }), "LIBFX_GROK_DEVICE_CODE_EXPIRED");
await rejectsWith(startDeviceSignIn({ fetch: async () => new Response("no", { status: 403 }) }), "LIBFX_GROK_DEVICE_CODE_UNAVAILABLE");
// Refresh: form request, rotation kept, and the account must not change.
{
const server = xai();
const next = await refreshCredentials({ accessToken: "at-old", refreshToken: "rt-1", accountId }, { fetch: server.fetch });
assert.equal(next.accessToken, "at-refresh_token");
assert.equal(next.refreshToken, "rt-rotated");
assert.equal(next.accountId, accountId);
assert.deepEqual(formOf(server.log[0]), { grant_type: "refresh_token", client_id: clientId, refresh_token: "rt-1" });
}
await rejectsWith(refreshCredentials({ accessToken: "at-old", refreshToken: "rt-1", accountId }, { fetch: xai({ account: "someone-else" }).fetch }), "LIBFX_GROK_ACCOUNT_CHANGED");
await rejectsWith(refreshCredentials({ accessToken: "at-old", refreshToken: "rt-1", accountId }, { fetch: xai({ tokenStatus: 400 }).fetch }), "LIBFX_GROK_REFRESH_FAILED");
// Credentials: an opaque token needs an accountId; a JWT's `sub` supplies one.
assert.throws(() => normalizeGrokCredentials({ accessToken: "opaque" }), /Grok credentials need an accountId/);
{
const part = (value) => Buffer.from(JSON.stringify(value)).toString("base64url");
const token = `${part({ alg: "none" })}.${part({ sub: "from-claim", exp: 2_000_000_000 })}.sig`;
const credentials = normalizeGrokCredentials({ accessToken: token });
assert.equal(credentials.accountId, "from-claim");
assert.equal(credentials.expiresAt, 2_000_000_000_000);
}
console.log("Grok sign-in passed");
sdk/tests/test-node-napi.mjs
+3
View File
@@ -18,6 +18,9 @@ const scripts = [
"test-agent-transport-retry.mjs",
"test-agent-codex.mjs",
"test-chatgpt-sign-in.mjs",
"test-agent-grok.mjs",
"test-agent-custom.mjs",
"test-grok-sign-in.mjs",
"test-native-core-stream.mjs",
"test-native-core-http-cleanup.mjs",
"test-native-core-fetch-failure.mjs",
sdk/tests/test-node-wasm.mjs
+2
View File
@@ -34,6 +34,8 @@ const commands = [
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-agent-request-context.mjs", import.meta.url)), "wasm"]],
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-agent-transport-retry.mjs", import.meta.url)), "wasm"]],
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-agent-codex.mjs", import.meta.url)), "wasm"]],
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-agent-grok.mjs", import.meta.url)), "wasm"]],
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-agent-custom.mjs", import.meta.url)), "wasm"]],
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-core.mjs", import.meta.url))]],
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-core-cancel.mjs", import.meta.url))]],
[process.execPath, ["--experimental-wasm-jspi", fileURLToPath(new URL("test-core-home-unavailable.mjs", import.meta.url))]],
sdk/tests/test-types.mjs
+1
View File
@@ -46,6 +46,7 @@ const entries = [
{ specifier: "libfx/durable-vercel", resolution: "node16", mode: "import", types: "types/durable-vercel.d.ts", runtime: "durable/vercel.mjs" },
{ specifier: "libfx/durable-world", resolution: "node16", mode: "import", types: "types/durable-world.d.ts", runtime: "durable/world.mjs" },
{ specifier: "libfx/chatgpt", resolution: "node16", mode: "import", types: "types/chatgpt.d.ts", runtime: "chatgpt-sign-in.js" },
{ specifier: "libfx/grok", resolution: "node16", mode: "import", types: "types/grok.d.ts", runtime: "grok-sign-in.js" },
{ specifier: "libfx/mcp", resolution: "node16", mode: "import", types: "types/mcp.d.ts", runtime: "mcp.js" },
{ specifier: "libfx/skills", resolution: "node16", mode: "import", types: "types/skills.d.ts", runtime: "skills.js" },
{ specifier: "libfx/skills/node", resolution: "node16", mode: "import", types: "types/skills-node.d.ts", runtime: "skills-node.js" },
sdk/tests/types/adapters.mts
+23 -4
View File
@@ -1,5 +1,5 @@
// The MCP and skills adapters, as the README combines them with an agent.
import { createFxAgent, type FxChatgptCredentials } from "libfx";
import { createFxAgent, type FxProviderCredentials } from "libfx";
import {
completeDeviceSignIn,
finishSignIn,
@@ -11,6 +11,8 @@ import {
type ChatgptSignIn,
type ChatgptSignInErrorCode,
} from "libfx/chatgpt";
import * as grok from "libfx/grok";
import type { GrokDeviceSignIn, GrokSignIn, GrokSignInErrorCode } from "libfx/grok";
import { createMcpAdapter, type McpAdapter, type McpClient } from "libfx/mcp";
import { createSkillsAdapter, type SkillRecord } from "libfx/skills";
import { createSkillsAdapter as createSkillsAdapterFromNode, loadSkillFile, type LoadedSkill } from "libfx/skills/node";
@@ -82,16 +84,33 @@ export async function skills(): Promise<void> {
// Sign in with ChatGPT on a website: start in one request, finish in another.
declare const sessionStore: { put(value: ChatgptSignIn): void; take(): ChatgptSignIn };
export async function chatgptSignIn(pastedAddress: string): Promise<FxChatgptCredentials> {
export async function chatgptSignIn(pastedAddress: string): Promise<FxProviderCredentials> {
const signIn = await startSignIn();
sessionStore.put(signIn);
const credentials = await finishSignIn(sessionStore.take(), pastedAddress);
// @ts-expect-error finishSignIn() needs the sign-in startSignIn() returned
await finishSignIn({ url: signIn.url }, pastedAddress);
const device: ChatgptDeviceSignIn = await startDeviceSignIn();
const polled: FxChatgptCredentials | null = await pollDeviceSignIn(device);
const waited: FxChatgptCredentials = await completeDeviceSignIn(device, { signal: AbortSignal.timeout(900_000) });
const polled: FxProviderCredentials | null = await pollDeviceSignIn(device);
const waited: FxProviderCredentials = await completeDeviceSignIn(device, { signal: AbortSignal.timeout(900_000) });
const code: ChatgptSignInErrorCode = "LIBFX_CHATGPT_SIGN_IN_EXPIRED";
void polled; void waited; void code;
return refreshCredentials(credentials);
}
// Sign in with a Grok plan: device code first, the pasted address as a fallback.
declare const grokStore: { put(value: GrokSignIn | GrokDeviceSignIn): void };
export async function grokSignIn(pastedAddress: string): Promise<FxProviderCredentials> {
const device: GrokDeviceSignIn = await grok.startDeviceSignIn();
grokStore.put(device);
const polled: FxProviderCredentials | null = await grok.pollDeviceSignIn(device);
const viaDevice: FxProviderCredentials = await grok.completeDeviceSignIn(device, { signal: AbortSignal.timeout(1_800_000) });
const signIn: GrokSignIn = await grok.startSignIn();
const redirect: string = signIn.redirectUri;
const viaAddress = await grok.finishSignIn(signIn, pastedAddress);
// @ts-expect-error finishSignIn() needs the sign-in startSignIn() returned
await grok.finishSignIn({ url: signIn.url, state: signIn.state, verifier: signIn.verifier }, pastedAddress);
const code: GrokSignInErrorCode = "LIBFX_GROK_ACCOUNT_CHANGED";
void polled; void viaDevice; void redirect; void code;
return grok.refreshCredentials(viaAddress);
}
sdk/tests/types/agent.mts
+1 -1
View File
@@ -362,5 +362,5 @@ export const wake: (request: Request) => Promise<Response> = agent.wakeHandler()
export const codexAgent: FxAgent = createFxAgent({
provider: "codex",
model: "gpt-6.1-sol",
chatgpt: async ({ sessionId }) => ({ accessToken: sessionId ?? "token", refreshToken: "refresh" }),
credentials: async ({ sessionId }) => ({ accessToken: sessionId ?? "token", refreshToken: "refresh" }),
});
sdk/tests/types/engine.mts
+56 -10
View File
@@ -10,7 +10,7 @@ import {
listModels,
fxSdkApiVersion,
supportsJspi,
type FxChatgptCredentials,
type FxProviderCredentials,
type FxEngine,
type FxEngineEvent,
type FxEngineTurn,
@@ -228,23 +228,69 @@ export async function events(): Promise<void> {
// A Codex engine on the host's ChatGPT credentials.
declare const tokenStore: {
load(sessionId?: string): Promise<FxChatgptCredentials>;
save(credentials: FxChatgptCredentials): Promise<void>;
load(provider: "codex", sessionId?: string): Promise<FxProviderCredentials>;
save(provider: "codex", credentials: FxProviderCredentials): Promise<void>;
};
declare const accountLocks: { run<T>(key: string, fn: () => Promise<T>): Promise<T> };
export async function codexEngine(): Promise<FxEngine> {
const engine = await createFxEngine({
provider: "codex",
model: "gpt-6.1-sol",
chatgpt: ({ sessionId }) => tokenStore.load(sessionId),
onChatgptCredentials: (credentials) => tokenStore.save(credentials),
credentials: ({ provider, sessionId }) => tokenStore.load(provider, sessionId),
onCredentials: (credentials, { provider }) => tokenStore.save(provider, credentials),
refreshLock: ({ provider, sessionId }, refresh) => accountLocks.run(`${provider}:${sessionId ?? "catalog"}`, refresh),
});
// @ts-expect-error a Codex engine needs chatgpt()
// @ts-expect-error a Codex engine needs credentials()
await createFxEngine({ provider: "codex", model });
// @ts-expect-error chatgpt() needs provider "codex"
await createFxEngine({ apiKey, model, chatgpt: () => tokenStore.load() });
// @ts-expect-error credentials() needs provider "codex"
await createFxEngine({ apiKey, model, credentials: () => tokenStore.load("codex") });
// @ts-expect-error onCredentials needs provider "codex"
await createFxEngine({ apiKey, model, onCredentials: () => {} });
// @ts-expect-error refreshLock needs provider "codex"
await createFxEngine({ apiKey, model, refreshLock: async (_info, refresh) => refresh() });
// @ts-expect-error a Gateway engine needs apiKey
await createFxEngine({ provider: "gateway", model });
// @ts-expect-error libfx runs Gateway or Codex
await createFxEngine({ provider: "grok", apiKey, model });
// @ts-expect-error a provider named by string is Gateway, Codex or Grok
await createFxEngine({ provider: "custom", apiKey, model });
return engine;
}
// A custom provider: a local server needs no key; a hosted one gets the host's.
declare const providerKeys: { load(provider: string, sessionId?: string): Promise<string> };
export async function customEngine(): Promise<FxEngine> {
const local = await createFxEngine({
provider: { name: "local", baseUrl: "http://localhost:11434/v1", models: { "qwen3:8b": { contextWindow: 32768, supportsToolUse: true } } },
model: "qwen3:8b",
});
await local.close();
const router = { name: "router", baseUrl: "https://openrouter.ai/api/v1" };
const engine = await createFxEngine({
provider: router,
model: "qwen/qwen3-coder",
credentials: async ({ provider, sessionId }) => ({ apiKey: await providerKeys.load(provider, sessionId) }),
});
// @ts-expect-error a custom provider's credentials are an API key
await createFxEngine({ provider: router, model, credentials: () => tokenStore.load("codex") });
// @ts-expect-error a custom provider's key is never refreshed
await createFxEngine({ provider: router, model, onCredentials: () => {} });
// @ts-expect-error a custom provider's key is never refreshed, so it takes no lock
await createFxEngine({ provider: router, model, refreshLock: async (_info, refresh) => refresh() });
// @ts-expect-error a custom provider needs a base URL
await createFxEngine({ provider: { name: "router" }, model });
// @ts-expect-error model details use libfx's option names
await createFxEngine({ provider: { ...router, models: { m: { context_window: 1 } } }, model });
return engine;
}
// A Grok engine on the host's Grok credentials; one store serves both plans.
declare const planTokens: { load(provider: "codex" | "grok", sessionId?: string): Promise<FxProviderCredentials> };
export async function grokEngine(): Promise<FxEngine> {
const engine = await createFxEngine({
provider: "grok",
model: "grok-4.5",
credentials: ({ provider, sessionId }) => planTokens.load(provider, sessionId),
});
// @ts-expect-error a Grok request names provider "grok"
await createFxEngine({ provider: "grok", model, credentials: ({ provider }) => tokenStore.load(provider) });
return engine;
}
sdk/types/browser.d.ts
Vendored
+6 -1
View File
@@ -27,11 +27,14 @@ export type {
FxAgent,
FxAgentEvent,
FxAgentOptions,
FxApiKeyCredentials,
FxBrowserBackendOptions,
FxBytes,
FxChatgptCredentials,
FxCheckpointMeta,
FxCodexProvider,
FxCustomProvider,
FxCustomProviderDefinition,
FxCustomProviderModel,
FxEffort,
FxEngine,
FxEngineEvent,
@@ -44,6 +47,7 @@ export type {
FxFetch,
FxFollowUp,
FxGatewayProvider,
FxGrokProvider,
FxHostTool,
FxIdleResult,
FxImageBlock,
@@ -58,6 +62,7 @@ export type {
FxPromptBlock,
FxPromptInput,
FxPromptOptions,
FxProviderCredentials,
FxProviderTool,
FxResizedImage,
FxResourceBlock,
sdk/types/chatgpt.d.ts
Vendored
+5 -5
View File
@@ -1,5 +1,5 @@
// `libfx/chatgpt` (chatgpt-sign-in.js): sign in with ChatGPT for a Codex agent.
import type { FxChatgptCredentials, FxFetch } from "./libfx.cjs";
import type { FxProviderCredentials, FxFetch } from "./libfx.cjs";
/** Options every network step accepts. */
export interface ChatgptRequestOptions {
@@ -43,16 +43,16 @@ export type ChatgptSignInErrorCode =
export declare function startSignIn(): Promise<ChatgptSignIn>;
/** Finishes a sign-in from the pasted address, its query, or either wrapped in quotes or backticks. */
export declare function finishSignIn(signIn: ChatgptSignIn, address: string, options?: ChatgptRequestOptions): Promise<FxChatgptCredentials>;
export declare function finishSignIn(signIn: ChatgptSignIn, address: string, options?: ChatgptRequestOptions): Promise<FxProviderCredentials>;
/** Starts a device code sign-in. The account or workspace must allow device code sign-in. */
export declare function startDeviceSignIn(options?: ChatgptRequestOptions): Promise<ChatgptDeviceSignIn>;
/** Checks once: credentials after the user entered the code, otherwise null. */
export declare function pollDeviceSignIn(device: ChatgptDeviceSignIn, options?: ChatgptRequestOptions): Promise<FxChatgptCredentials | null>;
export declare function pollDeviceSignIn(device: ChatgptDeviceSignIn, options?: ChatgptRequestOptions): Promise<FxProviderCredentials | null>;
/** Waits until the user enters the code, polling at OpenAI's interval. */
export declare function completeDeviceSignIn(device: ChatgptDeviceSignIn, options?: ChatgptRequestOptions): Promise<FxChatgptCredentials>;
export declare function completeDeviceSignIn(device: ChatgptDeviceSignIn, options?: ChatgptRequestOptions): Promise<FxProviderCredentials>;
/** Refreshes credentials. Codex agents refresh on their own; this is for hosts. */
export declare function refreshCredentials(credentials: FxChatgptCredentials, options?: ChatgptRequestOptions): Promise<FxChatgptCredentials>;
export declare function refreshCredentials(credentials: FxProviderCredentials, options?: ChatgptRequestOptions): Promise<FxProviderCredentials>;
sdk/types/grok.d.ts
Vendored
+61
View File
@@ -0,0 +1,61 @@
// `libfx/grok` (grok-sign-in.js): sign in with a Grok plan for a Grok agent.
import type { FxProviderCredentials, FxFetch } from "./libfx.cjs";
/** Options every network step accepts. */
export interface GrokRequestOptions {
/** Replaces the global `fetch` for xAI's sign-in endpoints. */
fetch?: FxFetch;
signal?: AbortSignal;
}
/** A sign-in in progress. Keep it on the server: `verifier` is secret. */
export interface GrokSignIn {
/** Where the user signs in. Afterwards xAI opens an address at `redirectUri` that does not load; the user pastes it back. */
readonly url: string;
readonly state: string;
readonly verifier: string;
readonly redirectUri: string;
}
/** A device code sign-in in progress. Plain data, so a later request can poll it. */
export interface GrokDeviceSignIn {
/** Where the user enters `userCode`. */
readonly verificationUrl: string;
readonly userCode: string;
readonly deviceCode: string;
/** How often to poll, in milliseconds. */
readonly intervalMs: number;
/** Milliseconds since the epoch. */
readonly expiresAt: number;
}
/** The `code` of an error from this module. */
export type GrokSignInErrorCode =
| "LIBFX_GROK_SIGN_IN_INVALID_ADDRESS"
| "LIBFX_GROK_SIGN_IN_STATE_MISMATCH"
| "LIBFX_GROK_SIGN_IN_DENIED"
| "LIBFX_GROK_SIGN_IN_EXPIRED"
| "LIBFX_GROK_SIGN_IN_FAILED"
| "LIBFX_GROK_DEVICE_CODE_UNAVAILABLE"
| "LIBFX_GROK_DEVICE_CODE_EXPIRED"
| "LIBFX_GROK_ACCOUNT_UNAVAILABLE"
| "LIBFX_GROK_ACCOUNT_CHANGED"
| "LIBFX_GROK_REFRESH_FAILED";
/** Starts a sign-in where the user pastes back the address xAI opens. */
export declare function startSignIn(): Promise<GrokSignIn>;
/** Finishes a sign-in from the pasted address, its query, or either wrapped in quotes or backticks. */
export declare function finishSignIn(signIn: GrokSignIn, address: string, options?: GrokRequestOptions): Promise<FxProviderCredentials>;
/** Starts a device code sign-in. */
export declare function startDeviceSignIn(options?: GrokRequestOptions): Promise<GrokDeviceSignIn>;
/** Checks once: credentials after the user entered the code, otherwise null. */
export declare function pollDeviceSignIn(device: GrokDeviceSignIn, options?: GrokRequestOptions): Promise<FxProviderCredentials | null>;
/** Waits until the user enters the code, polling at xAI's interval. */
export declare function completeDeviceSignIn(device: GrokDeviceSignIn, options?: GrokRequestOptions): Promise<FxProviderCredentials>;
/** Refreshes credentials. Grok agents refresh on their own; this is for hosts. */
export declare function refreshCredentials(credentials: FxProviderCredentials, options?: GrokRequestOptions): Promise<FxProviderCredentials>;
sdk/types/libfx.d.cts
+59 -13
View File
@@ -458,8 +458,8 @@ export type FxEngineStorage =
checkpoint?: FxBytes;
};
/** ChatGPT credentials for a Codex agent. `accountId` and `expiresAt` default to the access token's own claims. */
export interface FxChatgptCredentials {
/** A subscription provider's credentials: a ChatGPT plan's for Codex, or a Grok plan's. `accountId` and `expiresAt` default to the access token's own claims when it carries them. */
export interface FxProviderCredentials {
accessToken: string;
/** Lets libfx refresh the access token near expiry and once after a rejection. */
refreshToken?: string;
@@ -471,17 +471,63 @@ export interface FxChatgptCredentials {
/** Runs on AI Gateway, the default. */
export interface FxGatewayProvider {
provider?: "gateway" | undefined;
chatgpt?: undefined;
onChatgptCredentials?: undefined;
credentials?: undefined;
onCredentials?: undefined;
refreshLock?: undefined;
}
/** Runs on a user's plan instead of AI Gateway. */
interface FxSubscriptionProvider<Provider extends "codex" | "grok"> {
provider: Provider;
/** Credentials for the provider's requests, asked for in whichever process runs the turn. `sessionId` is set for model requests and absent for the model list. */
credentials: (request: { provider: Provider; sessionId?: string }) => FxProviderCredentials | Promise<FxProviderCredentials>;
/** Receives credentials libfx refreshed, so the host can save them. */
onCredentials?: (credentials: FxProviderCredentials, info: { provider: Provider; sessionId?: string }) => void | Promise<void>;
/**
* Runs `refresh` while the host holds a lock on the account, so a host with
* several servers refreshes it on one at a time. libfx asks `credentials()`
* again inside it and uses credentials another server already saved.
*/
refreshLock?: (info: { provider: Provider; sessionId?: string }, refresh: () => Promise<void>) => Promise<unknown>;
}
/** Runs on a ChatGPT plan through Codex instead of AI Gateway. */
export interface FxCodexProvider {
provider: "codex";
/** Credentials for Codex requests, asked for in whichever process runs the turn. `sessionId` is set for model requests and absent for the model list. */
chatgpt: (request: { sessionId?: string }) => FxChatgptCredentials | Promise<FxChatgptCredentials>;
/** Receives credentials libfx refreshed, so the host can save them. */
onChatgptCredentials?: (credentials: FxChatgptCredentials, info: { sessionId?: string }) => void | Promise<void>;
export type FxCodexProvider = FxSubscriptionProvider<"codex">;
/** Runs on a Grok plan instead of AI Gateway. */
export type FxGrokProvider = FxSubscriptionProvider<"grok">;
/** What libfx knows about a model a custom provider serves. Unset fields are unknown. */
export interface FxCustomProviderModel {
contextWindow?: number;
/** Less than `contextWindow` when both are set. */
maxOutputTokens?: number;
supportsToolUse?: boolean;
supportsVision?: boolean;
}
/** A server that speaks OpenAI Chat Completions, such as a local model server or a router. */
export interface FxCustomProviderDefinition {
/** Names the provider to `credentials()`: a letter, then letters, digits, `_` or `-`, at most 64 characters. Not `gateway`, `codex` or `grok`. */
name: string;
/** The API prefix: requests go to `{baseUrl}/chat/completions`. HTTPS, or HTTP on localhost. */
baseUrl: string;
/** The models it serves, by id. */
models?: Record<string, FxCustomProviderModel>;
}
/** A custom provider's API key. */
export interface FxApiKeyCredentials {
apiKey: string;
}
/** Runs on a custom provider instead of AI Gateway. */
export interface FxCustomProvider {
provider: FxCustomProviderDefinition;
/** The API key for the provider's requests, asked for in whichever process runs the turn. Omit it for a server that needs no key. */
credentials?: ((request: { provider: string; sessionId?: string }) => FxApiKeyCredentials | Promise<FxApiKeyCredentials>) | undefined;
onCredentials?: undefined;
refreshLock?: undefined;
}
/** Options for `createFxEngine()`. */
@@ -490,8 +536,8 @@ export type FxEngineOptions = FxSharedOptions<FxEngineEvent> & FxModelChoice & F
/** The AI Gateway credential. Required. */
apiKey: string;
})
| (FxCodexProvider & {
/** The AI Gateway credential, unused by Codex turns. */
| ((FxCodexProvider | FxGrokProvider | FxCustomProvider) & {
/** The AI Gateway credential, unused by this provider's turns. */
apiKey?: string | undefined;
})
) & {
@@ -524,7 +570,7 @@ export declare function memory(options?: MemoryOptions): Durability;
// Durable agents
/** Options for `createFxAgent()`. */
export type FxAgentOptions = FxSharedOptions<FxAgentEvent> & FxModelChoice & (FxGatewayProvider | FxCodexProvider) & {
export type FxAgentOptions = FxSharedOptions<FxAgentEvent> & FxModelChoice & (FxGatewayProvider | FxCodexProvider | FxGrokProvider | FxCustomProvider) & {
/** The AI Gateway credential. Without it, libfx uses `AI_GATEWAY_API_KEY`, then on Vercel the deployment's OIDC token. */
apiKey?: string | undefined;
/** Where sessions live. Without it, libfx picks one from the environment. */
sdk/types/node.d.cts
+6 -1
View File
@@ -30,6 +30,7 @@ export type {
FxAgent,
FxAgentEvent,
FxAgentOptions,
FxApiKeyCredentials,
FxBackendAttempt,
FxBackendChoice,
FxBackendInfo,
@@ -37,9 +38,11 @@ export type {
FxBackendReason,
FxBackendReasonCode,
FxBytes,
FxChatgptCredentials,
FxCheckpointMeta,
FxCodexProvider,
FxCustomProvider,
FxCustomProviderDefinition,
FxCustomProviderModel,
FxEffort,
FxEngine,
FxEngineEvent,
@@ -52,6 +55,7 @@ export type {
FxFetch,
FxFollowUp,
FxGatewayProvider,
FxGrokProvider,
FxHostTool,
FxIdleResult,
FxImageBlock,
@@ -68,6 +72,7 @@ export type {
FxPromptBlock,
FxPromptInput,
FxPromptOptions,
FxProviderCredentials,
FxProviderTool,
FxResizedImage,
FxResourceBlock,
sdk/types/wasm.d.ts
Vendored
+6 -1
View File
@@ -14,10 +14,13 @@ export {
} from "./libfx.cjs";
export type {
FxApiKeyCredentials,
FxBytes,
FxChatgptCredentials,
FxCheckpointMeta,
FxCodexProvider,
FxCustomProvider,
FxCustomProviderDefinition,
FxCustomProviderModel,
FxEffort,
FxEngine,
FxEngineEvent,
@@ -30,6 +33,7 @@ export type {
FxFetch,
FxFollowUp,
FxGatewayProvider,
FxGrokProvider,
FxHostTool,
FxImageBlock,
FxJsonSchema,
@@ -42,6 +46,7 @@ export type {
FxOnAmbiguous,
FxPromptBlock,
FxPromptInput,
FxProviderCredentials,
FxProviderTool,
FxResizedImage,
FxResourceBlock,
src/acp/client_instructions.zig
+1 -1
View File
@@ -177,7 +177,7 @@ test "client system prompt rejects invalid combinations before a session exists"
const oversized = try alloc.alloc(u8, max_bytes + 1);
defer alloc.free(oversized);
@memset(oversized, 'x');
const json = try std.fmt.allocPrint(alloc, "{{\"systemPrompt\":[{{\"type\":\"text\",\"text\":\"{s}\"}}]}}", .{oversized});
const json = try alloc.print("{{\"systemPrompt\":[{{\"type\":\"text\",\"text\":\"{s}\"}}]}}", .{oversized});
defer alloc.free(json);
try std.testing.expectError(error.SystemPromptTooLarge, parse(alloc, json));
}
src/acp/jsonrpc.zig
+2 -1
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const io_mod = @import("../core/shared/io.zig");
const host_target = @import("../core/hosts/target.zig");
@@ -468,7 +469,7 @@ test "Reader propagates allocation failure without publishing a partial frame" {
};
var reader = Reader.initForTesting(&source, ChunkedTestSource.read);
var failing = std.testing.FailingAllocator.init(
std.testing.allocator,
testing_allocator.no_resize,
.{ .fail_index = 0 },
);
try std.testing.expectError(
src/acp/libfx_journal.zig
+2 -2
View File
@@ -61,7 +61,7 @@ pub const Journal = struct {
barrier_next_progress: bool = false,
/// Inputs taken for the next model request; its progress places them.
/// Owned by the session's allocator.
placed_next: std.ArrayListUnmanaged([]const u8) = .empty,
placed_next: std.ArrayList([]const u8) = .empty,
/// Inputs the pending resume accepted and never placed, in order. Owned
/// by the session's allocator.
pending_inputs: []journal.PendingInput = &.{},
@@ -653,7 +653,7 @@ const TestRerunner = struct {
pub fn run(self: TestRerunner, alloc: Allocator, call: types.ToolCall) !?tool_dispatch.ToolResult {
if (self.stop_at) |id| if (std.mem.eql(u8, call.id, id)) return null;
try self.ran.append(std.testing.allocator, call.id);
return .{ .success = try std.fmt.allocPrint(alloc, "ran again as {s}", .{call.id}) };
return .{ .success = try alloc.print("ran again as {s}", .{call.id}) };
}
};
src/acp/libfx_steering.zig
+4 -3
View File
@@ -2,6 +2,7 @@ const std = @import("std");
const debug_trace = @import("../core/shared/debug_trace.zig");
const io_mod = @import("../core/shared/io.zig");
const jsonrpc = @import("jsonrpc.zig");
const text_utils = @import("../core/shared/text_utils.zig");
const Allocator = std.mem.Allocator;
const RequestId = jsonrpc.RequestId;
@@ -63,8 +64,8 @@ pub const Finished = struct {
/// Owns bounded steering text shared by the ACP reader and prompt worker.
pub const Runtime = struct {
mutex: std.Io.Mutex = .init,
messages: std.ArrayListUnmanaged(Entry) = .empty,
absorbed: std.ArrayListUnmanaged(RequestId) = .empty,
messages: std.ArrayList(Entry) = .empty,
absorbed: std.ArrayList(RequestId) = .empty,
queued_bytes: usize = 0,
accepting: bool = false,
@@ -338,7 +339,7 @@ test "libfx steering runtime bounds each message and total queue" {
@memset(oversized, 'x');
try std.testing.expectError(error.SteeringMessageTooLarge, runtime.enqueue(alloc, oversized));
const message = "x" ** (max_queued_bytes / max_messages);
const message = text_utils.repeat("x", max_queued_bytes / max_messages);
for (0..max_messages) |_| try runtime.enqueue(alloc, message);
try std.testing.expectError(error.SteeringQueueFull, runtime.enqueue(alloc, "overflow"));
runtime.clear(alloc, "test");
src/acp/mcp_servers.zig
+6 -6
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const acp_types = @import("types.zig");
const builtin_tools = @import("../builtins/tools.zig");
const mcp_contract = @import("../core/mcp/mcp_contract.zig");
@@ -134,7 +135,7 @@ fn prepareHost(
// Read only while the host starts.
var inherited = io_mod.cloneEnvironMap(alloc) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
else => return .{ .failed = try std.fmt.allocPrint(alloc, "MCP servers could not read the environment: {s}", .{@errorName(err)}) },
else => return .{ .failed = try alloc.print("MCP servers could not read the environment: {s}", .{@errorName(err)}) },
};
defer inherited.deinit();
const h = mcp_host.Host.create(alloc, io_mod.getIo(), configs, &inherited, .{
@@ -144,7 +145,7 @@ fn prepareHost(
.url_completions = url_completions,
}, builtin_tools.registry) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
else => return .{ .failed = try std.fmt.allocPrint(alloc, "MCP servers could not start: {s}", .{@errorName(err)}) },
else => return .{ .failed = try alloc.print("MCP servers could not start: {s}", .{@errorName(err)}) },
};
return .{ .host = h };
}
@@ -193,8 +194,7 @@ pub fn prepare(
"Authentication required; supply an Authorization header in the ACP MCP server configuration."
else
reason;
return .{ .failed = try std.fmt.allocPrint(
alloc,
return .{ .failed = try alloc.print(
"Required MCP server '{s}' failed to start: {s}",
.{ server_state.config.name, acp_reason },
) };
@@ -270,7 +270,7 @@ noinline fn parseServerInto(
if (command_value != .string or command_value.string.len == 0) {
return error.InvalidCommand;
}
if (!std.fs.path.isAbsolute(command_value.string)) {
if (!std.Io.Dir.path.isAbsolute(command_value.string)) {
return error.CommandNotAbsolute;
}
const args_value = object.get("args") orelse return error.MissingArgs;
@@ -609,7 +609,7 @@ fn checkPreparationOwnershipAllocFailures(alloc: Allocator) !void {
test "ACP MCP preparation releases heap runtime ownership on allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkPreparationOwnershipAllocFailures,
.{},
);
src/acp/prompt.zig
+35 -35
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const skill_contract = @import("../core/skills/skill_contract.zig");
const std_builtin = @import("builtin");
const command_admission = @import("../core/permissions/command_admission.zig");
@@ -703,8 +704,8 @@ const AcpElicitationResponderContext = struct {
acp: *AcpContext,
tool_call_id: []const u8,
operation_cancel_flag: ?*const std.atomic.Value(bool),
accepted_url_ids: std.ArrayListUnmanaged([]u8) = .empty,
accepted_legacy_urls: std.ArrayListUnmanaged(AcceptedLegacyUrl) = .empty,
accepted_url_ids: std.ArrayList([]u8) = .empty,
accepted_legacy_urls: std.ArrayList(AcceptedLegacyUrl) = .empty,
fn deinit(self: *AcpElicitationResponderContext) void {
for (self.accepted_url_ids.items) |id| self.acp.state.alloc.free(id);
@@ -1093,7 +1094,7 @@ pub fn handlePrompt(
else
null,
);
if (comptime @import("builtin").os.tag != .wasi) {
if (comptime @import("builtin").target.os.tag != .wasi) {
if (state.cfg.provider_set.select(session.provider).deferred_usage != null) {
if (session.credential_source == .host_managed) {
session.session_rt.usage.replaceHostManagedReconciliationAuthority(
@@ -1544,7 +1545,7 @@ const ParsedPromptInput = struct {
for (self.pending_images, 0..) |pending, index| {
var attachment = if (pending.bytes.len == 0) ref: {
if (pending.source_ref == null or !image_data.supportedMediaType(pending.media_type)) return error.InvalidPromptImage;
const path = try std.fmt.allocPrint(alloc, image_attachments.inline_image_path_prefix ++ "{d}", .{pending.id});
const path = try alloc.print(image_attachments.inline_image_path_prefix ++ "{d}", .{pending.id});
errdefer alloc.free(path);
break :ref types.ImageAttachment{
.id = pending.id,
@@ -1873,7 +1874,7 @@ fn localFileTargetPath(alloc: Allocator, uri_text: []const u8) Allocator.Error!?
write_index += 1;
}
const decoded_path = decoded_storage[0..write_index];
if (!std.fs.path.isAbsolute(decoded_path)) return null;
if (!std.Io.Dir.path.isAbsolute(decoded_path)) return null;
var components = std.mem.splitScalar(u8, decoded_path, '/');
while (components.next()) |component| {
@@ -2451,7 +2452,7 @@ fn describeToolActionDenied(raw_ctx: *anyopaque, arena: Allocator, call: ToolCal
.display_target = display_target,
.is_available_dynamic_mcp_tool = lifecycleDynamicMcpToolAvailable(ctx, call.name, advertised_dynamic_tool_names),
});
return std.fmt.allocPrint(arena, "{s}: {s}", .{ label, action });
return arena.print("{s}: {s}", .{ label, action });
}
fn lifecycleDynamicMcpToolAvailable(ctx: *AcpContext, name: []const u8, advertised_dynamic_tool_names: []const []const u8) bool {
@@ -2561,7 +2562,7 @@ fn completeToolCallTransport(
var buffer: [512]u8 = undefined;
const notice = tool_call_presentation.shellResultNotice(result.model_output, &buffer) orelse
break :blk preview;
notice_text = std.fmt.allocPrint(ctx.alloc, "{s}\n{s}", .{ notice, preview }) catch
notice_text = ctx.alloc.print("{s}\n{s}", .{ notice, preview }) catch
break :blk preview;
break :blk notice_text.?;
};
@@ -2954,9 +2955,15 @@ fn setRecoveryCheckpoint(
checkpoint: session_codec.RecoveryCheckpoint,
) !void {
const ctx: *AcpContext = @ptrCast(@alignCast(raw_ctx));
errdefer |err| if (err == error.SessionPersistenceUncertain) {
if (ctx.current_prompt_input) |input| input.retainImageSnapshots();
writeRecoveryCheckpoint(ctx, checkpoint) catch |err| {
if (err == error.SessionPersistenceUncertain) {
if (ctx.current_prompt_input) |input| input.retainImageSnapshots();
}
return err;
};
}
fn writeRecoveryCheckpoint(ctx: *AcpContext, checkpoint: session_codec.RecoveryCheckpoint) !void {
const session = if (ctx.state.active_session) |*value| value else return error.SessionPersistenceUnavailable;
session.session_write_mutex.lockUncancelable(io_mod.getIo());
defer session.session_write_mutex.unlock(io_mod.getIo());
@@ -3162,9 +3169,9 @@ fn pushHttpError(raw_ctx: *anyopaque, status: std.http.Status, detail: []const u
null;
defer if (owned_message) |message| ctx.alloc.free(message);
const msg = owned_message orelse if (detail.len > 0)
std.fmt.bufPrint(&buf, "HTTP {d}: {s}", .{ @intFromEnum(status), detail }) catch "HTTP error"
std.mem.print(&buf, "HTTP {d}: {s}", .{ @backingInt(status), detail }) catch "HTTP error"
else
std.fmt.bufPrint(&buf, "HTTP {d}", .{@intFromEnum(status)}) catch "HTTP error";
std.mem.print(&buf, "HTTP {d}", .{@backingInt(status)}) catch "HTTP error";
ctx.sendAgentText(ctx.operationalMessageId(), msg) catch {};
}
@@ -3282,7 +3289,7 @@ fn requestAcpElicitation(
var id_buffer: [48]u8 = undefined;
const url_id = if (input_request.mode == .url)
try std.fmt.bufPrint(&id_buffer, "fx-{d}", .{outbound_id})
try std.mem.print(&id_buffer, "fx-{d}", .{outbound_id})
else
null;
const legacy_source_id = if (origin.wire.isLegacy() and input_request.mode == .url)
@@ -3414,13 +3421,11 @@ fn formatAcpElicitationMessage(
request: mcp_elicitation.Request,
) ![]u8 {
return switch (request.mode) {
.form => std.fmt.allocPrint(
alloc,
.form => alloc.print(
"fx received a form request from MCP server {s}. {s}",
.{ server_name, request.message },
),
.url => std.fmt.allocPrint(
alloc,
.url => alloc.print(
"fx received a URL request from MCP server {s} for host {s}. {s}",
.{ server_name, request.url_host orelse "unknown", request.message },
),
@@ -4249,11 +4254,10 @@ test "parsePromptInput preserves resource text and accepts only local absolute f
defer alloc.free(root);
const expected_path = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "Fx Project/src/main.zig");
defer alloc.free(expected_path);
const local_uri = try std.fmt.allocPrint(alloc, "file://{s}/Fx%20Project/src/main.zig", .{root});
const local_uri = try alloc.print("file://{s}/Fx%20Project/src/main.zig", .{root});
defer alloc.free(local_uri);
const remote_uri = "https://example.test/reference.txt";
const params = try std.fmt.allocPrint(
alloc,
const params = try alloc.print(
"{{\"sessionId\":\"s1\",\"prompt\":[" ++
"{{\"type\":\"resource\",\"resource\":{{\"uri\":\"{s}\",\"text\":\"local body\"}}}}," ++
"{{\"type\":\"resource\",\"resource\":{{\"uri\":\"{s}\",\"text\":\"remote body\"}}}}]}}",
@@ -4264,8 +4268,7 @@ test "parsePromptInput preserves resource text and accepts only local absolute f
var parsed = try parsePromptInput(alloc, params);
defer parsed.deinit(alloc);
const expected_text = try std.fmt.allocPrint(
alloc,
const expected_text = try alloc.print(
"File: {s}\nlocal body\nFile: {s}\nremote body",
.{ local_uri, remote_uri },
);
@@ -4294,8 +4297,7 @@ test "parsePromptInput rejects unsafe file URI targeting without losing embedded
};
for (uris) |uri| {
const params = try std.fmt.allocPrint(
alloc,
const params = try alloc.print(
"{{\"sessionId\":\"s1\",\"prompt\":[{{\"type\":\"resource\",\"resource\":{{\"uri\":\"{s}\",\"text\":\"embedded text\"}}}}]}}",
.{uri},
);
@@ -4333,8 +4335,8 @@ test "parsePromptInput bounds remote resource omissions with a stable summary" {
try std.testing.expect(std.mem.find(u8, parsed.omissions[parsed.omissions.len - 1].source, "/31") != null);
const summary = parsed.omission_summary orelse return error.TestExpectedEqual;
try std.testing.expectEqual(@as(usize, 96), summary.omitted_count);
try std.testing.expectEqual(@as(usize, 96), summary.reason_counts[@intFromEnum(context_contract.OmissionReason.unsafe_target)]);
const zero_digest = [_]u8{0} ** std.crypto.hash.sha2.Sha256.digest_length;
try std.testing.expectEqual(@as(usize, 96), summary.reason_counts[@backingInt(context_contract.OmissionReason.unsafe_target)]);
const zero_digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = @splat(0);
try std.testing.expect(!std.mem.eql(u8, &summary.digest, &zero_digest));
}
@@ -4349,11 +4351,11 @@ test "localFileTargetPath canonicalizes a local symlink target" {
const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, ".");
defer alloc.free(root);
const alias_path = try std.fs.path.join(alloc, &.{ root, "alias.txt" });
const alias_path = try std.Io.Dir.path.join(alloc, &.{ root, "alias.txt" });
defer alloc.free(alias_path);
const expected = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "real.txt");
defer alloc.free(expected);
const uri = try std.fmt.allocPrint(alloc, "file://{s}", .{alias_path});
const uri = try alloc.print("file://{s}", .{alias_path});
defer alloc.free(uri);
const actual = (try localFileTargetPath(alloc, uri)) orelse
@@ -4363,15 +4365,14 @@ test "localFileTargetPath canonicalizes a local symlink target" {
}
test "parsePromptInput releases partial resource state across allocation failures" {
const backing = std.testing.allocator;
const backing = testing_allocator.no_resize;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var file = try tmp.dir.createFile(std.testing.io, "local.txt", .{});
file.close(std.testing.io);
const local_path = try io_mod.dirRealpathAlloc(backing, tmp.dir, "local.txt");
defer backing.free(local_path);
const params = try std.fmt.allocPrint(
backing,
const params = try backing.print(
"{{\"sessionId\":\"s1\",\"prompt\":[" ++
"{{\"type\":\"resource\",\"resource\":{{\"uri\":\"file://{s}\",\"text\":\"local body\"}}}}," ++
"{{\"type\":\"resource\",\"resource\":{{\"uri\":\"https://example.test/reference.txt\",\"text\":\"remote body\"}}}}]}}",
@@ -4752,7 +4753,7 @@ const AcpContextRegistryFixture = struct {
fn gather(alloc: Allocator, _: context_contract.InitialContextInput) context_contract.ProviderError!context_contract.ProviderContext {
gather_calls += 1;
if (gather_error) |err| return err;
return .{ .content = try std.fmt.allocPrint(alloc, "ACP registry context {d}", .{gather_calls}) };
return .{ .content = try alloc.print("ACP registry context {d}", .{gather_calls}) };
}
fn appendStatic(input: context_contract.StaticContextInput, alloc: Allocator, messages: *std.ArrayList(ChatMessage)) context_contract.ProviderError!void {
@@ -5258,7 +5259,7 @@ fn testPermissionRuleSet(alloc: Allocator, permission: []const u8, pattern: []co
}
fn createSymlinkOrSkip(dir: std.Io.Dir, target_path: []const u8, link_path: []const u8) !void {
if (comptime @import("builtin").os.tag == .windows) return error.SkipZigTest;
if (comptime @import("builtin").target.os.tag == .windows) return error.SkipZigTest;
dir.symLink(std.testing.io, target_path, link_path, .{ .is_directory = false }) catch |err| {
if (err == error.AccessDenied or std.mem.eql(u8, @errorName(err), "Permission" ++ "Denied")) {
return error.SkipZigTest;
@@ -5623,9 +5624,8 @@ test "ACP auto mode automatic review clears or cautions prepared external file m
),
};
const target_path = try std.fs.path.join(arena, &.{ external, "desktop-test.txt" });
const arguments_json = try std.fmt.allocPrint(
arena,
const target_path = try std.Io.Dir.path.join(arena, &.{ external, "desktop-test.txt" });
const arguments_json = try arena.print(
"{{\"path\":\"{s}\",\"content\":\"hello\\n\"}}",
.{target_path},
);
src/acp/server.zig
+39 -19
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const managed_execution = @import("../core/execution/managed_execution.zig");
const acp_runner = @import("../core/cli/acp_runner.zig");
const config_runtime = @import("../core/config/config_runtime.zig");
@@ -273,7 +274,7 @@ pub const ActiveSessionState = struct {
const next = try alloc.alloc(types.PermissionGrant, self.session_grants.len + 1);
errdefer alloc.free(next);
if (self.session_grants.len > 0) {
std.mem.copyForwards(types.PermissionGrant, next[0..self.session_grants.len], self.session_grants);
@memmove(next[0..self.session_grants.len], self.session_grants);
alloc.free(self.session_grants);
}
next[next.len - 1] = .{ .tool_name = name_copy, .target_path = target_copy };
@@ -366,7 +367,7 @@ pub const ServerState = struct {
next_mcp_message_id: std.atomic.Value(u64) = .init(1),
pending_outbound: std.AutoHashMapUnmanaged(u64, PendingOutbound) = .empty,
legacy_url_mutex: std.Io.Mutex = .init,
pending_legacy_urls: std.ArrayListUnmanaged(PendingLegacyUrl) = .empty,
pending_legacy_urls: std.ArrayList(PendingLegacyUrl) = .empty,
pub fn deinit(self: *ServerState) void {
// Terminals end with this process. Ending them first also releases
@@ -498,7 +499,7 @@ pub fn selectCredentialForProvider(
/// True when requests on the provider's route carry no credential from this
/// server because the host authorizes them at its fetch edge.
fn hostManagesRoute(cfg: Config, provider: model_provider.ProviderId) bool {
return provider == .codex and cfg.codex_auth_mode == .host_managed;
return provider != .gateway and cfg.provider_auth_mode == .host_managed;
}
/// A tokenless credential that marks a route as authorized by the host.
@@ -564,12 +565,12 @@ test "ACP credential preparation preserves the existing borrowed credential" {
try std.testing.expectEqual(types.CredentialSource.configured, state.credential_source.?);
}
test "ACP Codex route is host-managed only when configured and never leaks into Gateway" {
test "ACP provider routes are host-managed only when configured and never leak into Gateway" {
const alloc = std.testing.allocator;
var state: ServerState = undefined;
state.alloc = alloc;
state.cfg.auth_mode = .local;
state.cfg.codex_auth_mode = .host_managed;
state.cfg.provider_auth_mode = .host_managed;
state.cfg.credential_override = "gateway-key";
state.active_session = null;
state.api_key = &.{};
@@ -591,9 +592,24 @@ test "ACP Codex route is host-managed only when configured and never leaks into
try std.testing.expectEqual(types.CredentialSource.ai_gateway_api_key, gateway.source);
try std.testing.expectEqualStrings("gateway-key", gateway.token);
// Without the policy Codex needs its own credential.
state.cfg.codex_auth_mode = .local;
// A Grok turn is host-managed the same way.
var grok = (try prepareCredentialForProvider(&state, .grok)).?;
defer grok.deinit(alloc);
try std.testing.expectEqual(types.CredentialSource.host_managed, grok.source);
try std.testing.expectEqual(@as(usize, 0), grok.token.len);
// So is a configured provider: the host adds its API key, if any.
const configured = model_provider.parse("local").?;
var custom = (try prepareCredentialForProvider(&state, configured)).?;
defer custom.deinit(alloc);
try std.testing.expectEqual(types.CredentialSource.host_managed, custom.source);
try std.testing.expectEqual(@as(usize, 0), custom.token.len);
// Without the policy each provider needs its own credential.
state.cfg.provider_auth_mode = .local;
try std.testing.expect(!hostManagesRoute(state.cfg, .codex));
try std.testing.expect(!hostManagesRoute(state.cfg, .grok));
try std.testing.expect(!hostManagesRoute(state.cfg, configured));
try std.testing.expect(!hostManagesRoute(state.cfg, .gateway));
}
@@ -1786,7 +1802,7 @@ fn handleKernelCheckpoint(
error.AttachmentStoreFull => return state.writer.writeError(alloc, msg.id, unavailable),
};
var response: [64]u8 = undefined;
const written = std.fmt.bufPrint(&response, "{{\"checkpointAttachment\":{d}}}", .{attachment}) catch unreachable;
const written = try std.mem.print(&response, "{{\"checkpointAttachment\":{d}}}", .{attachment});
try state.writer.writeResponse(alloc, msg.id, written);
}
@@ -2054,7 +2070,7 @@ fn handleKernelSnapshot(
error.OutOfMemory => return error.OutOfMemory,
error.AttachmentStoreFull => return state.writer.writeResponse(alloc, msg.id, none),
};
const written = try std.fmt.allocPrint(alloc, "{{\"snapshotAttachment\":{d},\"atSeq\":{d}}}", .{ attachment, at_seq });
const written = try alloc.print("{{\"snapshotAttachment\":{d},\"atSeq\":{d}}}", .{ attachment, at_seq });
defer alloc.free(written);
try state.writer.writeResponse(alloc, msg.id, written);
}
@@ -2064,7 +2080,7 @@ const SnapshotSkip = enum { too_large, too_many_turns, invalid };
fn writeSnapshotSkipped(state: *ServerState, alloc: Allocator, id: ?jsonrpc.RequestId, reason: SnapshotSkip) !void {
debug_trace.logf("session", "event=libfx_snapshot_skipped reason={s}", .{@tagName(reason)});
const body = try std.fmt.allocPrint(alloc, "{{\"snapshotAttachment\":null,\"skipped\":\"{s}\"}}", .{@tagName(reason)});
const body = try alloc.print("{{\"snapshotAttachment\":null,\"skipped\":\"{s}\"}}", .{@tagName(reason)});
defer alloc.free(body);
try state.writer.writeResponse(alloc, id, body);
}
@@ -2516,12 +2532,17 @@ test "ordinary ACP ignores private libfx capabilities" {
fn loadConfiguredStartupState(state: *const ServerState, alloc: Allocator) !app_lifecycle.StartupState {
if (state.cfg.minimal_kernel) {
return app_lifecycle.loadLibfxStartupState(
var startup = try app_lifecycle.loadLibfxStartupState(
alloc,
state.cfg.workspace_root_override orelse "/",
state.cfg.model_override orelse state.cfg.default_model,
state.cfg.default_agent_step_limit,
);
errdefer startup.deinit(alloc);
if (state.cfg.configured_providers_json) |json| {
startup.configured_providers = try @import("../core/config/configured_provider.zig").Registry.parse_json(alloc, json);
}
return startup;
}
if (state.cfg.home_override) |home_dir| {
if (state.cfg.workspace_root_override) |workspace_root| {
@@ -2875,8 +2896,7 @@ fn effortOverrideRejection(
if (index > 0) try set.appendSlice(alloc, ", ");
try set.appendSlice(alloc, option.label());
}
return try std.fmt.allocPrint(
alloc,
return try alloc.print(
"Reasoning effort \"{s}\" is not available for model \"{s}\" (available: {s})",
.{ effort.label(), model, set.items },
);
@@ -2986,7 +3006,7 @@ fn fastOverrideRejection(
model: []const u8,
) Allocator.Error!?[]u8 {
if (capabilities.supports_fast_mode or capabilities.intrinsic_fast) return null;
return try std.fmt.allocPrint(alloc, "Fast mode is not available for model \"{s}\"", .{model});
return try alloc.print("Fast mode is not available for model \"{s}\"", .{model});
}
test "fastOverrideRejection accepts models with a fast path" {
@@ -3064,7 +3084,7 @@ fn ultrafastOverrideRejection(
model: []const u8,
) Allocator.Error!?[]u8 {
if (provider == .gateway and capabilities.supports_ultrafast_mode) return null;
return try std.fmt.allocPrint(alloc, "Ultrafast mode is not available for model \"{s}\"", .{model});
return try alloc.print("Ultrafast mode is not available for model \"{s}\"", .{model});
}
test "ultrafastOverrideRejection accepts only Gateway models with the catalog capability" {
@@ -3702,7 +3722,7 @@ fn applyActiveSessionFast(
) !bool {
if (fast) {
if (!activeSessionCapabilities(state, session).supports_fast_mode) {
const message = try std.fmt.allocPrint(alloc, "Fast mode is not available for model \"{s}\"", .{session.model});
const message = try alloc.print("Fast mode is not available for model \"{s}\"", .{session.model});
defer alloc.free(message);
try state.writer.writeError(alloc, msg.id, .{
.code = ErrorCode.invalid_params,
@@ -4257,7 +4277,7 @@ test "ACP legacy URL publication owns partial allocations" {
try std.testing.expect(reserved);
}
};
try std.testing.checkAllAllocationFailures(std.testing.allocator, Case.run, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, Case.run, .{});
}
test "ACP legacy URL state requires consent and completion" {
@@ -4446,7 +4466,7 @@ fn acpModelTestState(
}
test "ACP ultrafast writes preserve v2 baselines across other preference changes" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const home = try io_mod.dirRealpathAlloc(alloc, tmp.dir, ".");
@@ -4655,7 +4675,7 @@ test "ACP rejects refreshed Codex tokens for another account" {
}
test "ACP usage flush preserves snapshot ownership on allocation failure" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
var runtime: session_runtime.SessionRuntime = .{ .max_history_turns = 8 };
var runtime_owned = true;
defer if (runtime_owned) runtime.deinit(alloc);
src/acp/sessions.zig
+15 -19
View File
@@ -3,6 +3,7 @@ const builtin = @import("builtin");
const io_mod = @import("../core/shared/io.zig");
const debug_trace = @import("../core/shared/debug_trace.zig");
const mem_utils = @import("../core/shared/mem_utils.zig");
const text_utils = @import("../core/shared/text_utils.zig");
const jsonrpc = @import("jsonrpc.zig");
const acp_types = @import("types.zig");
const mcp_servers = @import("mcp_servers.zig");
@@ -1758,7 +1759,7 @@ fn writeSessionPage(
) !void {
var next_cursor_buf: [320]u8 = undefined;
const next_cursor = if (has_more and summaries.len > 0)
try std.fmt.bufPrint(&next_cursor_buf, "v1:{d}:{s}", .{
try std.mem.print(&next_cursor_buf, "v1:{d}:{s}", .{
summaries[summaries.len - 1].updated_at_ms,
summaries[summaries.len - 1].id,
})
@@ -1779,7 +1780,7 @@ fn writeSessionPage(
);
continue;
};
if (!std.fs.path.isAbsolute(workspace_root)) {
if (!std.Io.Dir.path.isAbsolute(workspace_root)) {
debug_trace.logf(
"acp",
"session operation=list outcome=omitted id={s} reason=workspace_not_absolute",
@@ -1829,7 +1830,7 @@ fn parseListSessionsParams(
var result = ListSessionsParams{};
if (parsed.value.object.get("cwd")) |cwd| {
if (cwd != .null) {
if (cwd != .string or !std.fs.path.isAbsolute(cwd.string)) {
if (cwd != .string or !std.Io.Dir.path.isAbsolute(cwd.string)) {
return error.InvalidParams;
}
result.cwd = cwd.string;
@@ -1944,7 +1945,7 @@ fn sendUserHistoryTurn(
.{ attachment.id, @errorName(err) },
);
var unavailable: [96]u8 = undefined;
const notice = try std.fmt.bufPrint(
const notice = try std.mem.print(
&unavailable,
"Image #{d} unavailable",
.{attachment.id},
@@ -2317,9 +2318,9 @@ fn formatIso8601(alloc: Allocator, timestamp_ms: i64) ![]u8 {
const year_day = epoch.getEpochDay().calculateYearDay();
const month_day = year_day.calculateMonthDay();
return std.fmt.allocPrint(alloc, "{d:0>4}-{d:0>2}-{d:0>2}T{d:0>2}:{d:0>2}:{d:0>2}Z", .{
return alloc.print("{d:0>4}-{d:0>2}-{d:0>2}T{d:0>2}:{d:0>2}:{d:0>2}Z", .{
year_day.year,
@intFromEnum(month_day.month),
@backingInt(month_day.month),
month_day.day_index + 1,
day.getHoursIntoDay(),
day.getMinutesIntoHour(),
@@ -3206,7 +3207,7 @@ test "ACP ultrafast new load and resume preserve configured baselines on both ba
const id = try arena.dupe(u8, active.session_id);
inline for (.{ handleLoadSession, handleResumeSession }) |restore| {
try server.releaseActiveSession(&state);
msg.params_raw = try std.fmt.allocPrint(arena, "{{\"sessionId\":\"{s}\",\"mcpServers\":[]}}", .{id});
msg.params_raw = try arena.print("{{\"sessionId\":\"{s}\",\"mcpServers\":[]}}", .{id});
try restore(&state, arena, &msg);
const resumed = &state.active_session.?;
try std.testing.expectEqual(process orelse baseline, resumed.ultrafast_mode);
@@ -3239,13 +3240,12 @@ test "ACP host-disabled new load and resume skip project MCP effects" {
"workspace",
);
defer alloc.free(workspace_path);
const marker_path = try std.fs.path.join(
const marker_path = try std.Io.Dir.path.join(
alloc,
&.{ workspace_path, "project-mcp-launched" },
);
defer alloc.free(marker_path);
const project_json = try std.fmt.allocPrint(
alloc,
const project_json = try alloc.print(
"{{\"mcpServers\":{{\"fixture\":{{\"command\":\"/bin/sh\",\"args\":[\"-c\",\"printf launched > {s}\"]}},\"remote\":{{\"type\":\"http\",\"url\":\"http://127.0.0.1:1/mcp\",\"startup_timeout_ms\":5000}}}}}}",
.{marker_path},
);
@@ -3285,8 +3285,7 @@ test "ACP host-disabled new load and resume skip project MCP effects" {
.{ .method = "session/load", .handler = handleLoadSession },
.{ .method = "session/resume", .handler = handleResumeSession },
}, 0..) |restore, index| {
const params = try std.fmt.allocPrint(
arena,
const params = try arena.print(
"{{\"sessionId\":\"{s}\",\"mcpServers\":[]}}",
.{session_id},
);
@@ -3299,8 +3298,7 @@ test "ACP host-disabled new load and resume skip project MCP effects" {
try std.testing.expect(state.active_session.?.mcp == null);
}
const local_request = try std.fmt.allocPrint(
arena,
const local_request = try arena.print(
"{{\"name\":\"request-local\",\"command\":\"/bin/sh\",\"args\":[\"-c\",\"printf launched > {s}\"],\"env\":[]}}",
.{marker_path},
);
@@ -3312,14 +3310,12 @@ test "ACP host-disabled new load and resume skip project MCP effects" {
.{ .method = "session/resume", .handler = handleResumeSession, .server = local_request },
}, 0..) |request, index| {
const params = if (std.mem.eql(u8, request.method, "session/new"))
try std.fmt.allocPrint(
arena,
try arena.print(
"{{\"mcpServers\":[{s}]}}",
.{request.server},
)
else
try std.fmt.allocPrint(
arena,
try arena.print(
"{{\"sessionId\":\"{s}\",\"mcpServers\":[{s}]}}",
.{ session_id, request.server },
);
@@ -3599,6 +3595,6 @@ test "libfx/new rejects a session id that is not header and path safe" {
for (bad) |params| {
try std.testing.expectError(error.InvalidSessionId, requestedLibfxSessionId(alloc, params));
}
const long = "{\"sessionId\":\"" ++ "a" ** 256 ++ "\"}";
const long = "{\"sessionId\":\"" ++ text_utils.repeat("a", 256) ++ "\"}";
try std.testing.expectError(error.InvalidSessionId, requestedLibfxSessionId(alloc, long));
}
src/acp/tool_call_identities.zig
+2 -2
View File
@@ -30,7 +30,7 @@ pub const Target = union(enum) {
pub const Record = struct {
mutex: std.Io.Mutex = .init,
entries: std.StringArrayHashMapUnmanaged(Identity) = .empty,
entries: std.array_hash_map.String(Identity) = .empty,
pub fn deinit(self: *Record, alloc: Allocator) void {
for (self.entries.keys(), self.entries.values()) |name, identity| {
@@ -247,7 +247,7 @@ test "tool identity record refuses names that would outgrow the stored record" {
var name_buf: [32]u8 = undefined;
var accepted: usize = 0;
const full = for (0..max_entries) |index| {
const name = try std.fmt.bufPrint(&name_buf, "mcp_s_tool_{d}", .{index});
const name = try std.mem.print(&name_buf, "mcp_s_tool_{d}", .{index});
record.remember(alloc, .none, name, .{ .server = server_name, .tool = @constCast("tool") }) catch |err| {
try std.testing.expectEqual(error.ToolIdentityRecordFull, err);
break name;
src/acp/tool_call_presentation.zig
+3 -3
View File
@@ -48,9 +48,9 @@ pub fn describeToolTitle(registry: tool_dispatch.Registry, arena: Allocator, cal
});
}
if (tool_dispatch.toolCallPresentation(arena, registry, call)) |presentation| {
return std.fmt.allocPrint(arena, "{s}", .{presentation.action_label});
return arena.print("{s}", .{presentation.action_label});
}
return std.fmt.allocPrint(arena, "{s}", .{call.name});
return arena.print("{s}", .{call.name});
}
pub const ToolCallPresentation = struct {
@@ -162,7 +162,7 @@ test "describeToolCall flags internal discovery and names MCP identity" {
}
test "toolUpdateContentText clips long output and guards unsafe bytes" {
const long = "x" ** 500;
const long = text_utils.repeat("x", 500);
const clipped = toolUpdateContentText(false, long);
try std.testing.expectEqual(@as(usize, 200), clipped.len);
src/acp/workspace_binding.zig
+1 -1
View File
@@ -67,7 +67,7 @@ pub fn prepare(state: *const server.ServerState, alloc: Allocator, params_raw: ?
const cwd = parsed.value.object.get("cwd") orelse return null;
if (cwd == .null) return null;
if (cwd != .string) return error.InvalidCwd;
if (!std.fs.path.isAbsolute(cwd.string)) return error.CwdNotAbsolute;
if (!std.Io.Dir.path.isAbsolute(cwd.string)) return error.CwdNotAbsolute;
const root = io_mod.realpathAlloc(alloc, cwd.string) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
src/builtins/context.zig
+57 -59
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const debug_trace = @import("../core/shared/debug_trace.zig");
const host = @import("../core/hosts/host.zig");
const io_mod = @import("../core/shared/io.zig");
@@ -301,8 +302,8 @@ fn selectProjectContext(alloc: Allocator, options: SelectionOptions) context_con
.host => |host_files| .{ .host = host_files.reader },
.host_unreadable => blk: {
if (options.initial) {
const source = if (std.fs.path.isAbsolute(options.workspace_root))
try std.fs.path.join(arena, &.{ options.workspace_root, "AGENTS.md" })
const source = if (std.Io.Dir.path.isAbsolute(options.workspace_root))
try std.Io.Dir.path.join(arena, &.{ options.workspace_root, "AGENTS.md" })
else
options.workspace_root;
try scratch.addOmission(source, .host_unreadable);
@@ -327,7 +328,7 @@ fn selectProjectContext(alloc: Allocator, options: SelectionOptions) context_con
.host => home,
};
if (canonical_home) |home_root| {
global_source_path = try std.fs.path.join(arena, &.{ home_root, ".fx", "AGENTS.md" });
global_source_path = try std.Io.Dir.path.join(arena, &.{ home_root, ".fx", "AGENTS.md" });
global_rule = try loadRuleForSelection(arena, &scratch, files, global_source_path.?, options.context_limits.project_instruction_file_bytes);
if (pathing.pathInside(home_root, options.workspace_root)) {
if (options.bounded_reconstruction) {
@@ -343,8 +344,8 @@ fn selectProjectContext(alloc: Allocator, options: SelectionOptions) context_con
try scratch.addOmission("HOME", .home_unavailable);
}
if (std.fs.path.isAbsolute(options.workspace_root)) {
const project_source = try std.fs.path.join(arena, &.{ options.workspace_root, "AGENTS.md" });
if (std.Io.Dir.path.isAbsolute(options.workspace_root)) {
const project_source = try std.Io.Dir.path.join(arena, &.{ options.workspace_root, "AGENTS.md" });
if (global_source_path == null or
!std.mem.eql(u8, global_source_path.?, project_source))
{
@@ -500,8 +501,8 @@ fn collectLaunchAncestorCandidates(
workspace_root: []const u8,
prior_delivered: []const []const u8,
) !void {
var current = std.fs.path.dirname(workspace_root);
while (current) |scope| : (current = std.fs.path.dirname(scope)) {
var current = std.Io.Dir.path.dirname(workspace_root);
while (current) |scope| : (current = std.Io.Dir.path.dirname(scope)) {
if (std.mem.eql(u8, scope, home)) break;
if (!pathing.pathInside(home, scope)) break;
if (!try appendRuleCandidate(arena, scratch, scope, .ancestor, prior_delivered)) break;
@@ -515,7 +516,7 @@ fn collectTargetCandidates(
target: context_contract.ApplicableTarget,
) !void {
const raw_endpoint = switch (target.kind) {
.file => std.fs.path.dirname(target.path) orelse target.path,
.file => std.Io.Dir.path.dirname(target.path) orelse target.path,
.directory => target.path,
};
const endpoint = if (raw_endpoint.len > 0) raw_endpoint else target.path;
@@ -523,7 +524,7 @@ fn collectTargetCandidates(
containsString(scratch.evaluated_endpoints.items, endpoint)) return;
try scratch.addEvaluated(endpoint);
if (!std.fs.path.isAbsolute(endpoint)) {
if (!std.Io.Dir.path.isAbsolute(endpoint)) {
try scratch.addOmission(if (target.path.len > 0) target.path else "(empty target)", .unsafe_target);
return;
}
@@ -531,7 +532,7 @@ fn collectTargetCandidates(
try scratch.addRankingEndpoint(endpoint);
var current: ?[]const u8 = endpoint;
while (current) |scope| : (current = std.fs.path.dirname(scope)) {
while (current) |scope| : (current = std.Io.Dir.path.dirname(scope)) {
if (std.mem.eql(u8, scope, options.workspace_root)) break;
if (!pathing.pathInside(options.workspace_root, scope)) break;
if (!try appendRuleCandidate(arena, scratch, scope, .target, options.delivered_sources)) break;
@@ -545,7 +546,7 @@ fn appendRuleCandidate(
class: CandidateClass,
prior_delivered: []const []const u8,
) !bool {
const source = try std.fs.path.join(arena, &.{ scope, "AGENTS.md" });
const source = try std.Io.Dir.path.join(arena, &.{ scope, "AGENTS.md" });
if (containsString(prior_delivered, source) or containsString(scratch.delivered_sources.items, source)) return true;
for (scratch.candidates.items) |candidate| {
// A previous walk already covered these ancestors, or stopped at the work cap.
@@ -660,7 +661,7 @@ fn loadFilesystemRule(
if (stat.kind != .file and stat.kind != .sym_link) return .{ .omitted = .non_regular };
var file = if (stat.kind == .sym_link) blk: {
const logical_parent = std.fs.path.dirname(path) orelse return .{ .omitted = .symlink };
const logical_parent = std.Io.Dir.path.dirname(path) orelse return .{ .omitted = .symlink };
const authority = io_mod.realpathAlloc(arena, logical_parent) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
return .{ .omitted = .symlink };
@@ -671,8 +672,8 @@ fn loadFilesystemRule(
};
if (!pathing.pathInside(authority, canonical_target)) return .{ .omitted = .symlink };
const target_parent = std.fs.path.dirname(canonical_target) orelse return .{ .omitted = .symlink };
const target_name = std.fs.path.basename(canonical_target);
const target_parent = std.Io.Dir.path.dirname(canonical_target) orelse return .{ .omitted = .symlink };
const target_name = std.Io.Dir.path.basename(canonical_target);
if (target_name.len == 0) return .{ .omitted = .symlink };
var parent_dir = io_mod.openDirAbsoluteNoFollow(target_parent, .{}) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
@@ -750,7 +751,7 @@ fn pathDepth(path: []const u8) usize {
var count: usize = 0;
var in_component = false;
for (path) |byte| {
if (byte == std.fs.path.sep) {
if (byte == std.Io.Dir.path.sep) {
in_component = false;
} else if (!in_component) {
count += 1;
@@ -777,7 +778,7 @@ fn renderCandidateLessThan(_: void, lhs: *RuleCandidate, rhs: *RuleCandidate) bo
fn omissionLessThan(_: void, lhs: context_contract.ContextOmissionInput, rhs: context_contract.ContextOmissionInput) bool {
if (!std.mem.eql(u8, lhs.source, rhs.source)) return std.mem.lessThan(u8, lhs.source, rhs.source);
return @intFromEnum(lhs.reason) < @intFromEnum(rhs.reason);
return @backingInt(lhs.reason) < @backingInt(rhs.reason);
}
fn stringLessThan(_: void, lhs: []const u8, rhs: []const u8) bool {
@@ -867,7 +868,7 @@ fn appendOmissionSummaryNotice(scratch: *SelectionScratch) !void {
.{ scratch.omission_summary.omitted_count, digest },
);
const hidden_count = scratch.omission_summary.reason_counts[@intFromEnum(context_contract.OmissionReason.home_outside_workspace)];
const hidden_count = scratch.omission_summary.reason_counts[@backingInt(context_contract.OmissionReason.home_outside_workspace)];
const actionable_count = scratch.omission_summary.omitted_count - hidden_count;
if (actionable_count == 0) return;
@@ -884,12 +885,12 @@ fn appendOmissionSummaryNotice(scratch: *SelectionScratch) !void {
fn writeOmissionReasonCounts(
writer: *std.Io.Writer,
counts: [std.meta.fields(context_contract.OmissionReason).len]usize,
counts: [@typeInfo(context_contract.OmissionReason).@"enum".field_names.len]usize,
actionable_only: bool,
) !void {
var wrote_any = false;
for (counts, 0..) |count, index| {
const reason: context_contract.OmissionReason = @enumFromInt(index);
const reason: context_contract.OmissionReason = @fromBackingInt(@intCast(index));
if (count == 0 or (actionable_only and reason == .home_outside_workspace)) continue;
if (wrote_any) try writer.writeAll(", ");
try writer.print("{s}:{d}", .{ reason.label(), count });
@@ -1072,8 +1073,7 @@ fn capProjectInstructionsTotal(
if (capped.written().len > 0) try capped.writer.writeAll("\n\n");
try capped.writer.writeAll(marker);
try scratch.addNotice(try std.fmt.allocPrint(
arena,
try scratch.addNotice(try arena.print(
"[context] project instructions omitted {d} source(s) ({s}): observed={d} bytes effective={d} bytes source={s}; override with --context-limit project_instructions_total_bytes=BYTES|off",
.{ omitted_count, omitted_names.written(), observed_bytes, effective_limit, limit.source.label() },
));
@@ -1096,15 +1096,14 @@ fn projectInstructionsLimitMarker(
source: []const u8,
omitted_count: usize,
) ![]u8 {
return std.fmt.allocPrint(
alloc,
return alloc.print(
"<context_limit name=\"project_instructions_total_bytes\" action=\"omitted\" omitted_count=\"{d}\" observed_bytes=\"{d}\" effective_bytes=\"{d}\" source=\"{s}\" override=\"--context-limit project_instructions_total_bytes=BYTES|off\" />",
.{ omitted_count, observed_bytes, effective_bytes, source },
);
}
fn writeTestFile(dir: std.Io.Dir, name: []const u8, content: []const u8) !void {
if (std.fs.path.dirname(name)) |parent| {
if (std.Io.Dir.path.dirname(name)) |parent| {
try dir.createDirPath(io_mod.getIo(), parent);
}
var file = try dir.createFile(io_mod.getIo(), name, .{ .truncate = true });
@@ -1113,8 +1112,8 @@ fn writeTestFile(dir: std.Io.Dir, name: []const u8, content: []const u8) !void {
}
fn createSymlinkOrSkip(dir: std.Io.Dir, target_path: []const u8, link_path: []const u8) !void {
if (comptime @import("builtin").os.tag == .windows) return error.SkipZigTest;
if (std.fs.path.dirname(link_path)) |parent| {
if (comptime @import("builtin").target.os.tag == .windows) return error.SkipZigTest;
if (std.Io.Dir.path.dirname(link_path)) |parent| {
try dir.createDirPath(io_mod.getIo(), parent);
}
dir.symLink(std.testing.io, target_path, link_path, .{ .is_directory = false }) catch |err| {
@@ -1127,7 +1126,7 @@ test "reconstruction budget bounds distinct candidates and both file reads" {
var budget = ReconstructionBudget{};
var names: [129][8]u8 = undefined;
for (&names, 0..) |*name, index| {
const source = try std.fmt.bufPrint(name, "r{d}", .{index});
const source = try std.mem.print(name, "r{d}", .{index});
try std.testing.expectEqual(if (index < 128) ReconstructionBudget.Admission.admitted else .exhausted, budget.admit_candidate(source));
}
try std.testing.expectEqual(ReconstructionBudget.Admission.duplicate, budget.admit_candidate("r0"));
@@ -1286,11 +1285,11 @@ test "project instruction file cap bounds retained memory across large candidate
for (0..8) |index| {
if (index > 0) try relative.writer.writeByte('/');
try relative.writer.print("level-{d}", .{index});
const rule_path = try std.fmt.allocPrint(alloc, "work/{s}/AGENTS.md", .{relative.written()});
const rule_path = try alloc.print("work/{s}/AGENTS.md", .{relative.written()});
defer alloc.free(rule_path);
try writeTestFile(tmp.dir, rule_path, large_rule);
}
const target_path = try std.fmt.allocPrint(alloc, "work/{s}/target.zig", .{relative.written()});
const target_path = try alloc.print("work/{s}/target.zig", .{relative.written()});
defer alloc.free(target_path);
try writeTestFile(tmp.dir, target_path, "");
@@ -1345,9 +1344,9 @@ test "combined project instruction cap keeps global and closest whole sections i
defer arena_state.deinit();
const arena = arena_state.allocator();
var scratch = SelectionScratch{ .arena = arena };
const full = "GUIDE\n\nGLOBAL_BODY\n" ++ ("g" ** 160) ++
"\n\nMIDDLE_BODY\n" ++ ("m" ** 400) ++
"\n\nCLOSE_BODY\n" ++ ("c" ** 160) ++
const full = "GUIDE\n\nGLOBAL_BODY\n" ++ text_utils.repeat("g", 160) ++
"\n\nMIDDLE_BODY\n" ++ text_utils.repeat("m", 400) ++
"\n\nCLOSE_BODY\n" ++ text_utils.repeat("c", 160) ++
"\n\n<project-rules-omitted from=\"/unsafe/AGENTS.md\" reason=\"unsafe target\" />";
const global_start = std.mem.find(u8, full, "\n\nGLOBAL_BODY").?;
const middle_start = std.mem.find(u8, full, "\n\nMIDDLE_BODY").?;
@@ -1464,7 +1463,7 @@ test "rule loader classifies bodies blanks oversized non-regular and symlink fil
const secret_path = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "secret.md");
defer alloc.free(secret_path);
try createSymlinkOrSkip(tmp.dir, secret_path, "linked.md");
const linked_path = try std.fs.path.join(alloc, &.{ std.fs.path.dirname(secret_path).?, "linked.md" });
const linked_path = try std.Io.Dir.path.join(alloc, &.{ std.Io.Dir.path.dirname(secret_path).?, "linked.md" });
defer alloc.free(linked_path);
switch (try loadRule(arena, linked_path, rule_limit)) {
.body => |body| try std.testing.expectEqualStrings("contained linked instructions", body.text),
@@ -1484,7 +1483,7 @@ test "initial gather loads a contained AGENTS symlink with logical provenance" {
defer alloc.free(home);
const workspace = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "home/work");
defer alloc.free(workspace);
const logical_source = try std.fs.path.join(alloc, &.{ workspace, "AGENTS.md" });
const logical_source = try std.Io.Dir.path.join(alloc, &.{ workspace, "AGENTS.md" });
defer alloc.free(logical_source);
var context = try gatherProjectContextWithHome(alloc, .{
@@ -1587,19 +1586,19 @@ test "scoped selection keeps the nearest readable cap and reports unusable and c
for (0..36) |index| {
if (index > 0) try relative.writer.writeByte('/');
try relative.writer.print("level-{d:0>2}", .{index});
const rule_path = try std.fmt.allocPrint(alloc, "home/work/{s}/AGENTS.md", .{relative.written()});
const rule_path = try alloc.print("home/work/{s}/AGENTS.md", .{relative.written()});
defer alloc.free(rule_path);
if (index == 2) {
try writeTestFile(tmp.dir, rule_path, " \n ");
} else if (index == 3) {
try writeTestFile(tmp.dir, rule_path, oversized);
} else {
const body = try std.fmt.allocPrint(alloc, "RULE_LEVEL_{d:0>2}", .{index});
const body = try alloc.print("RULE_LEVEL_{d:0>2}", .{index});
defer alloc.free(body);
try writeTestFile(tmp.dir, rule_path, body);
}
}
const target_path = try std.fmt.allocPrint(alloc, "home/work/{s}/target.zig", .{relative.written()});
const target_path = try alloc.print("home/work/{s}/target.zig", .{relative.written()});
defer alloc.free(target_path);
try writeTestFile(tmp.dir, target_path, "");
@@ -2340,7 +2339,7 @@ test "project omission consequences stay bounded across many sources" {
const omissions = try arena.alloc(context_contract.ContextOmissionInput, 128);
for (omissions, 0..) |*omission, index| {
omission.* = .{
.source = try std.fmt.allocPrint(arena, "https://example.test/resource/{d}", .{index}),
.source = try arena.print("https://example.test/resource/{d}", .{index}),
.reason = .unsafe_target,
};
}
@@ -2419,7 +2418,7 @@ fn checkLaterSelectionAllocationFailures(alloc: Allocator, workspace: []const u8
}
test "later applicable selection cleans partial allocation failures" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
@@ -2569,7 +2568,7 @@ fn currentWorkingDirectory(arena: Allocator) ![]const u8 {
/// Reports the shell the shell tool runs for the user profile, falling back to
/// the environment when no login shell resolves (for example on Windows).
fn shellPath(arena: Allocator) ?[]const u8 {
var login_shell_buffer: [std.fs.max_path_bytes]u8 = undefined;
var login_shell_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined;
const login_shell = shell_resolver.configuredLoginShellInto(&login_shell_buffer);
if (shell_resolver.environment(arena, login_shell, .user)) |resolved| {
switch (resolved) {
@@ -2591,7 +2590,7 @@ fn todayUtcText(arena: Allocator) ![]const u8 {
fn formatUtcDateFromMillis(arena: Allocator, epoch_ms: i64) ![]const u8 {
const days = @divFloor(epoch_ms, std.time.ms_per_day);
const date = civilFromUnixDays(days);
return std.fmt.allocPrint(arena, "{d:0>4}-{d:0>2}-{d:0>2}", .{
return arena.print("{d:0>4}-{d:0>2}-{d:0>2}", .{
@as(u64, @intCast(date.year)),
@as(u64, @intCast(date.month)),
@as(u64, @intCast(date.day)),
@@ -2619,7 +2618,7 @@ fn collectGitInfo(arena: Allocator, workspace_root: []const u8) !GitInfo {
const git_dir = try resolveGitDir(arena, workspace_root);
if (git_dir == null or gitReadExpired(started_ns)) return .{};
const head_path = try std.fs.path.join(arena, &.{ git_dir.?, "HEAD" });
const head_path = try std.Io.Dir.path.join(arena, &.{ git_dir.?, "HEAD" });
const head = readSmallFile(arena, head_path, context_contract.Limits.git_metadata_file_bytes) catch return .{};
if (gitReadExpired(started_ns)) return .{};
@@ -2637,7 +2636,7 @@ fn collectGitInfo(arena: Allocator, workspace_root: []const u8) !GitInfo {
}
fn resolveGitDir(arena: Allocator, workspace_root: []const u8) !?[]const u8 {
const dot_git = try std.fs.path.join(arena, &.{ workspace_root, ".git" });
const dot_git = try std.Io.Dir.path.join(arena, &.{ workspace_root, ".git" });
const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), dot_git, .{ .follow_symlinks = false }) catch return null;
if (stat.kind == .directory) return dot_git;
@@ -2650,22 +2649,22 @@ fn resolveGitDir(arena: Allocator, workspace_root: []const u8) !?[]const u8 {
const raw = std.mem.trim(u8, trimmed[prefix.len..], " \t\r\n");
if (raw.len == 0) return null;
if (std.fs.path.isAbsolute(raw)) return try arena.dupe(u8, raw);
return try std.fs.path.join(arena, &.{ workspace_root, raw });
if (std.Io.Dir.path.isAbsolute(raw)) return try arena.dupe(u8, raw);
return try std.Io.Dir.path.join(arena, &.{ workspace_root, raw });
}
fn resolveCommonGitDir(arena: Allocator, git_dir: []const u8) ![]const u8 {
const common_path = try std.fs.path.join(arena, &.{ git_dir, "commondir" });
const common_path = try std.Io.Dir.path.join(arena, &.{ git_dir, "commondir" });
const content = readSmallFile(arena, common_path, context_contract.Limits.git_metadata_file_bytes) catch return git_dir;
const raw = std.mem.trim(u8, content, " \t\r\n");
if (raw.len == 0) return git_dir;
if (std.fs.path.isAbsolute(raw)) return try arena.dupe(u8, raw);
return try std.fs.path.join(arena, &.{ git_dir, raw });
if (std.Io.Dir.path.isAbsolute(raw)) return try arena.dupe(u8, raw);
return try std.Io.Dir.path.join(arena, &.{ git_dir, raw });
}
fn collectGitHubRepoIdentity(arena: Allocator, git_dir: []const u8, started_ns: i128) !?GitHubRepoIdentity {
if (gitReadExpired(started_ns)) return null;
const config_path = try std.fs.path.join(arena, &.{ git_dir, "config" });
const config_path = try std.Io.Dir.path.join(arena, &.{ git_dir, "config" });
const config = readSmallFile(arena, config_path, context_contract.Limits.git_config_file_bytes) catch return null;
if (gitReadExpired(started_ns)) return null;
return try parseGitHubRepoIdentityFromConfig(arena, config);
@@ -2695,7 +2694,7 @@ fn detectGitWorktreeState(arena: Allocator, workspace_root: []const u8, git_dir:
}
if (gitReadExpired(started_ns)) return .unknown;
const index_path = std.fs.path.join(arena, &.{ git_dir, "index" }) catch return .unknown;
const index_path = std.Io.Dir.path.join(arena, &.{ git_dir, "index" }) catch return .unknown;
const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), index_path, .{ .follow_symlinks = false }) catch return .unknown;
if (stat.kind != .file or stat.size > context_contract.Limits.git_index_file_bytes) return .unknown;
@@ -2778,7 +2777,7 @@ fn indexEntryMatchesWorktree(arena: Allocator, workspace_root: []const u8, entry
const index_mtime_nsec = std.mem.readInt(u32, entry[12..16], .big);
const index_size = std.mem.readInt(u32, entry[36..40], .big);
const abs_path = try std.fs.path.join(arena, &.{ workspace_root, path });
const abs_path = try std.Io.Dir.path.join(arena, &.{ workspace_root, path });
const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), abs_path, .{ .follow_symlinks = false }) catch |err| {
return switch (err) {
error.FileNotFound => .dirty,
@@ -2800,7 +2799,7 @@ fn indexEntryMatchesWorktree(arena: Allocator, workspace_root: []const u8, entry
}
fn isSafeIndexPath(path: []const u8) bool {
if (path.len == 0 or std.fs.path.isAbsolute(path)) return false;
if (path.len == 0 or std.Io.Dir.path.isAbsolute(path)) return false;
var parts = std.mem.splitScalar(u8, path, '/');
while (parts.next()) |part| {
@@ -2891,7 +2890,7 @@ fn parseGitHubRepoPath(arena: Allocator, raw_path: []const u8) !?GitHubRepoIdent
if (!isSafeGitHubPathComponent(owner) or !isSafeGitHubPathComponent(repo_name)) return null;
return .{
.repo = try std.fmt.allocPrint(arena, "{s}/{s}", .{ owner, repo_name }),
.repo = try arena.print("{s}/{s}", .{ owner, repo_name }),
.repo_name = try arena.dupe(u8, repo_name),
};
}
@@ -2923,7 +2922,7 @@ fn branchFromHead(arena: Allocator, head: []const u8) !?[]const u8 {
}
const short_len = @min(trimmed.len, 12);
return try std.fmt.allocPrint(arena, "detached:{s}", .{trimmed[0..short_len]});
return try arena.print("detached:{s}", .{trimmed[0..short_len]});
}
fn gitReadExpired(started_ns: i128) bool {
@@ -3109,7 +3108,7 @@ test "turn context reports the shell tool's login shell" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
var buffer: [std.fs.max_path_bytes]u8 = undefined;
var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined;
const login_shell = shell_resolver.configuredLoginShellInto(&buffer) orelse
return error.SkipZigTest;
const expected = switch (try shell_resolver.environment(arena, login_shell, .user)) {
@@ -3117,8 +3116,8 @@ test "turn context reports the shell tool's login shell" {
else => return error.TestUnexpectedResult,
};
const fragment = try buildTurnContextFragment(arena, "/tmp");
const line = try std.fmt.allocPrint(arena, "shell_path: {s}\n", .{expected});
try std.testing.expect(std.mem.indexOf(u8, fragment, line) != null);
const line = try arena.print("shell_path: {s}\n", .{expected});
try std.testing.expect(std.mem.find(u8, fragment, line) != null);
}
test "turn context keeps branch metadata inside its field" {
@@ -3428,8 +3427,7 @@ fn appendTransient(input: TransientContextInput, arena: Allocator, messages: *st
const content = if (input.interactive)
turn_context
else
try std.fmt.allocPrint(
arena,
try arena.print(
"{s}\nRuntime context: this is a noninteractive run without live question UI; when a user-owned decision remains after inspection, stop and surface a concrete blocker in freeform text with the available options. Do not recommend or label one option as preferred.",
.{turn_context},
);
src/builtins/gateway.zig
+33 -18
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const builtin = @import("builtin");
pub const permission_reviewer = @import("gateway/permission_reviewer.zig");
@@ -637,7 +638,7 @@ fn streamAgentCompletion(
else
vercel_failure_diagnostics.collect(alloc, payload, result.err_body);
if (result.status != .ok) return .{ .failed = .{
.kind = failureKind(result.status),
.kind = gatewayFailureKind(alloc, result.status, result.err_body),
.detail = result.err_body,
.diagnostics = .{
.schema = diagnostics.schema,
@@ -707,6 +708,25 @@ const EventBridge = struct {
}
};
/// Gateway reports the last provider it tried, so a provider's size rejection
/// can arrive as a 400 from a later fallback. Classify it as too large.
fn gatewayFailureKind(alloc: Allocator, status: std.http.Status, err_body: ?[]const u8) agent_stream_provider_contract.FailureKind {
const kind = failureKind(status);
if (kind != .invalid_request) return kind;
const body = err_body orelse return kind;
return if (vercel_failure_diagnostics.providerRejectedRequestSize(alloc, body)) .request_too_large else kind;
}
test "Gateway 400 with a provider 413 attempt is a size failure" {
const alloc = std.testing.allocator;
const fallback_body = "{\"providerMetadata\":{\"gateway\":{\"routing\":{\"modelAttempts\":[{\"providerAttempts\":[{\"statusCode\":413},{\"statusCode\":400}]}]}}}}";
try std.testing.expectEqual(agent_stream_provider_contract.FailureKind.request_too_large, gatewayFailureKind(alloc, .bad_request, fallback_body));
try std.testing.expectEqual(agent_stream_provider_contract.FailureKind.invalid_request, gatewayFailureKind(alloc, .bad_request, "AI_APICallError: Bad Request"));
try std.testing.expectEqual(agent_stream_provider_contract.FailureKind.invalid_request, gatewayFailureKind(alloc, .bad_request, null));
try std.testing.expectEqual(agent_stream_provider_contract.FailureKind.request_too_large, gatewayFailureKind(alloc, .payload_too_large, null));
try std.testing.expectEqual(agent_stream_provider_contract.FailureKind.rate_limited, gatewayFailureKind(alloc, .too_many_requests, fallback_body));
}
fn failureKind(status: std.http.Status) agent_stream_provider_contract.FailureKind {
return switch (status) {
.bad_request => .invalid_request,
@@ -756,7 +776,7 @@ fn fetchCreditsWithFetch(
defer if (team_path) |path| alloc.free(path);
if (gateway_team) |team| {
if (shared_types.validGatewayTeam(team)) {
team_path = std.fmt.allocPrint(alloc, "{s}?teamId={s}", .{ credits_path, team }) catch {
team_path = alloc.print("{s}?teamId={s}", .{ credits_path, team }) catch {
return creditsErrorSnapshot(alloc, "failed to fetch credits from gateway");
};
} else {
@@ -1276,16 +1296,14 @@ fn normalizeGatewayCompletion(
break :blk false;
},
.reject_malformed_identity => |failure| blk: {
try content.append(alloc, .{ .error_text = try std.fmt.allocPrint(
alloc,
try content.append(alloc, .{ .error_text = try alloc.print(
"provider search tool identity is malformed ({s})",
.{@tagName(failure)},
) });
break :blk false;
},
.reject_unstorable_identity => |failure| blk: {
const detail = try std.fmt.allocPrint(
alloc,
const detail = try alloc.print(
"provider search tool identity cannot be stored ({s}: {s})",
.{ @tagName(failure.field), @tagName(failure.reason) },
);
@@ -1294,8 +1312,7 @@ fn normalizeGatewayCompletion(
break :blk false;
},
.reject_malformed_provider_result => |failure| blk: {
try content.append(alloc, .{ .error_text = try std.fmt.allocPrint(
alloc,
try content.append(alloc, .{ .error_text = try alloc.print(
"provider search result identity is malformed ({s})",
.{@tagName(failure)},
) });
@@ -1332,7 +1349,7 @@ fn normalizeGatewayCompletion(
search_requests += 1;
map_result: {
const hits = parseSearchHits(alloc, provider_result, request.max_results) catch |err| {
try content.append(alloc, .{ .error_text = try std.fmt.allocPrint(alloc, "provider search result decode failed: {s}", .{@errorName(err)}) });
try content.append(alloc, .{ .error_text = try alloc.print("provider search result decode failed: {s}", .{@errorName(err)}) });
break :map_result;
};
errdefer deinitHits(alloc, hits);
@@ -1462,7 +1479,7 @@ fn writePerplexityDomains(alloc: Allocator, writer: *std.Io.Writer, domains: []c
for (domains, 0..) |domain, index| {
if (index > 0) try writer.writeByte(',');
if (blocked) {
const prefixed = try std.fmt.allocPrint(alloc, "-{s}", .{domain});
const prefixed = try alloc.print("-{s}", .{domain});
defer alloc.free(prefixed);
try std.json.Stringify.value(prefixed, .{}, writer);
} else {
@@ -1743,8 +1760,7 @@ test "gateway worker returns one bounded error for malformed provider result ide
try std.testing.expectEqual(@as(usize, 1), response.content.len);
try std.testing.expect(response.content[0] == .error_text);
const expected = try std.fmt.allocPrint(
std.testing.allocator,
const expected = try std.testing.allocator.print(
"provider search result identity is malformed ({s})",
.{@tagName(failure)},
);
@@ -1755,11 +1771,11 @@ test "gateway worker returns one bounded error for malformed provider result ide
}
test "gateway worker rejects unstorable provider identities without returning search results" {
try std.testing.checkAllAllocationFailures(std.testing.allocator, expectUnstorableProviderIdentity, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, expectUnstorableProviderIdentity, .{});
}
fn expectUnstorableProviderIdentity(alloc: Allocator) !void {
const oversized = [_]u8{'i'} ** 257;
const oversized: [257]u8 = @splat('i');
var cancel_flag = std.atomic.Value(bool).init(false);
var response = try normalizeGatewayCompletion(alloc, .{
.backend = perplexity_search_backend_id,
@@ -2500,7 +2516,7 @@ fn modelCatalogTeamPath(
if (access.credentialSource() != .fx_login) return null;
const team = access.teamContext() orelse return null;
if (!shared_types.validGatewayTeam(team)) return null;
return try std.fmt.allocPrint(alloc, "{s}?teamId={s}", .{ path, team });
return try alloc.print("{s}?teamId={s}", .{ path, team });
}
fn modelCatalogHeaderTeam(access: credentials.CatalogAccess) ?[]const u8 {
@@ -2556,7 +2572,7 @@ fn modelCatalogUrl(alloc: Allocator, path: []const u8, base_url_override: ?[]con
break :blk default_model_catalog_base_url;
} else default_model_catalog_base_url;
return std.fmt.allocPrint(alloc, "{s}{s}", .{ base_url, path });
return alloc.print("{s}{s}", .{ base_url, path });
}
var stable_models_test_environ: ?*std.process.Environ.Map = null;
@@ -2599,8 +2615,7 @@ const ModelsUrlTestEnv = struct {
};
fn installLoopbackModelsEnv(alloc: std.mem.Allocator, port: u16) !*ModelsUrlTestEnv {
const models_url = try std.fmt.allocPrint(
alloc,
const models_url = try alloc.print(
"http://127.0.0.1:{d}/v1/models",
.{port},
);
src/builtins/gateway/permission_reviewer.zig
+4 -4
View File
@@ -220,7 +220,7 @@ fn sendGatewayReview(
debug_trace.logf(
"permission",
"event=auto_review_transport result={s} http_status={d}",
.{ @tagName(std.meta.activeTag(outcome)), @intFromEnum(stream.status) },
.{ @tagName(std.meta.activeTag(outcome)), @backingInt(stream.status) },
);
return outcome;
}
@@ -310,7 +310,7 @@ fn mapTransportError(
}
fn mapHttpStatus(status: std.http.Status) permission_auto_classifier.TransportOutcome {
const code: u16 = @intFromEnum(status);
const code: u16 = @backingInt(status);
if (code == 408 or code == 425 or code == 429 or code >= 500) {
return .transient_failure;
}
@@ -399,8 +399,8 @@ const FakeStream = struct {
},
.timeout => error.Timeout,
.cancelled => error.Cancelled,
.transient_http => .{ .status = @enumFromInt(429) },
.permanent_http => .{ .status = @enumFromInt(400) },
.transient_http => .{ .status = @fromBackingInt(@intCast(429)) },
.permanent_http => .{ .status = @fromBackingInt(@intCast(400)) },
};
}
};
src/builtins/gateway/typesafe_permission_reviewer.zig
+5 -6
View File
@@ -293,8 +293,7 @@ fn decisionArgumentsJson(alloc: Allocator, jev: JevDecision) Allocator.Error![]u
decision: []const u8,
rationale: []const u8,
};
const rationale = try std.fmt.allocPrint(
alloc,
const rationale = try alloc.print(
"jev choice={s} p_clear={d:.3} p_caution={d:.3} confidence={d:.3} input_tokens={d} output_tokens={d}",
.{
jev.decision,
@@ -340,7 +339,7 @@ fn sendReview(
if (cancel_flag.load(.seq_cst)) return .cancelled;
if (config.api_key.len == 0 or config.endpoint.len == 0) return .permanent_failure;
const auth_header = std.fmt.allocPrint(alloc, "Bearer {s}", .{config.api_key}) catch |err| return err;
const auth_header = alloc.print("Bearer {s}", .{config.api_key}) catch |err| return err;
defer alloc.free(auth_header);
var extra_buf: [3]std.http.Header = undefined;
@@ -383,7 +382,7 @@ fn sendReview(
};
if (cancel_flag.load(.seq_cst)) return .cancelled;
const status_code: u16 = @intFromEnum(result.status);
const status_code: u16 = @backingInt(result.status);
if (result.status != .ok) {
const outcome: permission_auto_classifier.TransportOutcome =
if (status_code == 408 or status_code == 425 or status_code == 429 or status_code >= 500)
@@ -568,7 +567,7 @@ const FakeJevServer = struct {
const address = try std.Io.net.IpAddress.parse("127.0.0.1", 0);
self.server = try address.listen(self.io_backend.io(), .{ .reuse_address = true });
errdefer self.server.deinit(self.io_backend.io());
self.url = try std.fmt.allocPrint(std.testing.allocator, "http://127.0.0.1:{d}{s}", .{ self.server.socket.address.getPort(), path });
self.url = try std.testing.allocator.print("http://127.0.0.1:{d}{s}", .{ self.server.socket.address.getPort(), path });
return self;
}
@@ -626,7 +625,7 @@ const FakeJevServer = struct {
.internal_error => "500 Internal Server Error",
};
var response: [2048]u8 = undefined;
const written = try std.fmt.bufPrint(&response, "HTTP/1.1 {s}\r\ncontent-type: application/json\r\ncontent-length: {d}\r\nconnection: close\r\n\r\n{s}", .{ status, body.len, body });
const written = try std.mem.print(&response, "HTTP/1.1 {s}\r\ncontent-type: application/json\r\ncontent-length: {d}\r\nconnection: close\r\n\r\n{s}", .{ status, body.len, body });
var write_buffer: [4096]u8 = undefined;
var writer = stream.writer(zio, &write_buffer);
try writer.interface.writeAll(written);
src/builtins/mcp.zig
+19 -22
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const builtin_tools = @import("tools.zig");
const debug_trace = @import("../core/shared/debug_trace.zig");
@@ -87,8 +88,7 @@ fn handleCommand(alloc: Allocator, rest: []const u8, command_request: CommandReq
const warning_text = try renderProfileWarning(alloc, warning);
defer alloc.free(warning_text);
return .{
.display = .{ .line = try std.fmt.allocPrint(
alloc,
.display = .{ .line = try alloc.print(
"{s}\nEvaluating trusted profile MCP configuration.",
.{warning_text},
) },
@@ -128,13 +128,13 @@ fn handleCommand(alloc: Allocator, rest: []const u8, command_request: CommandReq
if (tokens.next() != null) return lineLiteral(alloc, "usage: /mcp trust approve|reject <server>", false);
if (std.mem.eql(u8, operation, "approve")) {
return .{
.display = .{ .line = try std.fmt.allocPrint(alloc, "Approving project MCP server '{s}'.", .{name}) },
.display = .{ .line = try alloc.print("Approving project MCP server '{s}'.", .{name}) },
.project_action = .{ .approve = name },
};
}
if (std.mem.eql(u8, operation, "reject")) {
return .{
.display = .{ .line = try std.fmt.allocPrint(alloc, "Rejecting project MCP server '{s}'.", .{name}) },
.display = .{ .line = try alloc.print("Rejecting project MCP server '{s}'.", .{name}) },
.project_action = .{ .reject = name },
};
}
@@ -203,7 +203,7 @@ fn handleCommand(alloc: Allocator, rest: []const u8, command_request: CommandReq
if (std.mem.startsWith(u8, trimmed, "logout ")) {
const name = std.mem.trim(u8, trimmed[7..], " \t");
if (name.len == 0 or std.mem.indexOfAny(u8, name, " \t") != null) {
if (name.len == 0 or std.mem.findAny(u8, name, " \t") != null) {
return lineLiteral(alloc, "usage: /mcp logout <name>", false);
}
const logout = command_request.logout_server orelse
@@ -234,8 +234,7 @@ fn handleCommand(alloc: Allocator, rest: []const u8, command_request: CommandReq
}
if (result.repaired_entries > 0) {
const text = if (result.revocation_failed)
try std.fmt.allocPrint(
alloc,
try alloc.print(
"Logged out of MCP server '{s}' locally; remote revocation failed. Removed {d} unreadable MCP credential {s}.",
.{
name,
@@ -244,8 +243,7 @@ fn handleCommand(alloc: Allocator, rest: []const u8, command_request: CommandReq
},
)
else
try std.fmt.allocPrint(
alloc,
try alloc.print(
"Logged out of MCP server '{s}'. Removed {d} unreadable MCP credential {s}.",
.{
name,
@@ -490,7 +488,7 @@ fn handlePromptCommand(alloc: Allocator, rest: []const u8, command_request: Comm
fn takeToken(input: *[]const u8) ?[]const u8 {
input.* = std.mem.trimStart(u8, input.*, " \t");
if (input.*.len == 0) return null;
const end = std.mem.indexOfAny(u8, input.*, " \t") orelse input.*.len;
const end = std.mem.findAny(u8, input.*, " \t") orelse input.*.len;
const token = input.*[0..end];
input.* = input.*[end..];
return token;
@@ -1332,11 +1330,11 @@ fn tmpRoot(alloc: Allocator, tmp: std.testing.TmpDir) ![]u8 {
}
fn tmpPath(alloc: Allocator, root: []const u8, name: []const u8) ![]u8 {
return std.fs.path.join(alloc, &.{ root, name });
return std.Io.Dir.path.join(alloc, &.{ root, name });
}
test "saving MCP config replaces the file durably" {
if (@import("builtin").os.tag == .windows) return error.SkipZigTest;
if (@import("builtin").target.os.tag == .windows) return error.SkipZigTest;
const alloc = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
@@ -1392,7 +1390,7 @@ fn tmpDirPath(alloc: Allocator, dir: std.Io.Dir, sub_path: []const u8) ![]u8 {
}
fn writeTempFile(tmp: *std.testing.TmpDir, sub_path: []const u8, content: []const u8) !void {
if (std.fs.path.dirname(sub_path)) |parent| {
if (std.Io.Dir.path.dirname(sub_path)) |parent| {
try tmp.dir.createDirPath(io_mod.getIo(), parent);
}
var file = try tmp.dir.createFile(io_mod.getIo(), sub_path, .{ .truncate = true });
@@ -1474,7 +1472,7 @@ test "MCP config diagnostic preserves the startup parser error" {
}
test "MCP config diagnostic propagates allocation failure" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
const test_home = try TestHome.install(alloc, "/tmp");
defer test_home.deinit();
@@ -1490,7 +1488,7 @@ test "MCP config diagnostic propagates allocation failure" {
test "MCP config diagnostic preserves parser allocation failure" {
var failing = std.testing.FailingAllocator.init(
std.testing.allocator,
testing_allocator.no_resize,
.{ .fail_index = 0 },
);
try std.testing.expectError(
@@ -1543,8 +1541,7 @@ test "workspace MCP missing environment variable is actionable and secret free"
);
const workspace_root = try tmpDirPath(alloc, tmp.dir, "workspace");
defer alloc.free(workspace_root);
const settings = try std.fmt.allocPrint(
alloc,
const settings = try alloc.print(
"{{\"workspaces\":{{\"{s}\":{{\"enableAllProjectMcpServers\":true}}}}}}",
.{workspace_root},
);
@@ -1622,7 +1619,7 @@ test "built-in MCP runtime loading leaves enabled servers disconnected" {
}
test "built-in MCP runtime config transfer cleans its unvisited suffix on append failure" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
var configs = try loadConfigFromJson(alloc,
\\{"mcp":{"first":{"type":"local","command":["node"],"enabled":false},"second":{"type":"local","command":["node"],"environment":{"TOKEN":"value"},"enabled":false}}}
);
@@ -1916,7 +1913,7 @@ test "built-in MCP command rejects invalid remote add forms without mutation" {
}
test "saving MCP config refuses a symlinked target" {
if (@import("builtin").os.tag == .windows) return error.SkipZigTest;
if (@import("builtin").target.os.tag == .windows) return error.SkipZigTest;
const alloc = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
@@ -1928,7 +1925,7 @@ test "saving MCP config refuses a symlinked target" {
defer alloc.free(external_path);
try tmp.dir.symLink(io_mod.getIo(), external_path, "home/.fx/mcp.json", .{ .is_directory = false });
const path = try std.fs.path.join(alloc, &.{ std.fs.path.dirname(external_path).?, ".fx", "mcp.json" });
const path = try std.Io.Dir.path.join(alloc, &.{ std.Io.Dir.path.dirname(external_path).?, ".fx", "mcp.json" });
defer alloc.free(path);
// The durable helper refuses a target that is not a plain private file, so
@@ -1941,7 +1938,7 @@ test "saving MCP config refuses a symlinked target" {
}
test "built-in MCP command reports a failed save instead of a missing server" {
if (@import("builtin").os.tag == .windows) return error.SkipZigTest;
if (@import("builtin").target.os.tag == .windows) return error.SkipZigTest;
const alloc = std.testing.allocator;
var fixture = ListFixture{ .text = "" };
var tmp = std.testing.tmpDir(.{});
@@ -1964,7 +1961,7 @@ test "built-in MCP command reports a failed save instead of a missing server" {
}
test "adding an MCP server creates the profile directory privately" {
if (@import("builtin").os.tag == .windows) return error.SkipZigTest;
if (@import("builtin").target.os.tag == .windows) return error.SkipZigTest;
const alloc = std.testing.allocator;
var fixture = ListFixture{ .text = "" };
var tmp = std.testing.tmpDir(.{});
src/builtins/providers.zig
+7 -1
View File
@@ -7,9 +7,15 @@ const xai_grok = @import("../gateway/xai_grok.zig");
const xai_grok_models = @import("../gateway/xai_grok_models.zig");
const xai_grok_permission_reviewer = @import("../gateway/xai_grok_permission_reviewer.zig");
const provider_catalog = @import("../core/auth/provider_catalog.zig");
const chat_completions = @import("../gateway/chat_completions.zig");
const configured_provider = @import("../core/config/configured_provider.zig");
fn configuredBundle(_: ?*anyopaque, definition: *const configured_provider.Definition) provider_set.Bundle {
return chat_completions.bundle(definition);
}
pub const native = provider_set.Set{
.configured_fn = @import("../gateway/chat_completions.zig").bundle,
.configured_fn = configuredBundle,
.gateway = gateway.provider_bundle,
.codex = .{
.presentation = provider_catalog.find(.codex),
src/builtins/skills.zig
+70 -69
View File
@@ -1,10 +1,12 @@
const std = @import("std");
const testing_allocator = @import("../core/shared/testing_allocator.zig");
const debug_trace = @import("../core/shared/debug_trace.zig");
const io_mod = @import("../core/shared/io.zig");
const skill_commands = @import("../core/skills/skill_commands.zig");
const skill_contract = @import("../core/skills/skill_contract.zig");
const skill_runtime = @import("../core/skills/skill_runtime.zig");
const text_utils = @import("../core/shared/text_utils.zig");
const Allocator = std.mem.Allocator;
@@ -184,7 +186,7 @@ fn removeCommandResult(alloc: Allocator, request: CommandRequest, name: []const
return noticeFmt(alloc, "Skill '{s}' comes from {s}, not the fx managed install root. Remove it from {s}.", .{ name, skill.source_label, skill.path }, false);
}
removeSkill(request.skills_dir, std.fs.path.basename(skill.path)) catch {
removeSkill(request.skills_dir, std.Io.Dir.path.basename(skill.path)) catch {
return noticeFmt(alloc, "Failed to remove skill '{s}'.", .{name}, false);
};
return noticeFmt(alloc, "Removed skill '{s}'.", .{name}, true);
@@ -195,7 +197,7 @@ fn noticeLiteral(alloc: Allocator, text: []const u8, reload: bool) !CommandResul
}
fn noticeFmt(alloc: Allocator, comptime fmt: []const u8, args: anytype, reload: bool) !CommandResult {
return .{ .notice = .{ .text = try std.fmt.allocPrint(alloc, fmt, args), .reload = reload } };
return .{ .notice = .{ .text = try alloc.print(fmt, args), .reload = reload } };
}
pub fn installFromSource(alloc: Allocator, skills_dir: []const u8, source: []const u8, filter: ?[]const u8) !InstallResult {
@@ -244,15 +246,15 @@ pub fn createSkillTemplate(alloc: Allocator, skills_dir: []const u8, name: []con
try io_mod.makeDirRecursive(skills_dir);
const skill_dir = try std.fs.path.join(alloc, &.{ skills_dir, name });
const skill_dir = try std.Io.Dir.path.join(alloc, &.{ skills_dir, name });
defer alloc.free(skill_dir);
try io_mod.makeDirRecursive(skill_dir);
const skill_file_path = try std.fs.path.join(alloc, &.{ skill_dir, "SKILL.md" });
const skill_file_path = try std.Io.Dir.path.join(alloc, &.{ skill_dir, "SKILL.md" });
errdefer alloc.free(skill_file_path);
const template = try std.fmt.allocPrint(alloc, "---\nname: {s}\ndescription: Describe when this skill should activate\n---\n\n# {s}\n\nInstructions for this skill...\n", .{ name, name });
const template = try alloc.print("---\nname: {s}\ndescription: Describe when this skill should activate\n---\n\n# {s}\n\nInstructions for this skill...\n", .{ name, name });
defer alloc.free(template);
var file = try std.Io.Dir.createFileAbsolute(io_mod.getIo(), skill_file_path, .{});
@@ -265,7 +267,7 @@ pub fn createSkillTemplate(alloc: Allocator, skills_dir: []const u8, name: []con
fn installFromGitHub(alloc: Allocator, skills_dir: []const u8, url: []const u8, filter: ?[]const u8) !InstallResult {
try ensureDir(skills_dir);
const tmp_dir = try std.fmt.allocPrint(alloc, "/tmp/fx-skill-install-{d}", .{io_mod.milliTimestamp()});
const tmp_dir = try alloc.print("/tmp/fx-skill-install-{d}", .{io_mod.milliTimestamp()});
defer alloc.free(tmp_dir);
defer std.Io.Dir.cwd().deleteTree(io_mod.getIo(), tmp_dir) catch {};
@@ -297,7 +299,7 @@ fn installFromGitHub(alloc: Allocator, skills_dir: []const u8, url: []const u8,
fn installFromLocalDirectory(alloc: Allocator, skills_dir: []const u8, source: []const u8, filter: ?[]const u8) !?InstallResult {
const source_dir = try resolveInstallDirectory(alloc, source) orelse return null;
defer alloc.free(source_dir);
return try installFromDirectory(alloc, skills_dir, source_dir, std.fs.path.basename(source_dir), filter);
return try installFromDirectory(alloc, skills_dir, source_dir, std.Io.Dir.path.basename(source_dir), filter);
}
fn installFromDirectory(alloc: Allocator, skills_dir: []const u8, source_dir: []const u8, fallback_name: []const u8, filter: ?[]const u8) !InstallResult {
@@ -366,12 +368,12 @@ fn installFromDirectoryWithMetadataReader(
if (entry.kind != .file) continue;
if (!std.mem.eql(u8, entry.basename, "SKILL.md")) continue;
const parent_path = std.fs.path.dirname(entry.path) orelse continue;
const dir_name = std.fs.path.basename(parent_path);
const parent_path = std.Io.Dir.path.dirname(entry.path) orelse continue;
const dir_name = std.Io.Dir.path.basename(parent_path);
if (std.mem.startsWith(u8, dir_name, ".")) continue;
skill_contract.validateManagedSkillName(dir_name) catch continue;
const skill_file = try std.fs.path.join(alloc, &.{ source_dir, entry.path });
const skill_file = try std.Io.Dir.path.join(alloc, &.{ source_dir, entry.path });
defer alloc.free(skill_file);
var file = std.Io.Dir.openFileAbsolute(io_mod.getIo(), skill_file, .{}) catch |err| {
@@ -394,7 +396,7 @@ fn installFromDirectoryWithMetadataReader(
if (!std.mem.eql(u8, dir_name, requested) and !std.mem.eql(u8, metadata.name, requested)) continue;
}
const src_dir = try std.fs.path.join(alloc, &.{ source_dir, parent_path });
const src_dir = try std.Io.Dir.path.join(alloc, &.{ source_dir, parent_path });
defer alloc.free(src_dir);
try copySkillDir(alloc, src_dir, skills_dir, dir_name);
@@ -437,7 +439,7 @@ fn propagateInstallMetadataReadError(err: InstallMetadataReadError) InstallMetad
}
fn resolveInstallDirectory(alloc: Allocator, source: []const u8) !?[]u8 {
const resolved = if (std.fs.path.isAbsolute(source))
const resolved = if (std.Io.Dir.path.isAbsolute(source))
try alloc.dupe(u8, source)
else
io_mod.realpathAlloc(alloc, source) catch return null;
@@ -516,7 +518,7 @@ fn parseRepoSkillSource(input: []const u8) ?ParsedSource {
fn isLikelyRepoSkillSource(input: []const u8) bool {
if (std.mem.startsWith(u8, input, "http://") or std.mem.startsWith(u8, input, "https://") or std.mem.startsWith(u8, input, "git@")) return false;
if (std.fs.path.isAbsolute(input)) return false;
if (std.Io.Dir.path.isAbsolute(input)) return false;
if (std.mem.startsWith(u8, input, "./") or std.mem.startsWith(u8, input, "../") or std.mem.startsWith(u8, input, "~/")) return false;
const at = std.mem.findScalarLast(u8, input, '@') orelse return false;
@@ -543,7 +545,7 @@ fn cloneUrlForSource(alloc: Allocator, url: []const u8) ![]u8 {
if (std.mem.startsWith(u8, url, "http") or std.mem.startsWith(u8, url, "git@")) {
return alloc.dupe(u8, url);
}
return std.fmt.allocPrint(alloc, "https://github.com/{s}.git", .{url});
return alloc.print("https://github.com/{s}.git", .{url});
}
fn drainCloneOutput(alloc: Allocator, child: *std.process.Child) !void {
@@ -607,15 +609,14 @@ const InstallTransactionPaths = struct {
fn init(alloc: Allocator, skills_dir: []const u8, token: [16]u8) !InstallTransactionPaths {
const suffix = std.fmt.bytesToHex(token, .lower);
const root = try std.fmt.allocPrint(
alloc,
const root = try alloc.print(
"{s}/.skill-install-{s}",
.{ skills_dir, suffix },
);
errdefer alloc.free(root);
const staged = try std.fs.path.join(alloc, &.{ root, "staged" });
const staged = try std.Io.Dir.path.join(alloc, &.{ root, "staged" });
errdefer alloc.free(staged);
const backup = try std.fs.path.join(alloc, &.{ root, "backup" });
const backup = try std.Io.Dir.path.join(alloc, &.{ root, "backup" });
return .{ .root = root, .staged = staged, .backup = backup };
}
@@ -686,7 +687,7 @@ fn acquireSkillInstallLock(
deadline_ms: u64,
lock_ops: io_mod.LockOps,
) !io_mod.TimedAdvisoryLock {
const parent_path = std.fs.path.dirname(skills_dir) orelse return error.InvalidSkillsDirectory;
const parent_path = std.Io.Dir.path.dirname(skills_dir) orelse return error.InvalidSkillsDirectory;
var parent = try std.Io.Dir.openDirAbsolute(io_mod.getIo(), parent_path, .{
.iterate = true,
.follow_symlinks = false,
@@ -716,7 +717,7 @@ fn copySkillDirWithOptions(
try skill_contract.validateManagedSkillName(skill_name);
try ensureDir(skills_dir);
const dest_dir = try std.fs.path.join(alloc, &.{ skills_dir, skill_name });
const dest_dir = try std.Io.Dir.path.join(alloc, &.{ skills_dir, skill_name });
defer alloc.free(dest_dir);
var transaction = try createInstallTransaction(alloc, skills_dir, options.transaction_tokens);
defer transaction.deinit(alloc);
@@ -764,9 +765,9 @@ fn copyDirRecursive(alloc: Allocator, src_dir: []const u8, dest_dir: []const u8)
while (try it.next(io_mod.getIo())) |entry| {
if (std.mem.startsWith(u8, entry.name, ".git")) continue;
const src_path = try std.fs.path.join(alloc, &.{ src_dir, entry.name });
const src_path = try std.Io.Dir.path.join(alloc, &.{ src_dir, entry.name });
defer alloc.free(src_path);
const dst_path = try std.fs.path.join(alloc, &.{ dest_dir, entry.name });
const dst_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, entry.name });
defer alloc.free(dst_path);
switch (entry.kind) {
@@ -778,7 +779,7 @@ fn copyDirRecursive(alloc: Allocator, src_dir: []const u8, dest_dir: []const u8)
}
fn copyFile(alloc: Allocator, src_path: []const u8, dst_path: []const u8) !void {
if (std.fs.path.dirname(dst_path)) |parent| try ensureDir(parent);
if (std.Io.Dir.path.dirname(dst_path)) |parent| try ensureDir(parent);
try io_mod.copyFileAtomic(alloc, src_path, dst_path);
}
@@ -786,7 +787,7 @@ fn ensureDir(path: []const u8) !void {
std.Io.Dir.createDirAbsolute(io_mod.getIo(), path, .default_dir) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => {
if (std.fs.path.dirname(path)) |parent| {
if (std.Io.Dir.path.dirname(path)) |parent| {
try ensureDir(parent);
try std.Io.Dir.createDirAbsolute(io_mod.getIo(), path, .default_dir);
} else {
@@ -886,7 +887,7 @@ test "install transaction payloads are not immediate skill candidates" {
defer alloc.free(source_dir);
const skills_dir = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "dest");
defer alloc.free(skills_dir);
var transaction = try InstallTransactionPaths.init(alloc, skills_dir, [_]u8{1} ** 16);
var transaction = try InstallTransactionPaths.init(alloc, skills_dir, @as([16]u8, @splat(1)));
defer transaction.deinit(alloc);
defer std.Io.Dir.cwd().deleteTree(io_mod.getIo(), transaction.root) catch {};
@@ -904,8 +905,8 @@ test "install transaction creation retries collisions without reusing roots" {
const skills_dir = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "dest");
defer alloc.free(skills_dir);
const collision_token = [_]u8{1} ** 16;
const fresh_token = [_]u8{2} ** 16;
const collision_token: [16]u8 = @splat(1);
const fresh_token: [16]u8 = @splat(2);
var collision = try InstallTransactionPaths.init(alloc, skills_dir, collision_token);
defer collision.deinit(alloc);
try std.Io.Dir.createDirAbsolute(io_mod.getIo(), collision.root, .default_dir);
@@ -935,9 +936,9 @@ test "install transaction collision exhaustion leaves the destination unchanged"
defer alloc.free(source_dir);
const skills_dir = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "dest");
defer alloc.free(skills_dir);
const installed_path = try std.fs.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
defer alloc.free(installed_path);
const collision_token = [_]u8{3} ** 16;
const collision_token: [16]u8 = @splat(3);
var collision = try InstallTransactionPaths.init(alloc, skills_dir, collision_token);
defer collision.deinit(alloc);
try std.Io.Dir.createDirAbsolute(io_mod.getIo(), collision.root, .default_dir);
@@ -945,7 +946,7 @@ test "install transaction collision exhaustion leaves the destination unchanged"
var token_state = TestTransactionTokenState{
.collision_token = collision_token,
.fresh_token = [_]u8{4} ** 16,
.fresh_token = @as([16]u8, @splat(4)),
.collision_count = install_transaction_attempts,
};
try std.testing.expectError(
@@ -1003,7 +1004,7 @@ test "skill install lock leaves do not expand valid destination names" {
const skills_dir = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "dest");
defer alloc.free(skills_dir);
const long_name = "n" ** 240;
const long_name = text_utils.repeat("n", 240);
try skill_contract.validateManagedSkillName(long_name);
var long_lock = try acquireSkillInstallLock(skills_dir, long_name, 0, .{});
long_lock.release();
@@ -1012,7 +1013,7 @@ test "skill install lock leaves do not expand valid destination names" {
var unusual_lock = try acquireSkillInstallLock(skills_dir, unusual_name, 0, .{});
unusual_lock.release();
const parent_path = std.fs.path.dirname(skills_dir).?;
const parent_path = std.Io.Dir.path.dirname(skills_dir).?;
var parent = try std.Io.Dir.openDirAbsolute(io_mod.getIo(), parent_path, .{});
defer parent.close(io_mod.getIo());
var lock_dir = try parent.openDir(io_mod.getIo(), install_lock_dir_name, .{});
@@ -1032,7 +1033,7 @@ test "busy skill install lock preserves the destination and cleans staging" {
defer alloc.free(source_dir);
const skills_dir = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "dest");
defer alloc.free(skills_dir);
const installed_path = try std.fs.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
defer alloc.free(installed_path);
{
@@ -1065,7 +1066,7 @@ test "unsupported skill install locking fails closed before destination mutation
defer alloc.free(source_dir);
const skills_dir = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "dest");
defer alloc.free(skills_dir);
const installed_path = try std.fs.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
defer alloc.free(installed_path);
try std.testing.expectError(
@@ -1091,7 +1092,7 @@ test "skill install lock remains held through rollback" {
defer alloc.free(source_dir);
const skills_dir = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "dest");
defer alloc.free(skills_dir);
const installed_path = try std.fs.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
defer alloc.free(installed_path);
var rollback_state = RollbackLockTestState{ .skills_dir = skills_dir, .skill_name = "review" };
@@ -1121,12 +1122,12 @@ test "skill install coordination stays outside discovery" {
install_lock.release();
try std.testing.expectEqual(@as(usize, 1), try countImmediateSkillDirectories(skills_dir));
const parent_path = std.fs.path.dirname(skills_dir).?;
const parent_path = std.Io.Dir.path.dirname(skills_dir).?;
var parent = try std.Io.Dir.openDirAbsolute(io_mod.getIo(), parent_path, .{});
defer parent.close(io_mod.getIo());
var sibling = try parent.openDir(io_mod.getIo(), install_lock_dir_name, .{});
sibling.close(io_mod.getIo());
const inside_path = try std.fs.path.join(alloc, &.{ skills_dir, install_lock_dir_name });
const inside_path = try std.Io.Dir.path.join(alloc, &.{ skills_dir, install_lock_dir_name });
defer alloc.free(inside_path);
try std.testing.expectError(
error.FileNotFound,
@@ -1135,7 +1136,7 @@ test "skill install coordination stays outside discovery" {
}
test "copySkillDir preserves the installed skill across allocation failures" {
const backing = std.testing.allocator;
const backing = testing_allocator.no_resize;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
@@ -1147,7 +1148,7 @@ test "copySkillDir preserves the installed skill across allocation failures" {
defer backing.free(source_dir);
const skills_dir = try io_mod.dirRealpathAlloc(backing, tmp.dir, "dest");
defer backing.free(skills_dir);
const installed_path = try std.fs.path.join(backing, &.{ skills_dir, "review", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(backing, &.{ skills_dir, "review", "SKILL.md" });
defer backing.free(installed_path);
var counting = std.testing.FailingAllocator.init(backing, .{});
@@ -1213,7 +1214,7 @@ test "global skill roots cover compatibility installs only" {
}
fn writeTempFile(tmp: *std.testing.TmpDir, sub_path: []const u8, content: []const u8) !void {
if (std.fs.path.dirname(sub_path)) |parent| {
if (std.Io.Dir.path.dirname(sub_path)) |parent| {
try tmp.dir.createDirPath(io_mod.getIo(), parent);
}
var file = try tmp.dir.createFile(std.testing.io, sub_path, .{ .truncate = true });
@@ -1222,7 +1223,7 @@ fn writeTempFile(tmp: *std.testing.TmpDir, sub_path: []const u8, content: []cons
}
fn writeLargeTempSkill(tmp: *std.testing.TmpDir, sub_path: []const u8, name: []const u8) !void {
if (std.fs.path.dirname(sub_path)) |parent| {
if (std.Io.Dir.path.dirname(sub_path)) |parent| {
try tmp.dir.createDirPath(io_mod.getIo(), parent);
}
var file = try tmp.dir.createFile(std.testing.io, sub_path, .{ .truncate = true });
@@ -1230,7 +1231,7 @@ fn writeLargeTempSkill(tmp: *std.testing.TmpDir, sub_path: []const u8, name: []c
try file.writeStreamingAll(io_mod.getIo(), "---\nname: ");
try file.writeStreamingAll(io_mod.getIo(), name);
try file.writeStreamingAll(io_mod.getIo(), "\ndescription: valid large skill\n---\n\n");
const body_chunk = [_]u8{'x'} ** (16 * 1024);
const body_chunk: [16 * 1024]u8 = @splat('x');
for (0..257) |_| try file.writeStreamingAll(io_mod.getIo(), &body_chunk);
try file.writeStreamingAll(io_mod.getIo(), "\nLARGE_SKILL_TAIL\n");
}
@@ -1260,7 +1261,7 @@ test "removeSkill deletes managed skill directory from absolute root" {
try removeSkill(skills_dir, "review");
const removed_path = try std.fs.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
const removed_path = try std.Io.Dir.path.join(alloc, &.{ skills_dir, "review", "SKILL.md" });
defer alloc.free(removed_path);
try expectNoAbsoluteFile(removed_path);
}
@@ -1272,7 +1273,7 @@ test "createSkillTemplate preserves exact generated template content" {
const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, ".");
defer alloc.free(root);
const skills_dir = try std.fs.path.join(alloc, &.{ root, "skills" });
const skills_dir = try std.Io.Dir.path.join(alloc, &.{ root, "skills" });
defer alloc.free(skills_dir);
const path = try createSkillTemplate(alloc, skills_dir, "exact-skill");
@@ -1307,7 +1308,7 @@ test "valid skill name supports template creation and removal" {
const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, ".");
defer alloc.free(root);
const skills_dir = try std.fs.path.join(alloc, &.{ root, "skills" });
const skills_dir = try std.Io.Dir.path.join(alloc, &.{ root, "skills" });
defer alloc.free(skills_dir);
const path = try createSkillTemplate(alloc, skills_dir, "simple-skill");
@@ -1337,13 +1338,13 @@ test "installFromSource installs from a local directory" {
try std.testing.expectEqual(@as(usize, 1), result.installed.items.len);
try std.testing.expectEqualStrings("review", result.installed.items[0]);
const asset_path = try std.fs.path.join(alloc, &.{ dest_dir, "review", "assets", "data.txt" });
const asset_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "review", "assets", "data.txt" });
defer alloc.free(asset_path);
const asset = try readAbsoluteFile(alloc, asset_path, 1024);
defer alloc.free(asset);
try std.testing.expectEqualStrings("asset\n", asset);
const installed_skill_path = try std.fs.path.join(alloc, &.{ dest_dir, "review", "SKILL.md" });
const installed_skill_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "review", "SKILL.md" });
defer alloc.free(installed_skill_path);
const installed_skill = try readAbsoluteFile(alloc, installed_skill_path, 1024);
defer alloc.free(installed_skill);
@@ -1378,9 +1379,9 @@ test "installFromSource skips malformed metadata and installs a valid neighbor"
try std.testing.expectEqual(@as(usize, 1), result.installed.items.len);
try std.testing.expectEqualStrings("valid", result.installed.items[0]);
const valid_path = try std.fs.path.join(alloc, &.{ dest_dir, "valid", "SKILL.md" });
const valid_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "valid", "SKILL.md" });
defer alloc.free(valid_path);
const malformed_path = try std.fs.path.join(alloc, &.{ dest_dir, "malformed", "SKILL.md" });
const malformed_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "malformed", "SKILL.md" });
defer alloc.free(malformed_path);
const installed = try readAbsoluteFile(alloc, valid_path, 1024);
defer alloc.free(installed);
@@ -1407,7 +1408,7 @@ test "installFromSource installs root skill using fallback name without frontmat
try std.testing.expectEqual(@as(usize, 1), result.installed.items.len);
try std.testing.expectEqualStrings("pack", result.installed.items[0]);
const installed_path = try std.fs.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
defer alloc.free(installed_path);
const content = try readAbsoluteFile(alloc, installed_path, 1024 * 1024);
defer alloc.free(content);
@@ -1482,13 +1483,13 @@ test "installFromSource installs nested skills and preserves nested assets" {
try std.testing.expectEqual(@as(usize, 1), result.installed.items.len);
try std.testing.expectEqualStrings("parsed-review", result.installed.items[0]);
const asset_path = try std.fs.path.join(alloc, &.{ dest_dir, "review", "assets", "data.txt" });
const asset_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "review", "assets", "data.txt" });
defer alloc.free(asset_path);
const asset = try readAbsoluteFile(alloc, asset_path, 1024);
defer alloc.free(asset);
try std.testing.expectEqualStrings("nested asset\n", asset);
const installed_path = try std.fs.path.join(alloc, &.{ dest_dir, "review", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "review", "SKILL.md" });
defer alloc.free(installed_path);
const installed = try readAbsoluteFile(alloc, installed_path, 1024);
defer alloc.free(installed);
@@ -1516,9 +1517,9 @@ test "installFromSource copies root and nested skill bodies beyond the former co
try std.testing.expectEqualStrings("large-root", result.installed.items[0]);
try std.testing.expectEqualStrings("large-nested", result.installed.items[1]);
const root_path = try std.fs.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
const root_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
defer alloc.free(root_path);
const nested_path = try std.fs.path.join(alloc, &.{ dest_dir, "nested", "SKILL.md" });
const nested_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "nested", "SKILL.md" });
defer alloc.free(nested_path);
const root_content = try readAbsoluteFile(alloc, root_path, 5 * 1024 * 1024);
defer alloc.free(root_content);
@@ -1535,7 +1536,7 @@ test "installFromSource skips a symlinked root skill and installs a valid nested
try writeTempFile(&tmp, "outside/SKILL.md", "---\nname: outside\ndescription: outside root\n---\n\nbody\n");
try writeTempFile(&tmp, "pack/valid/SKILL.md", "---\nname: valid\ndescription: valid nested skill\n---\n\nbody\n");
if (comptime @import("builtin").os.tag == .windows) return error.SkipZigTest;
if (comptime @import("builtin").target.os.tag == .windows) return error.SkipZigTest;
tmp.dir.symLink(std.testing.io, "../outside/SKILL.md", "pack/SKILL.md", .{ .is_directory = false }) catch |err| {
if (err == error.AccessDenied or err == error.FileSystem) return error.SkipZigTest;
return err;
@@ -1553,9 +1554,9 @@ test "installFromSource skips a symlinked root skill and installs a valid nested
try std.testing.expectEqual(@as(usize, 1), result.installed.items.len);
try std.testing.expectEqualStrings("valid", result.installed.items[0]);
const root_path = try std.fs.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
const root_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
defer alloc.free(root_path);
const nested_path = try std.fs.path.join(alloc, &.{ dest_dir, "valid", "SKILL.md" });
const nested_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "valid", "SKILL.md" });
defer alloc.free(nested_path);
try expectNoAbsoluteFile(root_path);
const nested = try readAbsoluteFile(alloc, nested_path, 1024);
@@ -1658,7 +1659,7 @@ test "installFromDirectory propagates root and nested metadata operational failu
}
test "installFromDirectory propagates nested metadata allocation failures" {
const backing = std.testing.allocator;
const backing = testing_allocator.no_resize;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
@@ -1809,7 +1810,7 @@ test "installFromDirectory installs root skill using fallback when no frontmatte
try std.testing.expectEqual(@as(usize, 1), result.installed.items.len);
try std.testing.expectEqualStrings("pack", result.installed.items[0]);
const installed_path = try std.fs.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "pack", "SKILL.md" });
defer alloc.free(installed_path);
const content = try readAbsoluteFile(alloc, installed_path, 1024 * 1024);
defer alloc.free(content);
@@ -1882,7 +1883,7 @@ test "installFromDirectory installs nested skills with parent destination and pa
try std.testing.expectEqual(@as(usize, 1), result.installed.items.len);
try std.testing.expectEqualStrings("parsed-review", result.installed.items[0]);
const installed_path = try std.fs.path.join(alloc, &.{ dest_dir, "review", "SKILL.md" });
const installed_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "review", "SKILL.md" });
defer alloc.free(installed_path);
const content = try readAbsoluteFile(alloc, installed_path, 1024 * 1024);
defer alloc.free(content);
@@ -1907,15 +1908,15 @@ test "copySkillDir preserves nested assets and skips git entries" {
try copySkillDir(alloc, src_dir, dest_dir, "copied");
const asset_path = try std.fs.path.join(alloc, &.{ dest_dir, "copied", "assets", "data.txt" });
const asset_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "copied", "assets", "data.txt" });
defer alloc.free(asset_path);
const asset = try readAbsoluteFile(alloc, asset_path, 1024);
defer alloc.free(asset);
try std.testing.expectEqualStrings("hello\n", asset);
const git_path = try std.fs.path.join(alloc, &.{ dest_dir, "copied", ".git", "config" });
const git_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "copied", ".git", "config" });
defer alloc.free(git_path);
const github_path = try std.fs.path.join(alloc, &.{ dest_dir, "copied", ".github", "workflows", "test.yml" });
const github_path = try std.Io.Dir.path.join(alloc, &.{ dest_dir, "copied", ".github", "workflows", "test.yml" });
defer alloc.free(github_path);
try expectNoAbsoluteFile(git_path);
try expectNoAbsoluteFile(github_path);
@@ -2031,7 +2032,7 @@ test "built-in skills command installs from a local pack" {
defer alloc.free(dest_dir);
var static_ctx = StaticSkillCtx{ .skills = &.{} };
const args = try std.fmt.allocPrint(alloc, "install {s}", .{pack_dir});
const args = try alloc.print("install {s}", .{pack_dir});
defer alloc.free(args);
var result = try executeCommand(alloc, parseCommand(args), staticCommandRequest(dest_dir, &static_ctx));
@@ -2047,7 +2048,7 @@ test "built-in skills command installs from a local pack" {
}
test "built-in skills install cleans empty result before notice allocation failure" {
const backing = std.testing.allocator;
const backing = testing_allocator.no_resize;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
@@ -2104,9 +2105,9 @@ test "built-in skills command creates and removes managed skills" {
else => return error.TestExpectedEqual,
}
const skill_dir = try std.fs.path.join(alloc, &.{ skills_dir, "exact-skill" });
const skill_dir = try std.Io.Dir.path.join(alloc, &.{ skills_dir, "exact-skill" });
defer alloc.free(skill_dir);
const skill_file = try std.fs.path.join(alloc, &.{ skill_dir, "SKILL.md" });
const skill_file = try std.Io.Dir.path.join(alloc, &.{ skill_dir, "SKILL.md" });
defer alloc.free(skill_file);
const content = try readAbsoluteFile(alloc, skill_file, 1024 * 1024);
defer alloc.free(content);
@@ -2166,7 +2167,7 @@ test "built-in skills command creates a missing managed parent root" {
const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, ".");
defer alloc.free(root);
const skills_dir = try std.fs.path.join(alloc, &.{ root, "home", ".fx", "skills" });
const skills_dir = try std.Io.Dir.path.join(alloc, &.{ root, "home", ".fx", "skills" });
defer alloc.free(skills_dir);
var empty_ctx = StaticSkillCtx{ .skills = &.{} };
@@ -2181,7 +2182,7 @@ test "built-in skills command creates a missing managed parent root" {
else => return error.TestExpectedEqual,
}
const skill_file = try std.fs.path.join(alloc, &.{ skills_dir, "fresh-root", "SKILL.md" });
const skill_file = try std.Io.Dir.path.join(alloc, &.{ skills_dir, "fresh-root", "SKILL.md" });
defer alloc.free(skill_file);
const content = try readAbsoluteFile(alloc, skill_file, 1024 * 1024);
defer alloc.free(content);
src/builtins/tools.zig
+3 -3
View File
@@ -956,14 +956,14 @@ test "built-in model-facing tool contract stays byte exact" {
test "registry classifies every built-in progress label" {
inline for (all) |tool| {
var started_buf: [96]u8 = undefined;
const started = try std.fmt.bufPrint(&started_buf, "{s} value", .{tool.action_label});
const started = try std.mem.print(&started_buf, "{s} value", .{tool.action_label});
try std.testing.expectEqual(
tool_dispatch.ProgressLabelKind.started,
tool_dispatch.classifyProgressLabel(registry, started),
);
var completed_buf: [96]u8 = undefined;
const completed = try std.fmt.bufPrint(&completed_buf, "{s} value", .{tool.completed_action_label});
const completed = try std.mem.print(&completed_buf, "{s} value", .{tool.completed_action_label});
try std.testing.expectEqual(
tool_dispatch.ProgressLabelKind.completed,
tool_dispatch.classifyProgressLabel(registry, completed),
@@ -1031,7 +1031,7 @@ test "shell advertises only run interact and stop" {
const schema_json = try tool_specs.toolGatewaySchemaJson(alloc, shell);
defer alloc.free(schema_json);
for ([_][]const u8{ "run", "interact", "stop" }) |action| {
const needle = try std.fmt.allocPrint(alloc, "\"{s}\"", .{action});
const needle = try alloc.print("\"{s}\"", .{action});
defer alloc.free(needle);
try std.testing.expect(std.mem.find(u8, schema_json, needle) != null);
}
src/core/agent/agent_runtime.zig
+2
View File
@@ -54,7 +54,9 @@ test {
_ = @import("runtime/tests/tool_flow.zig");
_ = @import("runtime/tests/interruption_flow.zig");
_ = @import("runtime/tests/finalization_flow.zig");
_ = @import("runtime/agent.zig");
_ = @import("runtime/orchestrator.zig");
_ = @import("runtime/request_image_budget.zig");
_ = @import("runtime/text_completion.zig");
_ = @import("runtime/vision_contracts.zig");
}
src/core/agent/assistant_presentation.zig
+97 -96
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../shared/testing_allocator.zig");
const Allocator = std.mem.Allocator;
const display_width = @import("../shared/display_width.zig");
@@ -507,7 +508,7 @@ pub const MarkdownProcessor = struct {
var start: usize = 0;
while (start < self.pipe_buf.items.len) {
const end = std.mem.indexOfScalarPos(u8, self.pipe_buf.items, start, '\n') orelse self.pipe_buf.items.len;
const end = std.mem.findScalarPos(u8, self.pipe_buf.items, start, '\n') orelse self.pipe_buf.items.len;
const line_has_lf = if (end + 1 == self.pipe_buf.items.len) self.pipe_last_line_has_lf else true;
try self.processLine(alloc, self.pipe_buf.items[start..end], line_has_lf, out);
if (line_has_lf) try out.append(alloc, '\n');
@@ -637,7 +638,7 @@ test "markdown link is blue and underlined inside its OSC 8 scope" {
try processor.push(alloc, "see [docs](https://example.com) please\n", &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"see \x1b]8;id=fx-{d};https://example.com\x1b\\\x1b[38;5;75m\x1b[4mdocs\x1b[24m\x1b[39m\x1b]8;;\x1b\\ please\n",
.{id_before},
@@ -656,7 +657,7 @@ test "markdown link destination keeps balanced parentheses" {
try processor.push(alloc, "[w](https://en.wikipedia.org/wiki/Foo_(bar)) tail\n", &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b]8;id=fx-{d};https://en.wikipedia.org/wiki/Foo_(bar)\x1b\\\x1b[38;5;75m\x1b[4mw\x1b[24m\x1b[39m\x1b]8;;\x1b\\ tail\n",
.{id_before},
@@ -679,7 +680,7 @@ test "markdown link drops its title and unwraps angle destinations" {
);
var expected_buf: [512]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b]8;id=fx-{d};https://example.com\x1b\\\x1b[38;5;75m\x1b[4mt\x1b[24m\x1b[39m\x1b]8;;\x1b\\ " ++
"\x1b]8;id=fx-{d};https://example.com/s\x1b\\\x1b[38;5;75m\x1b[4ms\x1b[24m\x1b[39m\x1b]8;;\x1b\\ " ++
@@ -711,7 +712,7 @@ test "markdown image renders its alt text with an image marker inside one OSC 8
try processor.push(alloc, "see ![architecture diagram](https://example.com/diagram.png) please\n", &out);
var expected_buf: [512]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"see \x1b]8;id=fx-{d};https://example.com/diagram.png\x1b\\\x1b[38;5;75m\x1b[4m▧ architecture diagram\x1b[24m\x1b[39m\x1b]8;;\x1b\\ please\n",
.{id_before},
@@ -730,7 +731,7 @@ test "markdown image uses a stable fallback for empty alt text" {
try processor.push(alloc, "![](https://example.com/diagram.png)\n", &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b]8;id=fx-{d};https://example.com/diagram.png\x1b\\\x1b[38;5;75m\x1b[4m▧ image\x1b[24m\x1b[39m\x1b]8;;\x1b\\\n",
.{id_before},
@@ -749,7 +750,7 @@ test "markdown image unescapes alt punctuation through the existing link emitter
try processor.push(alloc, "![architecture \\*diagram\\*](https://example.com/diagram.png)\n", &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b]8;id=fx-{d};https://example.com/diagram.png\x1b\\\x1b[38;5;75m\x1b[4m▧ architecture *diagram*\x1b[24m\x1b[39m\x1b]8;;\x1b\\\n",
.{id_before},
@@ -766,26 +767,26 @@ test "escaped and malformed markdown images remain literal without OSC 8" {
try processor.push(alloc, "\\![alt](https://example.com/diagram.png) and \\!\n", &out);
try std.testing.expectEqualStrings("![alt](https://example.com/diagram.png) and !\n", out.items);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
out.clearRetainingCapacity();
try processor.push(alloc, "![alt](https://example.com/diagram.png\n", &out);
try std.testing.expectEqualStrings("![alt](https://example.com/diagram.png\n", out.items);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
out.clearRetainingCapacity();
try processor.push(alloc, "![alt](https://example.com/\x1bdiagram.png)\n", &out);
try std.testing.expectEqualStrings("![alt](https://example.com/\x1bdiagram.png)\n", out.items);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
var oversized_url: [max_link_url_bytes + 1]u8 = undefined;
@memset(&oversized_url, 'a');
var input_buf: [max_link_url_bytes + 16]u8 = undefined;
const oversized = try std.fmt.bufPrint(&input_buf, "![alt]({s})\n", .{oversized_url[0..]});
const oversized = try std.mem.print(&input_buf, "![alt]({s})\n", .{oversized_url[0..]});
out.clearRetainingCapacity();
try processor.push(alloc, oversized, &out);
try std.testing.expectEqualStrings(oversized, out.items);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
}
test "markdown images preserve code isolation, chunk buffering, and heading underline" {
@@ -800,19 +801,19 @@ test "markdown images preserve code isolation, chunk buffering, and heading unde
"\x1b[38;5;245m![literal](https://example.com/literal.png)\x1b[39m\n",
out.items,
);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
out.clearRetainingCapacity();
try processor.push(alloc, "![architecture", &out);
try std.testing.expectEqual(@as(usize, 0), out.items.len);
try processor.push(alloc, "](https://example.com/diagram.png)\n", &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "▧ architecture") != null);
try std.testing.expect(std.mem.find(u8, out.items, "▧ architecture") != null);
out.clearRetainingCapacity();
const id_before = link_id_counter;
try processor.push(alloc, "### before ![diagram](https://example.com/diagram.png) after\n", &out);
var expected_buf: [512]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b[4mbefore \x1b]8;id=fx-{d};https://example.com/diagram.png\x1b\\\x1b[38;5;75m\x1b[4m▧ diagram\x1b[24m\x1b[39m\x1b]8;;\x1b\\\x1b[4m after\x1b[24m\n",
.{id_before},
@@ -831,8 +832,8 @@ test "table payload measures markdown images by their visible label" {
defer table.deinit(alloc);
const cell = table.rows[1].cells[0];
try std.testing.expect(std.mem.indexOf(u8, cell, "▧ diagram") != null);
try std.testing.expect(std.mem.indexOf(u8, cell, "![diagram]") == null);
try std.testing.expect(std.mem.find(u8, cell, "▧ diagram") != null);
try std.testing.expect(std.mem.find(u8, cell, "![diagram]") == null);
try std.testing.expectEqual(
display_width.visibleWidthIgnoringAnsi("▧ diagram"),
display_width.visibleWidthIgnoringAnsi(cell),
@@ -860,12 +861,12 @@ test "table payload links use the shared OSC 8 identifier sequence" {
defer second.deinit(alloc);
var first_id_buf: [64]u8 = undefined;
const first_id = try std.fmt.bufPrint(&first_id_buf, "\x1b]8;id=fx-{d};https://first.example", .{id_before});
try std.testing.expect(std.mem.indexOf(u8, first.rows[1].cells[0], first_id) != null);
const first_id = try std.mem.print(&first_id_buf, "\x1b]8;id=fx-{d};https://first.example", .{id_before});
try std.testing.expect(std.mem.find(u8, first.rows[1].cells[0], first_id) != null);
var second_id_buf: [64]u8 = undefined;
const second_id = try std.fmt.bufPrint(&second_id_buf, "\x1b]8;id=fx-{d};https://second.example", .{id_before +% 1});
try std.testing.expect(std.mem.indexOf(u8, second.rows[1].cells[0], second_id) != null);
const second_id = try std.mem.print(&second_id_buf, "\x1b]8;id=fx-{d};https://second.example", .{id_before +% 1});
try std.testing.expect(std.mem.find(u8, second.rows[1].cells[0], second_id) != null);
}
test "bare URL is underlined and leaves sentence punctuation literal" {
@@ -879,7 +880,7 @@ test "bare URL is underlined and leaves sentence punctuation literal" {
try processor.push(alloc, "visit https://example.com/docs, now\n", &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"visit \x1b]8;id=fx-{d};https://example.com/docs\x1b\\\x1b[38;5;75m\x1b[4mhttps://example.com/docs\x1b[24m\x1b[39m\x1b]8;;\x1b\\, now\n",
.{id_before},
@@ -900,7 +901,7 @@ test "bare URL is recognized after streamed input chunks" {
try processor.push(alloc, "://example.com/docs\n", &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"visit \x1b]8;id=fx-{d};https://example.com/docs\x1b\\\x1b[38;5;75m\x1b[4mhttps://example.com/docs\x1b[24m\x1b[39m\x1b]8;;\x1b\\\n",
.{id_before},
@@ -923,7 +924,7 @@ test "angle autolinks use literal URI and email labels" {
);
var expected_buf: [1024]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"see \x1b]8;id=fx-{d};https://example.com/docs\\_literal\x1b\\\x1b[38;5;75m\x1b[4mhttps://example.com/docs\\_literal\x1b[24m\x1b[39m\x1b]8;;\x1b\\ and " ++
"\x1b]8;id=fx-{d};mailto:dev@example.com\x1b\\\x1b[38;5;75m\x1b[4mdev@example.com\x1b[24m\x1b[39m\x1b]8;;\x1b\\ plus " ++
@@ -963,7 +964,7 @@ test "rejected angle candidates suppress nested link emission" {
try processor.push(alloc, input, &out);
try std.testing.expectEqualStrings(expected, out.items);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
}
test "angle email autolink applies the destination cap including mailto" {
@@ -978,11 +979,11 @@ test "angle email autolink applies the destination cap including mailto" {
@memset(&local, 'a');
const local_len = max_link_url_bytes - "mailto:".len - suffix.len + 1;
var input_buf: [max_link_url_bytes + 32]u8 = undefined;
const input = try std.fmt.bufPrint(&input_buf, "<{s}{s}>\n", .{ local[0..local_len], suffix });
const input = try std.mem.print(&input_buf, "<{s}{s}>\n", .{ local[0..local_len], suffix });
try processor.push(alloc, input, &out);
try std.testing.expectEqualStrings(input, out.items);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
}
test "standalone URL in code is linked while excluded boundaries stay literal" {
@@ -999,7 +1000,7 @@ test "standalone URL in code is linked while excluded boundaries stay literal" {
&out,
);
var expected_buf: [512]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b[38;5;245m\x1b]8;id=fx-{d};https://code.example\x1b\\https://code.example\x1b]8;;\x1b\\\x1b[39m wordhttps://word.example <<https://angle.example>>\n",
.{id_before},
@@ -1017,7 +1018,7 @@ test "inline code URLs keep literal bytes and trailing punctuation outside links
const id_before = link_id_counter;
try processor.push(alloc, "See `https://example.com/path_~*.` and `https://example.com/a\\_b`.\n", &out);
var expected_buf: [1024]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"See \x1b[38;5;245m\x1b]8;id=fx-{d};https://example.com/path_~*\x1b\\https://example.com/path_~*\x1b]8;;\x1b\\.\x1b[39m and " ++
"\x1b[38;5;245m\x1b]8;id=fx-{d};https://example.com/a\\_b\x1b\\https://example.com/a\\_b\x1b]8;;\x1b\\\x1b[39m.\n",
@@ -1041,11 +1042,11 @@ test "inline code URLs preserve literal trailing URI punctuation" {
var out: std.ArrayList(u8) = .empty;
defer out.deinit(alloc);
var input_buf: [128]u8 = undefined;
const input = try std.fmt.bufPrint(&input_buf, "Open `{s}`\n", .{url});
const input = try std.mem.print(&input_buf, "Open `{s}`\n", .{url});
const id_before = link_id_counter;
try processor.push(alloc, input, &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"Open \x1b[38;5;245m\x1b]8;id=fx-{d};{s}\x1b\\{s}\x1b]8;;\x1b\\\x1b[39m\n",
.{ id_before, url, url },
@@ -1076,7 +1077,7 @@ test "non-URL and unsafe inline code stays literal" {
var oversized: [max_link_url_bytes + 1]u8 = undefined;
@memset(&oversized, 'a');
var input_buf: [max_link_url_bytes + 32]u8 = undefined;
const input = try std.fmt.bufPrint(&input_buf, "`https://{s}`\n", .{oversized[0..]});
const input = try std.mem.print(&input_buf, "`https://{s}`\n", .{oversized[0..]});
var processor = MarkdownProcessor{};
defer processor.deinit(alloc);
var out: std.ArrayList(u8) = .empty;
@@ -1095,7 +1096,7 @@ test "unsafe bare URL renders literally" {
var oversized: [max_link_url_bytes + 1]u8 = undefined;
@memset(&oversized, 'a');
var input_buf: [max_link_url_bytes + 16]u8 = undefined;
const input = try std.fmt.bufPrint(&input_buf, "https://{s}\n", .{oversized[0..]});
const input = try std.mem.print(&input_buf, "https://{s}\n", .{oversized[0..]});
try processor.push(alloc, input, &out);
try std.testing.expectEqualStrings(input, out.items);
@@ -1117,7 +1118,7 @@ test "bare URLs leave closing emphasis delimiters for the inline scanner" {
);
var expected_buf: [1024]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b[1m\x1b]8;id=fx-{d};https://bold.example\x1b\\\x1b[38;5;75m\x1b[4mhttps://bold.example\x1b[24m\x1b[39m\x1b]8;;\x1b\\\x1b[22m " ++
"\x1b[3m\x1b]8;id=fx-{d};https://italic.example\x1b\\\x1b[38;5;75m\x1b[4mhttps://italic.example\x1b[24m\x1b[39m\x1b]8;;\x1b\\\x1b[23m " ++
@@ -1150,7 +1151,7 @@ test "heading underline resumes after a link closes its local underline" {
try processor.push(alloc, "### before [link](https://example.com) after\nbody\n", &out);
var expected_buf: [512]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b[4mbefore \x1b]8;id=fx-{d};https://example.com\x1b\\\x1b[38;5;75m\x1b[4mlink\x1b[24m\x1b[39m\x1b]8;;\x1b\\\x1b[4m after\x1b[24m\nbody\n",
.{id_before},
@@ -1167,7 +1168,7 @@ test "two markdown links get distinct ids" {
try processor.push(alloc, "[a](https://a.example) and [b](https://b.example)\n", &out);
var seq_buf: [32]u8 = undefined;
const first_marker = try std.fmt.bufPrint(&seq_buf, "id=fx-", .{});
const first_marker = try std.mem.print(&seq_buf, "id=fx-", .{});
var occurrences: usize = 0;
var idx: usize = 0;
while (std.mem.find(u8, out.items[idx..], first_marker)) |off| : (idx += off + first_marker.len) {
@@ -1187,7 +1188,7 @@ test "url over OSC 8 size cap renders literally" {
@memset(&oversized_buf, 'a');
const url = oversized_buf[0 .. max_link_url_bytes + 1];
var input_buf: [4000]u8 = undefined;
const input = try std.fmt.bufPrint(&input_buf, "see [x]({s}) end\n", .{url});
const input = try std.mem.print(&input_buf, "see [x]({s}) end\n", .{url});
try processor.push(alloc, input, &out);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
@@ -1244,7 +1245,7 @@ test "backslash escapes keep inline punctuation literal" {
"literal *em* **bold** _italic_ __strong__ ~~strike~~ `code` [docs](https://example.com) \\ | ! # >\n",
out.items,
);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b") == null);
}
test "backslash escapes survive heading preprocessing and code spans" {
@@ -1378,7 +1379,7 @@ test "link labels unescape visible punctuation" {
try processor.push(alloc, "[docs \\*literal\\*](https://example.com)\n", &out);
var expected_buf: [256]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b]8;id=fx-{d};https://example.com\x1b\\\x1b[38;5;75m\x1b[4mdocs *literal*\x1b[24m\x1b[39m\x1b]8;;\x1b\\\n",
.{id_before},
@@ -1480,8 +1481,8 @@ test "underscore emphasis renders in list, blockquote, and table cells" {
defer table.deinit(alloc);
out.clearRetainingCapacity();
try renderTablePayload(alloc, table, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[3mrow\x1b[23m") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[1mcell\x1b[22m") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[3mrow\x1b[23m") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[1mcell\x1b[22m") != null);
}
test "underscore formatted bare URLs retain path underscores and matching closers" {
@@ -1499,7 +1500,7 @@ test "underscore formatted bare URLs retain path underscores and matching closer
);
var expected_buf: [1024]u8 = undefined;
const expected = try std.fmt.bufPrint(
const expected = try std.mem.print(
&expected_buf,
"\x1b[3m\x1b]8;id=fx-{d};https://example.com/snake_case\x1b\\\x1b[38;5;75m\x1b[4mhttps://example.com/snake_case\x1b[24m\x1b[39m\x1b]8;;\x1b\\\x1b[23m tail " ++
"\x1b[1m\x1b]8;id=fx-{d};https://example.com/snake_case\x1b\\\x1b[38;5;75m\x1b[4mhttps://example.com/snake_case\x1b[24m\x1b[39m\x1b]8;;\x1b\\\x1b[22m tail\n",
@@ -1524,7 +1525,7 @@ test "underscore formatted URLs require exact active markers" {
defer out.deinit(alloc);
try processor.push(alloc, input, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;") == null);
}
// Trailing underscores are outside the URL, as in GFM autolinks, so the
@@ -1547,7 +1548,7 @@ test "underscore formatted URLs require exact active markers" {
defer out.deinit(alloc);
try processor.push(alloc, case.input, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, case.url) != null);
try std.testing.expect(std.mem.find(u8, out.items, case.url) != null);
try std.testing.expect(std.mem.startsWith(u8, out.items, case.prefix));
try std.testing.expect(std.mem.endsWith(u8, out.items, case.tail));
}
@@ -1623,7 +1624,7 @@ test "table payload headers reassert outer bold after inline strong closes" {
defer out.deinit(alloc);
try renderTablePayload(alloc, table, &out);
try std.testing.expect(std.mem.indexOf(
try std.testing.expect(std.mem.find(
u8,
out.items,
"\x1b[1mprefix \x1b[1mstrong\x1b[22m\x1b[1m suffix\x1b[22m",
@@ -1667,7 +1668,7 @@ test "numeric entities for control characters stay literal" {
try processor.push(alloc, "x&#27;[2Ky &#x1b;[31m &#7; &#127;&#x9b; &#0; &#x41;\n", &out);
try std.testing.expectEqualStrings("x&#27;[2Ky &#x1b;[31m &#7; &#127;&#x9b; \xef\xbf\xbd A\n", out.items);
try std.testing.expect(std.mem.indexOfScalar(u8, out.items, 0x1b) == null);
try std.testing.expect(std.mem.findScalar(u8, out.items, 0x1b) == null);
}
test "entity lookup is bounded and a long ampersand line renders in bounded time" {
@@ -1776,7 +1777,7 @@ test "headings use level-specific ANSI styles" {
try processor.push(alloc, case.input, &out);
try std.testing.expectEqualStrings(case.expected, out.items);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[1;4m") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[1;4m") == null);
}
}
@@ -1909,8 +1910,8 @@ test "definition state stops before a fenced code block" {
);
try processor.flushWithCompletions(alloc, &out, .{ .thematic_rule = &completion });
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[2m \x1b[22maccepted\n") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\n: stale\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[2m \x1b[22maccepted\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\n: stale\n") != null);
}
test "definition state stops before a pipe table" {
@@ -1936,8 +1937,8 @@ test "definition state stops before a pipe table" {
);
try processor.flushWithCompletions(alloc, &out, .{ .thematic_rule = &completion });
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[2m \x1b[22maccepted\n") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\n: stale\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[2m \x1b[22maccepted\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\n: stale\n") != null);
}
test "definition marker cannot skip a fenced code block" {
@@ -1963,8 +1964,8 @@ test "definition marker cannot skip a fenced code block" {
);
try processor.flushWithCompletions(alloc, &out, .{ .thematic_rule = &completion });
try std.testing.expect(std.mem.indexOf(u8, out.items, "Status\n") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\n: stale\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "Status\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\n: stale\n") != null);
}
test "definition marker cannot skip a pipe table" {
@@ -1990,8 +1991,8 @@ test "definition marker cannot skip a pipe table" {
);
try processor.flushWithCompletions(alloc, &out, .{ .thematic_rule = &completion });
try std.testing.expect(std.mem.indexOf(u8, out.items, "Status\n") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\n: stale\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "Status\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\n: stale\n") != null);
}
test "definition markers without a term stay literal" {
@@ -2175,7 +2176,7 @@ fn checkSetextLookaheadAllocationFailures(alloc: Allocator) !void {
test "setext lookahead frees allocation failures" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkSetextLookaheadAllocationFailures,
.{},
);
@@ -2212,7 +2213,7 @@ fn checkDefinitionListAllocationFailures(alloc: Allocator) !void {
test "definition lists free allocation failures" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkDefinitionListAllocationFailures,
.{},
);
@@ -2257,8 +2258,8 @@ test "table cell footnote references project deferred definitions" {
);
try processor.flush(alloc, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "[^state]") == null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[2m[1] \x1b[22m") != null);
try std.testing.expect(std.mem.find(u8, out.items, "[^state]") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[2m[1] \x1b[22m") != null);
try std.testing.expect(std.mem.endsWith(
u8,
out.items,
@@ -2354,11 +2355,11 @@ test "escaped malformed and code footnote candidates remain literal" {
);
try processor.flush(alloc, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "[1]") == null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "[^escaped]") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "[^code]") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "[^missing]:") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "[^block]: literal code") != null);
try std.testing.expect(std.mem.find(u8, out.items, "[1]") == null);
try std.testing.expect(std.mem.find(u8, out.items, "[^escaped]") != null);
try std.testing.expect(std.mem.find(u8, out.items, "[^code]") != null);
try std.testing.expect(std.mem.find(u8, out.items, "[^missing]:") != null);
try std.testing.expect(std.mem.find(u8, out.items, "[^block]: literal code") != null);
}
fn checkFootnoteAllocationFailures(alloc: Allocator) !void {
@@ -2379,7 +2380,7 @@ fn checkFootnoteAllocationFailures(alloc: Allocator) !void {
test "footnotes free allocation failures" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkFootnoteAllocationFailures,
.{},
);
@@ -2760,9 +2761,9 @@ test "lazy blockquote continuations retain inline links" {
defer out.deinit(alloc);
try processor.push(alloc, "> source\nlazy [link](https://example.com/lazy)\n", &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\n\x1b[2m\xe2\x94\x82 \x1b[22mlazy ") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b]8;id=fx-") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "https://example.com/lazy") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\n\x1b[2m\xe2\x94\x82 \x1b[22mlazy ") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b]8;id=fx-") != null);
try std.testing.expect(std.mem.find(u8, out.items, "https://example.com/lazy") != null);
}
test "lazy blockquotes stop before structural rows and preserve EOF backslashes" {
@@ -3197,10 +3198,10 @@ test "indented code requires a blank boundary and yields to list and lazy quote
);
try std.testing.expect(capture.block == null);
try std.testing.expect(std.mem.indexOf(u8, out.items, " continuation") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, " three spaces") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[2m\xe2\x94\x82 \x1b[22m lazy continuation") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[2m\xe2\x94\x82 \x1b[22m- list item") == null);
try std.testing.expect(std.mem.find(u8, out.items, " continuation") != null);
try std.testing.expect(std.mem.find(u8, out.items, " three spaces") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[2m\xe2\x94\x82 \x1b[22m lazy continuation") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[2m\xe2\x94\x82 \x1b[22m- list item") == null);
}
test "reset clears an unfinished indented code block" {
@@ -3221,7 +3222,7 @@ test "reset clears an unfinished indented code block" {
}
test "code block payload clone frees language if code allocation fails" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
var source = CodeBlockPayload{
.language = try alloc.dupe(u8, "zig"),
.code = try alloc.dupe(u8, "const value = 1;\n"),
@@ -3390,7 +3391,7 @@ test "emphasis lookahead agrees with links, code spans, and bare URLs" {
// URL still closes the span around it.
out.clearRetainingCapacity();
try processor.push(alloc, "https://example.com/`x*y` **bold** `code`\n", &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, ";https://example.com/`x*y`\x1b\\") != null);
try std.testing.expect(std.mem.find(u8, out.items, ";https://example.com/`x*y`\x1b\\") != null);
try std.testing.expect(std.mem.endsWith(u8, out.items, " \x1b[1mbold\x1b[22m \x1b[38;5;245mcode\x1b[39m\n"));
// A trailing delimiter is trimmed from the URL and closes the span.
out.clearRetainingCapacity();
@@ -3407,7 +3408,7 @@ test "emphasis lookahead agrees with links, code spans, and bare URLs" {
out.clearRetainingCapacity();
try processor.push(alloc, "*https://example.com/a*`some code` **bold** `x`\n", &out);
try std.testing.expect(std.mem.startsWith(u8, out.items, "*\x1b]8;"));
try std.testing.expect(std.mem.indexOf(u8, out.items, ";https://example.com/a*`some\x1b\\") != null);
try std.testing.expect(std.mem.find(u8, out.items, ";https://example.com/a*`some\x1b\\") != null);
try std.testing.expect(std.mem.endsWith(u8, out.items, " code\x1b[38;5;245m**bold**\x1b[39mx`\n"));
out.clearRetainingCapacity();
try processor.push(alloc, "text **** https://example.com\n", &out);
@@ -3490,7 +3491,7 @@ test "long lines of unmatched or unbalanced delimiters render in linear time" {
const residual_started = io_mod.nanoTimestamp();
try processor.push(alloc, line.items, &out);
try std.testing.expect(@divTrunc(io_mod.nanoTimestamp() - residual_started, std.time.ns_per_ms) < 500);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[3mb c\x1b[23m_ ") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[3mb c\x1b[23m_ ") != null);
for (shapes) |shape| {
out.clearRetainingCapacity();
line.clearRetainingCapacity();
@@ -3563,8 +3564,8 @@ test "single-column pipe table has no junction on separator" {
try processor.push(alloc, input, &out);
try processor.flush(alloc, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\xbc") == null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\xbc") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80") != null);
}
test "pipe table with markdown in cells aligns dividers across all rows" {
@@ -3622,8 +3623,8 @@ test "three-column pipe table aligns junctions with every column pipe" {
const expected_separator =
"\xe2\x94\x80\xe2\x94\x80\xe2\x94\xbc\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\xbc\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\n";
try std.testing.expect(std.mem.indexOf(u8, out.items, expected_separator) != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "1 \xe2\x94\x82 22 \xe2\x94\x82 333\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, expected_separator) != null);
try std.testing.expect(std.mem.find(u8, out.items, "1 \xe2\x94\x82 22 \xe2\x94\x82 333\n") != null);
}
test "pipe table without separator falls back to plain lines" {
@@ -3673,9 +3674,9 @@ test "borderless GFM table (no leading/trailing pipes) is detected and rendered"
try processor.push(alloc, input, &out);
try processor.flush(alloc, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\x82") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\xbc") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\x1b[1mName\x1b[22m") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\x82") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\xbc") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\x1b[1mName\x1b[22m") != null);
}
test "borderless separator with spaces around pipe is accepted" {
@@ -3692,8 +3693,8 @@ test "borderless separator with spaces around pipe is accepted" {
try processor.push(alloc, input, &out);
try processor.flush(alloc, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\x82") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\xbc") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\x82") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\xbc") != null);
}
test "paragraph with single inline pipe falls back to plain (no separator)" {
@@ -3708,9 +3709,9 @@ test "paragraph with single inline pipe falls back to plain (no separator)" {
"Then review the output.\n";
try processor.push(alloc, input, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\x82") == null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "Use cmd | grep foo") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "Then review the output.") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\x82") == null);
try std.testing.expect(std.mem.find(u8, out.items, "Use cmd | grep foo") != null);
try std.testing.expect(std.mem.find(u8, out.items, "Then review the output.") != null);
}
test "header with nested **bold** stays fully bold across the whole line" {
@@ -3823,8 +3824,8 @@ test "horizontal rule with asterisks and spaces also renders" {
defer out.deinit(alloc);
try processor.push(alloc, "* * *\n", &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, table_horiz) != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "*") == null);
try std.testing.expect(std.mem.find(u8, out.items, table_horiz) != null);
try std.testing.expect(std.mem.find(u8, out.items, "*") == null);
}
test "hyphen-space line is a list, not a rule" {
@@ -3835,8 +3836,8 @@ test "hyphen-space line is a list, not a rule" {
defer out.deinit(alloc);
try processor.push(alloc, "- item\n", &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x80\xa2") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "\xe2\x94\x80") == null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x80\xa2") != null);
try std.testing.expect(std.mem.find(u8, out.items, "\xe2\x94\x80") == null);
}
test "nested unordered list preserves indent" {
@@ -3875,7 +3876,7 @@ test "right-aligned GFM column pads cell on the left" {
try processor.push(alloc, input, &out);
try processor.flush(alloc, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, " 5\n") != null);
try std.testing.expect(std.mem.find(u8, out.items, " 5\n") != null);
}
test "center-aligned GFM column pads both sides" {
@@ -3892,7 +3893,7 @@ test "center-aligned GFM column pads both sides" {
try processor.push(alloc, input, &out);
try processor.flush(alloc, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, " ok \n") != null);
try std.testing.expect(std.mem.find(u8, out.items, " ok \n") != null);
}
test "pipe table parsing retains styled cells alignment and ragged rows" {
@@ -3930,6 +3931,6 @@ test "pipe table inside code block stays literal" {
"```\n";
try processor.push(alloc, input, &out);
try std.testing.expect(std.mem.indexOf(u8, out.items, "| not | a |") != null);
try std.testing.expect(std.mem.indexOf(u8, out.items, "| table | really |") != null);
try std.testing.expect(std.mem.find(u8, out.items, "| not | a |") != null);
try std.testing.expect(std.mem.find(u8, out.items, "| table | really |") != null);
}
src/core/agent/execution_memory.zig
+8 -7
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../shared/testing_allocator.zig");
const debug_trace = @import("../shared/debug_trace.zig");
const io_mod = @import("../shared/io.zig");
const message = @import("../shared/message.zig");
@@ -818,7 +819,7 @@ fn durableIdentifier(alloc: Allocator, value: []const u8) ![]u8 {
var digest: [32]u8 = undefined;
std.crypto.hash.sha2.Sha256.hash(value, &digest, .{});
const hex = std.fmt.bytesToHex(digest[0..12].*, .lower);
return std.fmt.allocPrint(alloc, "redacted-{s}", .{&hex});
return alloc.print("redacted-{s}", .{&hex});
}
test "execution memory persists secret-bearing arguments results and provider output verbatim" {
@@ -1103,7 +1104,7 @@ test "durable execution memory preserves committed file presentation verbatim an
}
test "file evidence does not parse unknown tool arguments" {
var failing_allocator = std.testing.FailingAllocator.init(std.testing.allocator, .{});
var failing_allocator = std.testing.FailingAllocator.init(testing_allocator.no_resize, .{});
const alloc = failing_allocator.allocator();
var files: std.ArrayList(types.FileEvidence) = .empty;
defer files.deinit(alloc);
@@ -1114,7 +1115,7 @@ test "file evidence does not parse unknown tool arguments" {
.{
.id = "large_command",
.name = "run_command",
.arguments_json = "{\"command\":\"" ++ ("x\\n" ** 20_000) ++ "\"}",
.arguments_json = "{\"command\":\"" ++ text_utils.repeat("x\\n", 20_000) ++ "\"}",
},
.success,
null,
@@ -1609,12 +1610,12 @@ test "normal execution-memory builders clean every allocation failure" {
};
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
ChatCheck.run,
.{},
);
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
MessageCheck.run,
.{},
);
@@ -1632,11 +1633,11 @@ test "normal execution-memory builders skip non-tool messages without allocation
message.Message.assistantBorrowed("ignored", &.{}),
};
var chat_failing = std.testing.FailingAllocator.init(
std.testing.allocator,
testing_allocator.no_resize,
.{ .fail_index = 0 },
);
var message_failing = std.testing.FailingAllocator.init(
std.testing.allocator,
testing_allocator.no_resize,
.{ .fail_index = 0 },
);
src/core/agent/presentation/block_parse.zig
+3 -3
View File
@@ -44,7 +44,7 @@ fn isCodeFence(line: []const u8) bool {
pub fn codeFenceLanguage(line: []const u8) []const u8 {
const fence = parseCodeFence(line) orelse return "";
const info = std.mem.trim(u8, line[fence.indent + fence.run ..], " \t");
const end = std.mem.indexOfAny(u8, info, " \t") orelse info.len;
const end = std.mem.findAny(u8, info, " \t") orelse info.len;
return info[0..end];
}
@@ -136,7 +136,7 @@ pub const ParsedFootnoteDefinition = struct {
pub fn parseFootnoteDefinition(line: []const u8) ?ParsedFootnoteDefinition {
if (line.len < 6 or line[0] != '[' or line[1] != '^') return null;
const close = std.mem.indexOfScalarPos(u8, line, 2, ']') orelse return null;
const close = std.mem.findScalarPos(u8, line, 2, ']') orelse return null;
if (close == 2 or close + 1 >= line.len or line[close + 1] != ':') return null;
var body_start = close + 2;
@@ -307,7 +307,7 @@ pub fn isValidTable(buf: []const u8) bool {
var saw_separator = false;
var start: usize = 0;
while (start < buf.len) {
const end = std.mem.indexOfScalarPos(u8, buf, start, '\n') orelse buf.len;
const end = std.mem.findScalarPos(u8, buf, start, '\n') orelse buf.len;
const line = buf[start..end];
if (line_count == 1 and isSeparatorLine(line)) saw_separator = true;
line_count += 1;
src/core/agent/presentation/block_render.zig
+7 -7
View File
@@ -201,7 +201,7 @@ fn parseAndRenderRows(
var line_idx: usize = 0;
var start: usize = 0;
while (start < buf.len) {
const end = std.mem.indexOfScalarPos(u8, buf, start, '\n') orelse buf.len;
const end = std.mem.findScalarPos(u8, buf, start, '\n') orelse buf.len;
defer {
line_idx += 1;
start = end + 1;
@@ -316,7 +316,7 @@ fn writeTableRow(
pub fn writeTableHeaderCell(alloc: Allocator, out: *std.ArrayList(u8), cell: []const u8) !void {
try out.appendSlice(alloc, ansi.bold_open);
var remaining = cell;
while (std.mem.indexOf(u8, remaining, ansi.bold_close)) |close_index| {
while (std.mem.find(u8, remaining, ansi.bold_close)) |close_index| {
try out.appendSlice(alloc, remaining[0..close_index]);
try out.appendSlice(alloc, ansi.bold_close);
try out.appendSlice(alloc, ansi.bold_open);
@@ -435,7 +435,7 @@ pub fn parseTablePayloadWithFootnotes(
var line_idx: usize = 0;
var start: usize = 0;
while (start < buf.len) {
const end = std.mem.indexOfScalarPos(u8, buf, start, '\n') orelse buf.len;
const end = std.mem.findScalarPos(u8, buf, start, '\n') orelse buf.len;
defer {
line_idx += 1;
start = end + 1;
@@ -493,7 +493,7 @@ fn deinitRenderedRow(alloc: Allocator, row: *RenderedRow) void {
pub fn writeFootnoteDefinitionMarker(alloc: Allocator, out: *std.ArrayList(u8), number: usize) !void {
var marker: [32]u8 = undefined;
const bytes = try std.fmt.bufPrint(&marker, "[{d}] ", .{number});
const bytes = try std.mem.print(&marker, "[{d}] ", .{number});
try ansi.writeDim(alloc, out, bytes);
}
@@ -508,11 +508,11 @@ pub fn writeFootnoteBody(
var start: usize = 0;
var is_first = true;
while (start <= body.len) {
const end = std.mem.indexOfScalarPos(u8, body, start, '\n') orelse body.len;
const end = std.mem.findScalarPos(u8, body, start, '\n') orelse body.len;
if (!is_first) {
try out.append(alloc, '\n');
var marker: [32]u8 = undefined;
const marker_bytes = try std.fmt.bufPrint(&marker, "[{d}] ", .{number});
const marker_bytes = try std.mem.print(&marker, "[{d}] ", .{number});
@memset(marker[0..marker_bytes.len], ' ');
try ansi.writeDim(alloc, out, marker[0..marker_bytes.len]);
}
@@ -531,7 +531,7 @@ pub fn renderCodeBlockPayload(
) !void {
var start: usize = 0;
while (start < block.code.len) {
const end = std.mem.indexOfScalarPos(u8, block.code, start, '\n') orelse block.code.len;
const end = std.mem.findScalarPos(u8, block.code, start, '\n') orelse block.code.len;
try ansi.writeDim(alloc, out, ansi.vertical_rule_prefix);
try out.appendSlice(alloc, block.code[start..end]);
try out.append(alloc, '\n');
src/core/agent/presentation/code_highlight.zig
+67 -67
View File
@@ -300,7 +300,7 @@ fn blockCommentEnd(source: []const u8, index: usize, block_comment: ?languages.B
const comment = block_comment orelse return null;
if (!std.mem.startsWith(u8, source[index..], comment.start)) return null;
const content_start = index + comment.start.len;
const close_start = std.mem.indexOfPos(u8, source, content_start, comment.end) orelse return source.len;
const close_start = std.mem.findPos(u8, source, content_start, comment.end) orelse return source.len;
return close_start + comment.end.len;
}
@@ -312,11 +312,11 @@ fn lineCommentEnd(source: []const u8, index: usize, prefixes: []const []const u8
}
fn lineEnd(source: []const u8, start: usize) usize {
return std.mem.indexOfScalarPos(u8, source, start, '\n') orelse source.len;
return std.mem.findScalarPos(u8, source, start, '\n') orelse source.len;
}
fn isQuote(byte: u8, quotes: []const u8) bool {
return std.mem.indexOfScalar(u8, quotes, byte) != null;
return std.mem.findScalar(u8, quotes, byte) != null;
}
fn quotedEnd(source: []const u8, start: usize) usize {
@@ -476,7 +476,7 @@ fn stripAnsi(alloc: std.mem.Allocator, text: []const u8) ![]u8 {
fn count(text: []const u8, needle: []const u8) usize {
var result: usize = 0;
var start: usize = 0;
while (std.mem.indexOfPos(u8, text, start, needle)) |index| {
while (std.mem.findPos(u8, text, start, needle)) |index| {
result += 1;
start = index + needle.len;
}
@@ -492,12 +492,12 @@ test "supported source gains balanced colors without changing code bytes" {
const plain = try stripAnsi(alloc, styled);
defer alloc.free(plain);
try std.testing.expectEqualStrings(source, plain);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;") != null);
try std.testing.expectEqual(count(styled, "\x1b[38;5;"), count(styled, "\x1b[39m"));
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252mconst") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252mconst") != null);
try std.testing.expectEqual(@as(usize, 1), count(styled, "\x1b[38;5;252mconst"));
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m\"const\"\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252mreturn") == null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m\"const\"\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252mreturn") == null);
}
test "light theme uses a readable syntax palette without changing code bytes" {
@@ -509,9 +509,9 @@ test "light theme uses a readable syntax palette without changing code bytes" {
const plain = try stripAnsi(alloc, styled);
defer alloc.free(plain);
try std.testing.expectEqualStrings(source, plain);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;238mconst\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;241m\"ready\"\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;243m// comment\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;238mconst\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;241m\"ready\"\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;243m// comment\x1b[39m") != null);
try std.testing.expectEqual(count(styled, "\x1b[38;5;"), count(styled, "\x1b[39m"));
}
@@ -554,7 +554,7 @@ test "every registered profile highlights representative source" {
const plain = try stripAnsi(alloc, styled);
defer alloc.free(plain);
try std.testing.expectEqualStrings(case.source, plain);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;") != null);
}
}
@@ -568,9 +568,9 @@ test "profiles use configured block comments and case-insensitive keywords" {
const html = try highlight(alloc, source[35..], languages.resolve("html").?, .dark, null);
defer alloc.free(html);
try std.testing.expect(std.mem.indexOf(u8, css, "\x1b[38;5;245m/* comment */\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, sql, "\x1b[38;5;252mSELECT\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, html, "\x1b[38;5;245m<!-- note -->\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, css, "\x1b[38;5;245m/* comment */\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, sql, "\x1b[38;5;252mSELECT\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, html, "\x1b[38;5;245m<!-- note -->\x1b[39m") != null);
}
test "themed attribute slots close fully without bleeding into later text" {
@@ -587,8 +587,8 @@ test "themed attribute slots close fully without bleeding into later text" {
defer alloc.free(styled);
// Italic comment and bold keyword each close with their attributes reset.
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[3;38;2;106;153;85m// note\x1b[39m\x1b[23m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[1;38;2;130;210;206mconst\x1b[39m\x1b[22m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[3;38;2;106;153;85m// note\x1b[39m\x1b[23m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[1;38;2;130;210;206mconst\x1b[39m\x1b[22m") != null);
// Nothing stays bold or italic past the final close.
try std.testing.expect(!std.mem.endsWith(u8, styled, "\x1b[3;38;2;106;153;85m"));
}
@@ -600,7 +600,7 @@ test "base style wraps the span and restores after each token" {
// The span opens with the base, and every token close re-establishes it.
try std.testing.expect(std.mem.startsWith(u8, styled, "<base>\x1b[38;5;252mecho\x1b[39m<base> "));
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m'hi there'\x1b[39m<base>") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m'hi there'\x1b[39m<base>") != null);
// Bare number arguments stay plain in shell.
try std.testing.expect(std.mem.endsWith(u8, styled, "<base> 42"));
}
@@ -628,20 +628,20 @@ test "shell operators and variables take the keyword color" {
const styled = try highlight(alloc, "cd /tmp && echo $HOME | head -2 > out; echo $? # done", languages.resolve("sh").?, .dark, null);
defer alloc.free(styled);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m&&\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m|\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m>\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m;\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m$HOME\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m$?\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m&&\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m|\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m>\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m;\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m$HOME\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m$?\x1b[39m") != null);
// Comments keep their color, flags take the number color as a unit, and
// a $ inside quotes stays string.
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;245m# done\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m-2\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;245m# done\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m-2\x1b[39m") != null);
const quoted = try highlight(alloc, "echo '$HOME'", languages.resolve("sh").?, .dark, null);
defer alloc.free(quoted);
try std.testing.expect(std.mem.indexOf(u8, quoted, "\x1b[38;5;250m'$HOME'\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, quoted, "\x1b[38;5;250m'$HOME'\x1b[39m") != null);
// Other languages do not pick up shell operators.
const zig_src = try highlight(alloc, "a < b", languages.resolve("zig").?, .dark, null);
@@ -656,11 +656,11 @@ test "digit runs glued to words by a dash stay plain" {
// The date suffix in the path is a name segment and keeps the plain text,
// as does the literal-looking "null" in /dev/null.
try std.testing.expect(std.mem.indexOf(u8, styled, "build-20260918") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "/dev/null") != null);
try std.testing.expect(std.mem.find(u8, styled, "build-20260918") != null);
try std.testing.expect(std.mem.find(u8, styled, "/dev/null") != null);
// The numeric flag colors as a unit, and the redirect fd still colors.
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m-80\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m2\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m-80\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m2\x1b[39m") != null);
}
test "dash flags color as units only at word boundaries" {
@@ -668,16 +668,16 @@ test "dash flags color as units only at word boundaries" {
const styled = try highlight(alloc, "tail -8 --json && cat - < in", languages.resolve("sh").?, .dark, null);
defer alloc.free(styled);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m-8\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m--json\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m-8\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m--json\x1b[39m") != null);
// A lone dash (stdin marker) stays plain between the verb and the
// redirect, and the redirect target is an argument, not a keyword.
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252mcat\x1b[39m - \x1b[38;5;252m<\x1b[39m in") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252mcat\x1b[39m - \x1b[38;5;252m<\x1b[39m in") != null);
// Flags after operators still count as boundaries.
const after_pipe = try highlight(alloc, "echo x | head -1", languages.resolve("sh").?, .dark, null);
defer alloc.free(after_pipe);
try std.testing.expect(std.mem.indexOf(u8, after_pipe, "\x1b[38;5;250m-1\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, after_pipe, "\x1b[38;5;250m-1\x1b[39m") != null);
// Other languages keep minus signs plain.
const zig_src = try highlight(alloc, "a - b", languages.resolve("zig").?, .dark, null);
@@ -693,18 +693,18 @@ test "command position colors any command word and only command words" {
// Unknown binaries color in command position, matching the bash grammar's
// variable.function; a builtin used as an argument stays plain, and the
// redirect target is a path, not a command.
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252mgh\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252mxargs\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252mgh\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252mxargs\x1b[39m") != null);
// echo is an argument here and stays plain; the redirect target too.
try std.testing.expect(std.mem.indexOf(u8, styled, " echo \x1b[38;5;252m>\x1b[39m out.txt") != null);
try std.testing.expect(std.mem.find(u8, styled, " echo \x1b[38;5;252m>\x1b[39m out.txt") != null);
// Control keywords hand command position to the command they govern.
const chain = try highlight(alloc, "if cd /x; then echo hi; fi", languages.resolve("sh").?, .dark, null);
defer alloc.free(chain);
for ([_][]const u8{ "if", "cd", "then", "echo", "fi" }) |word| {
const wrapped = try std.fmt.allocPrint(alloc, "\x1b[38;5;252m{s}\x1b[39m", .{word});
const wrapped = try alloc.print("\x1b[38;5;252m{s}\x1b[39m", .{word});
defer alloc.free(wrapped);
try std.testing.expect(std.mem.indexOf(u8, chain, wrapped) != null);
try std.testing.expect(std.mem.find(u8, chain, wrapped) != null);
}
}
@@ -713,10 +713,10 @@ test "bare number arguments stay plain but redirect fds color" {
const styled = try highlight(alloc, "sleep 5; exit 7 2>&1", languages.resolve("sh").?, .dark, null);
defer alloc.free(styled);
try std.testing.expect(std.mem.indexOf(u8, styled, " 5") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, " 7 ") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m2\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m1\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, " 5") != null);
try std.testing.expect(std.mem.find(u8, styled, " 7 ") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m2\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m1\x1b[39m") != null);
}
test "braced variables tildes globs and substitution parse like the grammar" {
@@ -724,18 +724,18 @@ test "braced variables tildes globs and substitution parse like the grammar" {
const styled = try highlight(alloc, "cp ${SRC}/*.log ~/out && echo $(date +%F)", languages.resolve("sh").?, .dark, null);
defer alloc.free(styled);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m${SRC}\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m*\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m~\x1b[39m/out") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m$(\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m${SRC}\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m*\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m~\x1b[39m/out") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m$(\x1b[39m") != null);
// The substitution contents open in command position.
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252mdate\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252mdate\x1b[39m") != null);
// Backtick contents parse as code rather than one flat string.
const ticks = try highlight(alloc, "echo `uname -s`", languages.resolve("sh").?, .dark, null);
defer alloc.free(ticks);
try std.testing.expect(std.mem.indexOf(u8, ticks, "`\x1b[38;5;252muname\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, ticks, "\x1b[38;5;250m-s\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, ticks, "`\x1b[38;5;252muname\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, ticks, "\x1b[38;5;250m-s\x1b[39m") != null);
}
test "double quotes interpolate variables inside the string color" {
@@ -743,13 +743,13 @@ test "double quotes interpolate variables inside the string color" {
const styled = try highlight(alloc, "echo \"hi $USER from ${HOME}\"", languages.resolve("sh").?, .dark, null);
defer alloc.free(styled);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;250m\"hi \x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m$USER\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m${HOME}\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;250m\"hi \x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m$USER\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m${HOME}\x1b[39m") != null);
// Single quotes do not interpolate.
const single = try highlight(alloc, "echo '$USER'", languages.resolve("sh").?, .dark, null);
defer alloc.free(single);
try std.testing.expect(std.mem.indexOf(u8, single, "\x1b[38;5;250m'$USER'\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, single, "\x1b[38;5;250m'$USER'\x1b[39m") != null);
}
test "diff lines paint with the caller's marker colors" {
@@ -758,12 +758,12 @@ test "diff lines paint with the caller's marker colors" {
const styled = try highlightDiff(alloc, patch, .dark, "<added>", "<removed>");
defer alloc.free(styled);
try std.testing.expect(std.mem.indexOf(u8, styled, "<added>+new line") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "<removed>-old line") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;252m@@ -1,2 +1,2 @@\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;245m--- a/f\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;245mdiff --git a/f b/f\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\n context") != null);
try std.testing.expect(std.mem.find(u8, styled, "<added>+new line") != null);
try std.testing.expect(std.mem.find(u8, styled, "<removed>-old line") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;252m@@ -1,2 +1,2 @@\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;245m--- a/f\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;245mdiff --git a/f b/f\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\n context") != null);
var no_syntax = shared_theme.fx_dark;
no_syntax.syntax.enabled = false;
@@ -783,7 +783,7 @@ test "text blocks stay byte-identical and markdown colors inline code" {
const md = try highlight(alloc, "run `fx upgrade` to update", languages.resolve("md").?, .dark, null);
defer alloc.free(md);
try std.testing.expect(std.mem.indexOf(u8, md, "\x1b[38;5;250m`fx upgrade`\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, md, "\x1b[38;5;250m`fx upgrade`\x1b[39m") != null);
}
test "split slots let themes color commands variables and operators apart" {
@@ -803,9 +803,9 @@ test "split slots let themes color commands variables and operators apart" {
// Control keywords keep the keyword color; command words take the
// function color; variables and operators take their own.
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;204mwhile\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;201mtrue\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;201mecho\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;202m$HOME\x1b[39m") != null);
try std.testing.expect(std.mem.indexOf(u8, styled, "\x1b[38;5;203m|\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;204mwhile\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;201mtrue\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;201mecho\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;202m$HOME\x1b[39m") != null);
try std.testing.expect(std.mem.find(u8, styled, "\x1b[38;5;203m|\x1b[39m") != null);
}
src/core/agent/presentation/code_highlight_languages.zig
+5 -5
View File
@@ -478,12 +478,12 @@ fn matchesDetection(alloc: Allocator, detection: Detection, source: []const u8)
fn matchesDiffPatch(source: []const u8) bool {
const line = firstNonblankLine(source);
if (std.mem.startsWith(u8, line, "diff --git ") or std.mem.startsWith(u8, line, "@@ ")) return true;
return std.mem.startsWith(u8, line, "--- ") and std.mem.indexOf(u8, source, "\n+++ ") != null;
return std.mem.startsWith(u8, line, "--- ") and std.mem.find(u8, source, "\n+++ ") != null;
}
fn matchesTypeScriptAssertion(source: []const u8) bool {
var start: usize = 0;
while (std.mem.indexOfPos(u8, source, start, "} as ")) |assertion_start| {
while (std.mem.findPos(u8, source, start, "} as ")) |assertion_start| {
const type_start = assertion_start + "} as ".len;
if (type_start < source.len and std.ascii.isUpper(source[type_start])) return true;
start = type_start;
@@ -502,7 +502,7 @@ fn isValidJson(alloc: Allocator, source: []const u8) bool {
fn matchesShellShebang(source: []const u8) bool {
const line = firstNonblankLine(source);
return std.mem.startsWith(u8, line, "#!") and
(std.mem.indexOf(u8, line, "bash") != null or std.mem.indexOf(u8, line, "zsh") != null or std.mem.indexOf(u8, line, "/sh") != null);
(std.mem.find(u8, line, "bash") != null or std.mem.find(u8, line, "zsh") != null or std.mem.find(u8, line, "/sh") != null);
}
fn matchesPythonHeader(source: []const u8) bool {
@@ -518,7 +518,7 @@ fn matchesSqlSelect(source: []const u8) bool {
fn matchesRustFunction(source: []const u8) bool {
const line = firstNonblankLine(source);
if (!std.mem.startsWith(u8, line, "fn ") and !std.mem.startsWith(u8, line, "pub fn ")) return false;
return std.mem.indexOf(u8, source, "let ") != null or std.mem.indexOf(u8, source, "println!") != null or std.mem.indexOf(u8, line, "->") != null;
return std.mem.find(u8, source, "let ") != null or std.mem.find(u8, source, "println!") != null or std.mem.find(u8, line, "->") != null;
}
fn firstNonblankLine(source: []const u8) []const u8 {
@@ -533,7 +533,7 @@ fn firstNonblankLine(source: []const u8) []const u8 {
fn containsLineStart(source: []const u8, prefix: []const u8) bool {
if (std.mem.startsWith(u8, source, prefix)) return true;
var start: usize = 0;
while (std.mem.indexOfPos(u8, source, start, "\n")) |newline| {
while (std.mem.findPos(u8, source, start, "\n")) |newline| {
const line_start = newline + 1;
if (std.mem.startsWith(u8, source[line_start..], prefix)) return true;
start = line_start;
src/core/agent/presentation/inline_render.zig
+23 -11
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const display_width = @import("../../shared/display_width.zig");
const Allocator = std.mem.Allocator;
const ansi = @import("ansi.zig");
const tu = @import("text_util.zig");
@@ -119,7 +120,7 @@ fn tokenize(
// is not followed by `(` cannot be a link or image, and a footnote label
// ends there too. Without this every `[` in a long run rescans to the
// same `]`.
const last_close_bracket = std.mem.lastIndexOfScalar(u8, text, ']');
const last_close_bracket = std.mem.findScalarLast(u8, text, ']');
var next_close_bracket: ?usize = null;
// End of the most recent delimiter run that can open emphasis. A bare URL
// may follow such a run directly, and the tokenizer owns that decision
@@ -130,7 +131,7 @@ fn tokenize(
const c = text[i];
const after_opening_delimiter = i > 0 and opening_delimiter_end == i;
if (last_close_bracket != null and i <= last_close_bracket.? and (next_close_bracket == null or next_close_bracket.? < i)) {
next_close_bracket = std.mem.indexOfScalarPos(u8, text, i, ']');
next_close_bracket = std.mem.findScalarPos(u8, text, i, ']');
}
const bracket_link_possible = next_close_bracket != null and next_close_bracket.? >= i and
next_close_bracket.? + 1 < text.len and text[next_close_bracket.? + 1] == '(';
@@ -320,7 +321,7 @@ fn codepointBefore(text: []const u8, index: usize) u21 {
var start = index - 1;
var steps: usize = 0;
while (start > 0 and steps < 3 and text[start] & 0xC0 == 0x80) : (steps += 1) start -= 1;
return std.unicode.utf8Decode(text[start..index]) catch text[index - 1];
return display_width.decodeUtf8Sequence(text[start..index]) catch text[index - 1];
}
/// Code point starting at `index`, or a space at the line end.
@@ -328,7 +329,7 @@ fn codepointAt(text: []const u8, index: usize) u21 {
if (index >= text.len) return ' ';
const len = std.unicode.utf8ByteSequenceLength(text[index]) catch return text[index];
if (index + len > text.len) return text[index];
return std.unicode.utf8Decode(text[index .. index + len]) catch text[index];
return display_width.decodeUtf8Sequence(text[index .. index + len]) catch text[index];
}
fn isFlankingWhitespace(cp: u21) bool {
@@ -353,7 +354,7 @@ fn matchDelimiters(alloc: Allocator, tokens: []Token, matches: *std.ArrayList(Ma
var next_seq: u32 = 1;
// Sequence number of the top entry when a search for this closer kind
// last failed; entries at or below it stay unusable for that kind.
var openers_bottom: [3][2][3]u32 = .{.{.{ 0, 0, 0 }} ** 2} ** 3;
var openers_bottom: [3][2][3]u32 = @splat(@splat(@splat(0)));
for (tokens, 0..) |*token, token_index| {
const closer = switch (token.*) {
@@ -445,7 +446,7 @@ fn emitTokens(
const id = link_id.*;
link_id.* +%= 1;
var id_buf: [32]u8 = undefined;
const open = try std.fmt.bufPrint(&id_buf, "\x1b]8;id=fx-{d};", .{id});
const open = try std.mem.print(&id_buf, "\x1b]8;id=fx-{d};", .{id});
try out.appendSlice(alloc, open);
try out.appendSlice(alloc, content[0..url_end]);
try out.appendSlice(alloc, "\x1b\\");
@@ -474,7 +475,7 @@ fn emitTokens(
}
fn emitStyle(alloc: Allocator, out: *std.ArrayList(u8), style: Style, opening: bool, depth: *[3]usize, options: InlineOptions) !void {
const slot = @intFromEnum(style);
const slot = @backingInt(style);
const open_seq: []const u8 = switch (style) {
.bold => ansi.bold_open,
.italic => ansi.italic_open,
@@ -514,7 +515,7 @@ fn parseFootnoteReference(text: []const u8, start: usize, close: usize) ?ParsedF
fn writeFootnoteMarker(alloc: Allocator, out: *std.ArrayList(u8), number: usize) !void {
var marker: [32]u8 = undefined;
const bytes = try std.fmt.bufPrint(&marker, "[{d}]", .{number});
const bytes = try std.mem.print(&marker, "[{d}]", .{number});
try ansi.writeDim(alloc, out, bytes);
}
@@ -537,7 +538,7 @@ fn emitInlineLink(
const id = link_id.*;
link_id.* +%= 1;
var id_buf: [32]u8 = undefined;
const open = std.fmt.bufPrint(&id_buf, "\x1b]8;id=fx-{d};", .{id}) catch unreachable;
const open = try std.mem.print(&id_buf, "\x1b]8;id=fx-{d};", .{id});
try out.appendSlice(alloc, open);
try out.appendSlice(alloc, link.destination_prefix);
try out.appendSlice(alloc, link.url);
@@ -600,7 +601,7 @@ fn parseLinkDestination(text: []const u8, start: usize) ?LinkDestination {
var url: []const u8 = undefined;
if (text[k] == '<') {
const close = std.mem.indexOfScalarPos(u8, text, k + 1, '>') orelse return null;
const close = std.mem.findScalarPos(u8, text, k + 1, '>') orelse return null;
url = text[k + 1 .. close];
for (url) |byte| if (byte == '<' or byte == '\n') return null;
k = close + 1;
@@ -723,7 +724,7 @@ fn decodeEntity(text: []const u8, start: usize) ?DecodedEntity {
if (start >= text.len or text[start] != '&') return null;
// Bound the terminator search so a line full of ampersands stays linear.
const window_end = @min(text.len, start + 1 + max_entity_name_len + 1);
const semicolon = std.mem.indexOfScalarPos(u8, text[0..window_end], start + 1, ';') orelse return null;
const semicolon = std.mem.findScalarPos(u8, text[0..window_end], start + 1, ';') orelse return null;
const name = text[start + 1 .. semicolon];
if (name.len == 0) return null;
@@ -941,3 +942,14 @@ fn isBareUrlTerminator(c: u8) bool {
fn isTrailingBareUrlByte(c: u8) bool {
return tu.isTrailingUrlPunctuation(c) or c == '*' or c == '_' or c == '~';
}
test "emphasis after a truncated UTF-8 sequence renders instead of panicking" {
const alloc = std.testing.allocator;
var out: std.ArrayList(u8) = .empty;
defer out.deinit(alloc);
var link_id: u32 = 0;
// The delimiter follows a three-byte lead with only one continuation
// byte, so the code point before the run has a mismatched length.
try writeInline(alloc, "\xe4\x80*bold*", &out, false, null, &link_id);
try std.testing.expect(std.mem.find(u8, out.items, "bold") != null);
}
src/core/agent/presentation/text_util.zig
+1 -1
View File
@@ -79,7 +79,7 @@ pub fn nthLine(buf: []const u8, n: usize) ?[]const u8 {
var idx: usize = 0;
var start: usize = 0;
while (start < buf.len) {
const end = std.mem.indexOfScalarPos(u8, buf, start, '\n') orelse buf.len;
const end = std.mem.findScalarPos(u8, buf, start, '\n') orelse buf.len;
if (idx == n) return buf[start..end];
idx += 1;
start = end + 1;
src/core/agent/question_answer.zig
+2 -1
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../shared/testing_allocator.zig");
const types = @import("../shared/types.zig");
const Allocator = std.mem.Allocator;
@@ -115,7 +116,7 @@ test "question answer codec exposes structural and allocation failures" {
);
var failing = std.testing.FailingAllocator.init(
std.testing.allocator,
testing_allocator.no_resize,
.{ .fail_index = 0 },
);
try std.testing.expectError(
src/core/agent/question_prompt.zig
+8 -7
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../shared/testing_allocator.zig");
const debug_trace = @import("../shared/debug_trace.zig");
const edit_contract = @import("../input/editor_state.zig");
const io_mod = @import("../shared/io.zig");
@@ -597,7 +598,7 @@ pub const QuestionPrompt = struct {
if (entry.freeform_cursor == 0) return;
const start = text_boundaries.previousCharacterStart(entry.freeform_buffer.items, entry.freeform_cursor);
const count = entry.freeform_cursor - start;
std.mem.copyForwards(u8, entry.freeform_buffer.items[start..], entry.freeform_buffer.items[entry.freeform_cursor..]);
@memmove(entry.freeform_buffer.items[start .. entry.freeform_buffer.items.len - count], entry.freeform_buffer.items[entry.freeform_cursor..]);
entry.freeform_buffer.items.len -= count;
entry.freeform_cursor = start;
}
@@ -815,7 +816,7 @@ fn checkQuestionPromptSyncAllocationFailures(alloc: Allocator) !void {
test "question prompt frees partial owned-entry allocations" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkQuestionPromptSyncAllocationFailures,
.{},
);
@@ -828,7 +829,7 @@ test "question prompt freeform allocation failure preserves prompt state" {
prompt.moveChoice(-1);
var failing = std.testing.FailingAllocator.init(
std.testing.allocator,
testing_allocator.no_resize,
.{ .fail_index = 0 },
);
try std.testing.expectError(
@@ -849,7 +850,7 @@ test "question prompt submission allocation failure preserves the draft" {
prompt.applyFreeformCursorMove(1, 7);
var failing = std.testing.FailingAllocator.init(
std.testing.allocator,
testing_allocator.no_resize,
.{ .fail_index = 0 },
);
try std.testing.expectError(
@@ -1539,7 +1540,7 @@ test "question prompt traces discarded drafts with their reason" {
defer tmp.cleanup();
const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, ".");
defer alloc.free(root);
const trace_path = try std.fs.path.join(alloc, &.{ root, "question-draft.log" });
const trace_path = try std.Io.Dir.path.join(alloc, &.{ root, "question-draft.log" });
defer alloc.free(trace_path);
debug_trace.resetForTest();
@@ -1571,7 +1572,7 @@ test "question prompt traces drafts that differ from accepted submissions" {
defer tmp.cleanup();
const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, ".");
defer alloc.free(root);
const trace_path = try std.fs.path.join(alloc, &.{ root, "question-draft-accepted.log" });
const trace_path = try std.Io.Dir.path.join(alloc, &.{ root, "question-draft-accepted.log" });
defer alloc.free(trace_path);
debug_trace.resetForTest();
@@ -1681,7 +1682,7 @@ test "compact submission keeps input distinct from actions and choice prompts cl
}
test "compact submission allocation failure preserves the unanswered input" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
var prompt: QuestionPrompt = .{};
defer prompt.deinit(alloc);
try prompt.syncFrom(alloc, &.{.{ .question = "Input", .options = &.{}, .submission = .input }});
src/core/agent/runtime/agent.zig
+66 -2
View File
@@ -1,8 +1,10 @@
const std = @import("std");
const testing_allocator = @import("../../shared/testing_allocator.zig");
const types = @import("../../shared/types.zig");
const checkpoint_codec = @import("checkpoint.zig");
const runtime_prompt_context = @import("prompt_context.zig");
const debug_trace = @import("../../shared/debug_trace.zig");
const text_utils = @import("../../shared/text_utils.zig");
const Allocator = std.mem.Allocator;
@@ -21,6 +23,9 @@ pub const Agent = struct {
/// definitions, which compaction cannot shrink. Manual compaction sizes
/// its result with it.
request_fixed_tokens: ?usize = null,
/// A request size limit learned from a provider rejection, kept for later
/// requests to the same model in this conversation. Plain value state.
request_byte_limit: ?RequestByteLimitState = null,
pub fn deinit(self: *Agent, alloc: Allocator) void {
self.clearHistory(alloc);
@@ -88,6 +93,27 @@ pub const Agent = struct {
self.history.clearRetainingCapacity();
// Cleared history no longer resembles the calibrated request.
self.request_token_calibration = null;
self.request_byte_limit = null;
}
/// Returns the learned request size limit for `model`, if any.
pub fn requestByteLimit(self: *const Agent, model: []const u8) ?usize {
const state = self.request_byte_limit orelse return null;
if (!std.mem.eql(u8, state.modelSlice(), model)) return null;
return state.bytes;
}
/// Lowers the learned limit for `model` to `bytes`. A limit learned for
/// another model is replaced, and an unmatchable model id is ignored.
pub fn lowerRequestByteLimit(self: *Agent, model: []const u8, bytes: usize) void {
if (model.len == 0 or model.len > max_request_calibration_model_bytes) {
debug_trace.logf("agent", "request size limit not learned: unmatchable model id len={d}", .{model.len});
return;
}
const limit = if (self.requestByteLimit(model)) |current| @min(current, bytes) else bytes;
var state = RequestByteLimitState{ .model_len = model.len, .bytes = limit };
@memcpy(state.model[0..model.len], model);
self.request_byte_limit = state;
}
/// Records the calibration for later turns. An empty or oversized model
@@ -146,10 +172,21 @@ pub const Agent = struct {
previous.deinit(alloc);
// Replaced history no longer resembles the calibrated request.
self.request_token_calibration = null;
self.request_byte_limit = null;
self.fresh = false;
}
};
pub const RequestByteLimitState = struct {
model: [max_request_calibration_model_bytes]u8 = undefined,
model_len: usize = 0,
bytes: usize = 0,
pub fn modelSlice(self: *const RequestByteLimitState) []const u8 {
return self.model[0..self.model_len];
}
};
const max_request_calibration_model_bytes: usize = 128;
pub const RequestTokenCalibrationState = struct {
@@ -190,7 +227,7 @@ test "Agent request token calibration survives startTurn and dies with cleared h
try std.testing.expect(agent.request_token_calibration != null);
// An unmatchable model id clears rather than stores.
agent.storeRequestTokenCalibration("x" ** (max_request_calibration_model_bytes + 1), cost);
agent.storeRequestTokenCalibration(text_utils.repeat("x", max_request_calibration_model_bytes + 1), cost);
try std.testing.expectEqual(@as(?RequestTokenCalibrationState, null), agent.request_token_calibration);
agent.storeRequestTokenCalibration("fixture/model", cost);
@@ -203,6 +240,33 @@ test "Agent request token calibration survives startTurn and dies with cleared h
try std.testing.expectEqual(@as(?RequestTokenCalibrationState, null), agent.request_token_calibration);
}
test "Agent request size limit only lowers, follows the model, and dies with cleared history" {
const alloc = std.testing.allocator;
var agent: Agent = .{};
defer agent.deinit(alloc);
try std.testing.expectEqual(@as(?usize, null), agent.requestByteLimit("fixture/model"));
agent.lowerRequestByteLimit("fixture/model", 4000);
agent.lowerRequestByteLimit("fixture/model", 9000);
try std.testing.expectEqual(@as(?usize, 4000), agent.requestByteLimit("fixture/model"));
try std.testing.expectEqual(@as(?usize, null), agent.requestByteLimit("other/model"));
agent.startTurn();
try std.testing.expectEqual(@as(?usize, 4000), agent.requestByteLimit("fixture/model"));
agent.lowerRequestByteLimit("other/model", 7000);
try std.testing.expectEqual(@as(?usize, 7000), agent.requestByteLimit("other/model"));
try std.testing.expectEqual(@as(?usize, null), agent.requestByteLimit("fixture/model"));
agent.lowerRequestByteLimit(text_utils.repeat("x", max_request_calibration_model_bytes + 1), 10);
try std.testing.expectEqual(@as(?usize, 7000), agent.requestByteLimit("other/model"));
agent.clearHistory(alloc);
try std.testing.expectEqual(@as(?usize, null), agent.requestByteLimit("other/model"));
agent.lowerRequestByteLimit("other/model", 7000);
try agent.restoreHistory(alloc, &.{});
try std.testing.expectEqual(@as(?usize, null), agent.requestByteLimit("other/model"));
}
test "Agent startTurn consumes freshness and resets usage" {
var agent: Agent = .{};
agent.turn_usage = .{ .input_tokens = 4 };
@@ -228,7 +292,7 @@ test "Agent accumulates per-turn usage with saturation" {
}
test "Agent history restore is transactional" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
var agent: Agent = .{};
defer agent.deinit(alloc);
src/core/agent/runtime/assistant_stream.zig
+38 -20
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../../shared/testing_allocator.zig");
const command_admission = @import("../../permissions/command_admission.zig");
const permission_auto_classifier = @import("../../permissions/auto_classifier.zig");
const types = @import("../../shared/types.zig");
@@ -564,7 +565,7 @@ fn deliverSemanticThematicRule(raw: *anyopaque, out: *std.ArrayList(u8)) !void {
pub fn emitProviderLengthNotice(hooks: *const AgentRuntimeDeps, arena: Allocator, disposition: types.ProviderCompletionDisposition) !void {
if (disposition != .length_limited) return;
const line = try std.fmt.allocPrint(arena, "response hit provider length limit; output may be incomplete", .{});
const line = try arena.print("response hit provider length limit; output may be incomplete", .{});
try hooks.push_system_notice(hooks.ctx, line);
}
@@ -579,7 +580,7 @@ pub fn finishLengthLimitedToolCallCompletion(
const has_content = completion.content != null and completion.content.?.len > 0;
const normalized = if (has_content) try normalizeAssistantTextForDisplay(arena, completion.content.?) else "";
const rendered = if (normalized.len > 0)
try std.fmt.allocPrint(arena, "{s}\n\n{s}", .{ normalized, blocker })
try arena.print("{s}\n\n{s}", .{ normalized, blocker })
else
blocker;
@@ -718,7 +719,7 @@ const NoticeCapture = struct {
}
fn describeDenied(_: *anyopaque, arena: Allocator, call: ToolCall, _: ?[]const u8, label: []const u8, _: []const []const u8) ![]const u8 {
return std.fmt.allocPrint(arena, "{s} {s}", .{ label, call.name });
return arena.print("{s} {s}", .{ label, call.name });
}
fn permissionTarget(_: *anyopaque, arena: Allocator, _: ToolCall, _: []const []const u8) ![]const u8 {
@@ -754,7 +755,7 @@ const NoticeCapture = struct {
fn httpError(_: *anyopaque, _: std.http.Status, _: []const u8, _: ?types.CredentialSource) !void {}
fn formatError(_: *anyopaque, arena: Allocator, _: []const u8, err: anyerror) ![]const u8 {
return std.fmt.allocPrint(arena, "{s}", .{@errorName(err)});
return arena.print("{s}", .{@errorName(err)});
}
};
@@ -837,7 +838,7 @@ const StreamCapture = struct {
return arena.dupe(u8, call.name);
}
fn noopDescribeDenied(_: *anyopaque, arena: Allocator, call: ToolCall, _: ?[]const u8, label: []const u8, _: []const []const u8) ![]const u8 {
return std.fmt.allocPrint(arena, "{s} {s}", .{ label, call.name });
return arena.print("{s} {s}", .{ label, call.name });
}
fn noopPermissionTarget(_: *anyopaque, arena: Allocator, _: ToolCall, _: []const []const u8) ![]const u8 {
return arena.dupe(u8, "");
@@ -855,7 +856,7 @@ const StreamCapture = struct {
fn noopCommandOutputComplete(_: *anyopaque, _: ?types.ToolLifecycleId) !void {}
fn noopHttpError(_: *anyopaque, _: std.http.Status, _: []const u8, _: ?types.CredentialSource) !void {}
fn noopFormatError(_: *anyopaque, arena: Allocator, _: []const u8, err: anyerror) ![]const u8 {
return std.fmt.allocPrint(arena, "{s}", .{@errorName(err)});
return arena.print("{s}", .{@errorName(err)});
}
fn captureToolLifecycle(raw: *anyopaque, event: types.ToolLifecycleEvent) !void {
@@ -1015,7 +1016,7 @@ fn assert_frozen_ansi_span_fixture() !void {
var oversized_url: [3000]u8 = undefined;
@memset(&oversized_url, 'a');
var oversized_line_buf: [3072]u8 = undefined;
const oversized_line = try std.fmt.bufPrint(
const oversized_line = try std.mem.print(
&oversized_line_buf,
"literal [x]({s})\n",
.{oversized_url[0..]},
@@ -1323,8 +1324,8 @@ test "streamed markdown image retains raw source and publishes its rendered labe
stream_ctx.raw_text.items,
);
try std.testing.expectEqual(@as(usize, 1), capture.text_spans.items.len);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "▧ architecture") != null);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "![architecture]") == null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "▧ architecture") != null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "![architecture]") == null);
}
test "streamed angle autolinks retain raw source and publish rendered labels" {
@@ -1344,11 +1345,11 @@ test "streamed angle autolinks retain raw source and publish rendered labels" {
stream_ctx.raw_text.items,
);
try std.testing.expectEqual(@as(usize, 1), capture.text_spans.items.len);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "\x1b]8;") != null);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "https://example.com/docs") != null);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "mailto:dev@example.com") != null);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "<https://example.com/docs>") == null);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "<dev@example.com>") == null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "\x1b]8;") != null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "https://example.com/docs") != null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "mailto:dev@example.com") != null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "<https://example.com/docs>") == null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "<dev@example.com>") == null);
}
test "streamed setext heading waits for its underline and flushes before a tool" {
@@ -1425,10 +1426,27 @@ test "streamed presentation preserves ANSI OSC 8 code fence and table spans" {
defer environ.deinit();
try environ.put(ansi_span_fixture_env, "1");
// The fixture reaches the shell tool, which imports the `test_paths`
// module that `zig build test` generates, so the nested compile needs the
// same module.
const test_paths = @import("test_paths");
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const test_paths_source = try alloc.print(
"pub const product_exe: []const u8 = \"{f}\";\npub const source_root: []const u8 = \"{f}\";\n",
.{ std.zig.fmtString(test_paths.product_exe), std.zig.fmtString(test_paths.source_root) },
);
defer alloc.free(test_paths_source);
try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "test_paths.zig", .data = test_paths_source });
const test_paths_path = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "test_paths.zig");
defer alloc.free(test_paths_path);
const test_paths_module = try alloc.print("-Mtest_paths={s}", .{test_paths_path});
defer alloc.free(test_paths_module);
const cwd = try std.process.currentPathAlloc(std.testing.io, alloc);
defer alloc.free(cwd);
const result = try std.process.run(alloc, std.testing.io, .{
.argv = &.{ "zig", "test", "-lc", "-Mroot=src/main.zig", "--test-filter", ansi_span_test_name },
.argv = &.{ "zig", "test", "-lc", "--dep", "test_paths", "-Mroot=src/main.zig", test_paths_module, "--test-filter", ansi_span_test_name },
.cwd = .{ .path = cwd },
.environ_map = &environ,
.stdout_limit = .limited(64 * 1024),
@@ -1480,7 +1498,7 @@ test "streamed footnotes retain raw source and flush before a tool" {
&.{ .text, .table, .text },
capture.presentation_order.items,
);
try std.testing.expect(std.mem.indexOf(u8, capture.text_spans.items[0], "\x1b[2m[1]\x1b[22m") != null);
try std.testing.expect(std.mem.find(u8, capture.text_spans.items[0], "\x1b[2m[1]\x1b[22m") != null);
try std.testing.expectEqualStrings(
"\n\x1b[2m[1] \x1b[22mOBSERVER_FOOTNOTE_PROJECTION\n",
capture.text_spans.items[1],
@@ -1989,7 +2007,7 @@ test "stream callbacks publish absolute raw output token progress" {
}
test "streamed presentation suppresses callback-time failures" {
const alloc = std.testing.allocator;
const alloc = testing_allocator.no_resize;
{
var capture = StreamCapture{};
@@ -2171,7 +2189,7 @@ fn checkSemanticCodeBlockCaptureAllocationFailures(alloc: Allocator) !void {
test "streamed presentation cleans up every allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkStreamPresentationAllocationFailures,
.{},
);
@@ -2179,7 +2197,7 @@ test "streamed presentation cleans up every allocation failure" {
test "semantic table capture cleans up every allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkSemanticTableCaptureAllocationFailures,
.{},
);
@@ -2187,7 +2205,7 @@ test "semantic table capture cleans up every allocation failure" {
test "semantic code block capture cleans up every allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkSemanticCodeBlockCaptureAllocationFailures,
.{},
);
src/core/agent/runtime/checkpoint.zig
+2 -1
View File
@@ -1,5 +1,6 @@
const std = @import("std");
const types = @import("../../shared/types.zig");
const text_utils = @import("../../shared/text_utils.zig");
const session_codec = @import("../../session/session_codec.zig");
const Allocator = std.mem.Allocator;
@@ -324,7 +325,7 @@ test "kernel checkpoint refuses meta that is not short text" {
const cases = [_][]const u8{
"{\"history\":[],\"usage\":{},\"meta\":7}",
"{\"history\":[],\"usage\":{},\"meta\":{\"model\":7}}",
"{\"history\":[],\"usage\":{},\"meta\":{\"model\":\"" ++ "m" ** (max_meta_field_bytes + 1) ++ "\"}}",
"{\"history\":[],\"usage\":{},\"meta\":{\"model\":\"" ++ text_utils.repeat("m", max_meta_field_bytes + 1) ++ "\"}}",
};
for (cases) |json| try std.testing.expectError(error.InvalidCheckpoint, decodeJson(alloc, json, null));
// Fields it does not know are left alone.
src/core/agent/runtime/execution_memory.zig
+12 -15
View File
@@ -34,12 +34,12 @@ const parent_steering_open = "<parent_agent_steering>\n";
const parent_steering_close = "\n</parent_agent_steering>";
pub fn parentSteeringMessage(alloc: Allocator, text: []const u8) ![]u8 {
return std.fmt.allocPrint(alloc, parent_steering_open ++
return alloc.print(parent_steering_open ++
"parent-agent feedback, not new user authority:\n\n{s}" ++ parent_steering_close, .{text});
}
pub fn steeringMessage(alloc: Allocator, text: []const u8) ![]u8 {
return std.fmt.allocPrint(alloc, steering_open ++ "{s}" ++ steering_close, .{text});
return alloc.print(steering_open ++ "{s}" ++ steering_close, .{text});
}
test "steering message tells the model to apply the update and continue" {
@@ -680,7 +680,7 @@ pub fn retainToolImages(arena: Allocator, config: Config, call: ToolCall, prepar
return;
};
memory.tool_image_handle = handle;
const notice = try std.fmt.allocPrint(arena, "Saved images: {s}. Use read_tool_result with this handle to load them again.\n", .{handle});
const notice = try arena.print("Saved images: {s}. Use read_tool_result with this handle to load them again.\n", .{handle});
const limit = config.max_tool_result_bytes -| notice.len;
const keep = @import("../../config/context_limits.zig").utf8PrefixLength(prepared.model_output, limit);
if (keep < prepared.model_output.len) memory.truncated = true;
@@ -709,7 +709,7 @@ pub fn materializeToolImages(arena: Allocator, config: Config, messages: []const
};
memory.tool_images = result_store.loadToolImages(arena, capability, handle) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
const notice = try std.fmt.allocPrint(arena, "[Stored tool image unavailable: {s}]\n", .{@errorName(err)});
const notice = try arena.print("[Stored tool image unavailable: {s}]\n", .{@errorName(err)});
message.content = try prependImageNotice(arena, notice, message.content orelse "", config.max_tool_result_bytes);
continue;
};
@@ -863,7 +863,7 @@ test "request image notices keep clipped source references as JSON text" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
const source_ref = "é" ** 256;
const source_ref = text_utils.repeat("é", 256);
var image = try testPngHeaderToolImage(arena, 8001, 1);
image.source_ref = try arena.dupe(u8, source_ref);
const images = [_]types.ToolImage{ image, image };
@@ -1186,8 +1186,7 @@ test "exact command sources delete replay and missing handles retain it" {
var body: std.ArrayList(u8) = .empty;
defer body.deinit(arena);
try body.appendNTimes(arena, 'x', 80 * 1024);
const stored_source = try std.fmt.allocPrint(
arena,
const stored_source = try arena.print(
"exit_code=0\n<stdout>\n{s}\n</stdout>\n",
.{body.items},
);
@@ -1245,8 +1244,7 @@ test "exact command sources delete replay and missing handles retain it" {
);
transformed_capture.appendAccepted(arena, .stdout, case.raw);
transformed_capture.seal(arena);
const projected_source = try std.fmt.allocPrint(
arena,
const projected_source = try arena.print(
"exit_code=0\n<stdout>\n{s}\n</stdout>\n",
.{case.projected},
);
@@ -1427,8 +1425,7 @@ test "required terminal exec stores large output only as replay" {
var cancel = std.atomic.Value(bool).init(false);
const body = try arena.alloc(u8, result_store.large_result_threshold_bytes + 1024);
@memset(body, 'x');
const raw_output = try std.fmt.allocPrint(
arena,
const raw_output = try arena.print(
"exit_code=0\n<stdout>\n{s}\n</stdout>\n",
.{body},
);
@@ -1481,7 +1478,7 @@ test "saved preparation stores complete output verbatim on cap loss" {
defer arena_state.deinit();
const arena = arena_state.allocator();
var cancel = std.atomic.Value(bool).init(false);
const raw = "MY_UNIT_TOKEN=abcdefgh" ** 47;
const raw = text_utils.repeat("MY_UNIT_TOKEN=abcdefgh", 47);
try std.testing.expect(raw.len > tool_result_limits.min_configured_tool_result_bytes);
const prepared = try prepareToolModelOutput(
@@ -1520,7 +1517,7 @@ test "no-save preparation preserves capped output verbatim without a result hand
defer arena_state.deinit();
const arena = arena_state.allocator();
var cancel = std.atomic.Value(bool).init(false);
const raw = "MY_UNIT_TOKEN=abcdefgh" ** 47;
const raw = text_utils.repeat("MY_UNIT_TOKEN=abcdefgh", 47);
const prepared = try prepareToolModelOutput(
arena,
@@ -1651,7 +1648,7 @@ test "retrieved output storage failure does not publish an unbacked result" {
.cancel_flag = &cancel,
.max_tool_result_bytes = 1024,
.session_child_capability = &readonly,
}, toolCall("retrieval-store-error", "read_tool_result", "{}"), "x" ** 1025));
}, toolCall("retrieval-store-error", "read_tool_result", "{}"), text_utils.repeat("x", 1025)));
}
test "saved tool output preparation keeps builtins and dynamic tools compactable" {
@@ -1667,7 +1664,7 @@ test "saved tool output preparation keeps builtins and dynamic tools compactable
var arena_state = std.heap.ArenaAllocator.init(alloc);
defer arena_state.deinit();
const arena = arena_state.allocator();
const raw = "COMPLETE_TOOL_BODY\n" ++ "x" ** 8192;
const raw = "COMPLETE_TOOL_BODY\n" ++ text_utils.repeat("x", 8192);
const prepared = try prepareToolModelOutput(arena, .{
.system_prompt = "",
.gateway_retry_count = 0,
src/core/agent/runtime/gateway_step.zig
+2 -2
View File
@@ -88,7 +88,7 @@ pub fn streamModelCompletion(
completed.completion,
completed.usage,
);
if (comptime @import("builtin").os.tag != .wasi) {
if (comptime @import("builtin").target.os.tag != .wasi) {
if (std.meta.activeTag(completed.usage) == .deferred) if (usage) |ledger| {
if (request.credential.secret()) |credential| {
ledger.startDeferredReconciliation(
@@ -438,7 +438,7 @@ pub fn gatewayHttpErrorDetail(
model: []const u8,
capabilities: model_capabilities.Capabilities,
) ![]const u8 {
if (@intFromEnum(status) != 413) return detail;
if (@backingInt(status) != 413) return detail;
var out: std.Io.Writer.Allocating = .init(alloc);
defer out.deinit();
if (detail.len > 0) try out.writer.print("{s}\n\n", .{detail});
src/core/agent/runtime/journal.zig
+5 -6
View File
@@ -34,6 +34,7 @@
//! runs it.
const std = @import("std");
const types = @import("../../shared/types.zig");
const testing_allocator = @import("../../shared/testing_allocator.zig");
const session_codec = @import("../../session/session_codec.zig");
const checkpoint_codec = @import("checkpoint.zig");
const compactor = @import("../../compactor/compactor.zig");
@@ -534,11 +535,9 @@ pub fn foldFrom(alloc: Allocator, events_json: []const u8, base: Base) LoadError
else => return error.InvalidJournal,
};
errdefer if (open_turn) |*checkpoint| checkpoint.deinit(alloc);
const announced = session_codec.parseToolCalls(alloc, .{ .array = .{
.items = running.items,
.capacity = running.items.len,
.allocator = alloc,
} }) catch |err| switch (err) {
const announced = session_codec.parseToolCalls(alloc, .{
.array = std.json.Array.fromOwnedSlice(alloc, running.items),
}) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.InvalidJournal,
};
@@ -1016,7 +1015,7 @@ test "a fold that runs out of memory frees what it built" {
.{ .tool_intent = &calls },
});
defer alloc.free(bytes);
try std.testing.checkAllAllocationFailures(alloc, foldAndFree, .{bytes});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, foldAndFree, .{bytes});
}
fn foldAndFree(alloc: Allocator, bytes: []const u8) !void {
src/core/agent/runtime/lifecycle.zig
+3 -2
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../../shared/testing_allocator.zig");
const hooks = @import("../../hooks/hooks.zig");
const types = @import("../../shared/types.zig");
const tool_result_errors = @import("../../tooling/tool_result_errors.zig");
@@ -193,7 +194,7 @@ fn checkNonObjectPreparationAllocationFailures(alloc: Allocator) !void {
}
test "non-object preparation cleans every failed allocation" {
try std.testing.checkAllAllocationFailures(std.testing.allocator, checkNonObjectPreparationAllocationFailures, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, checkNonObjectPreparationAllocationFailures, .{});
}
fn expectMalformedFeedback(model_output: []const u8, failure: []const u8, raw: []const u8) !void {
@@ -250,7 +251,7 @@ fn checkMalformedPreparationAllocationFailures(alloc: Allocator) !void {
}
test "malformed preparation cleans every failed allocation" {
try std.testing.checkAllAllocationFailures(std.testing.allocator, checkMalformedPreparationAllocationFailures, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, checkMalformedPreparationAllocationFailures, .{});
}
pub noinline fn prepareToolCallForLifecycle(
src/core/agent/runtime/orchestrator.zig
+166 -30
View File
@@ -1,4 +1,5 @@
const std = @import("std");
const testing_allocator = @import("../../shared/testing_allocator.zig");
const skill_runtime = @import("../../skills/skill_runtime.zig");
const skill_contract = @import("../../skills/skill_contract.zig");
const skill_invocation = @import("../../skills/skill_invocation.zig");
@@ -57,6 +58,7 @@ const image_data = @import("../../images/image_data.zig");
const runtime_assistant_stream = @import("assistant_stream.zig");
const runtime_tool_presentation = @import("tool_presentation.zig");
const runtime_execution_memory = @import("execution_memory.zig");
const request_image_budget = @import("request_image_budget.zig");
const execution_memory_helpers = @import("../execution_memory.zig");
const runtime_agent = @import("agent.zig");
const runtime_tool_admission = @import("tool_admission.zig");
@@ -593,7 +595,7 @@ test "legacy exec argument projection releases temporary allocations" {
try std.testing.expectEqualStrings(":", request.object.get("command").?.string);
}
};
try std.testing.checkAllAllocationFailures(std.testing.allocator, Probe.run, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, Probe.run, .{});
}
fn findLegacyCall(
@@ -616,14 +618,12 @@ fn legacyToolSummary(
const body = content orelse "";
const bounded = text_utils.utf8PrefixByBytes(body, 4096);
return if (bounded.len == 0)
std.fmt.allocPrint(
alloc,
alloc.print(
"[Prior terminal {s} action completed.]",
.{action},
)
else
std.fmt.allocPrint(
alloc,
alloc.print(
"[Prior terminal {s} action completed. Stored result follows.]\n{s}",
.{ action, bounded },
);
@@ -957,10 +957,9 @@ fn subagent_history_summary(
) Allocator.Error![]u8 {
const bounded = text_utils.utf8PrefixByBytes(content orelse "", 4096);
return if (bounded.len == 0)
std.fmt.allocPrint(alloc, "[Prior subagent {s} action completed.]", .{action})
alloc.print("[Prior subagent {s} action completed.]", .{action})
else
std.fmt.allocPrint(
alloc,
alloc.print(
"[Prior subagent {s} action completed. Stored result follows.]\n{s}",
.{ action, bounded },
);
@@ -1289,7 +1288,7 @@ fn check_subagent_history_projection_allocation_failures(alloc: Allocator) !void
test "subagent history projection cleans every partial allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
check_subagent_history_projection_allocation_failures,
.{},
);
@@ -2078,7 +2077,7 @@ fn check_legacy_replay_projection_allocations(alloc: Allocator) !void {
}
test "legacy request projection preserves replay ownership and source binding on failure" {
try std.testing.checkAllAllocationFailures(std.testing.allocator, check_legacy_replay_projection_allocations, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, check_legacy_replay_projection_allocations, .{});
}
test "mixed legacy terminal batches become inert in every order" {
@@ -2261,7 +2260,7 @@ fn check_terminal_request_projection_allocation_failures(alloc: Allocator) !void
test "terminal request projection cleans every partial allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
check_terminal_request_projection_allocation_failures,
.{},
);
@@ -2449,7 +2448,7 @@ fn check_terminal_request_normalization_allocation_failures(alloc: Allocator) !v
test "terminal request normalization cleans every partial allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
check_terminal_request_normalization_allocation_failures,
.{},
);
@@ -2566,7 +2565,7 @@ fn prepareSkillCall(deps: *const AgentRuntimeDeps, arena: Allocator, call: ToolC
const prepare = deps.prepare_skill_call orelse return null;
return prepare(deps.ctx, arena, call, locations) catch |err| switch (err) {
error.OutOfMemory, error.Cancelled => return err,
else => .{ .failure = .{ .model_output = try std.fmt.allocPrint(arena, "skill failed: {s}", .{@errorName(err)}) } },
else => .{ .failure = .{ .model_output = try arena.print("skill failed: {s}", .{@errorName(err)}) } },
};
}
@@ -2703,8 +2702,7 @@ fn preparedTerminalModelOutput(
) Allocator.Error![]const u8 {
if (terminal.model_output) |model_output| return model_output;
return switch (terminal.kind) {
.unsupported => try std.fmt.allocPrint(
alloc,
.unsupported => try alloc.print(
"Unsupported tool: {s}",
.{call.name},
),
@@ -4102,12 +4100,12 @@ test "recovery checkpoints do not accumulate temporary history copies in the tur
const alloc = turn.allocator();
var messages: std.ArrayList(ChatMessage) = .empty;
for (0..1000) |step_index| {
const id = try std.fmt.allocPrint(alloc, "read_{d}", .{step_index});
const id = try alloc.print("read_{d}", .{step_index});
const calls = try alloc.alloc(ToolCall, 1);
calls[0] = .{ .id = id, .name = "read_file", .arguments_json = "{\"path\":\"evidence.txt\"}" };
try messages.appendSlice(alloc, &.{
.{ .role = .assistant, .tool_calls = calls },
.{ .role = .tool, .tool_call_id = id, .tool_name = "read_file", .tool_result_status = .success, .content = "current evidence" ** 16 },
.{ .role = .tool, .tool_call_id = id, .tool_name = "read_file", .tool_result_status = .success, .content = text_utils.repeat("current evidence", 16) },
});
const retained_bytes = turn.queryCapacity();
try persistRecoveryCheckpoint(&deps, &finalization, fixture.job(), messages.items, "partial", "model", false, false, 10, 1, false, .transport_interrupted, .retry_request, .confirmed, .{});
@@ -4400,7 +4398,7 @@ test "discarded completion prose preserves ownership and fails atomically" {
}
};
try Probe.run(std.testing.allocator);
try std.testing.checkAllAllocationFailures(std.testing.allocator, Probe.run, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, Probe.run, .{});
}
fn isRetryableModelFailure(kind: agent_stream_provider.FailureKind) bool {
@@ -4511,6 +4509,95 @@ test "context overflow recovery is typed safe and bounded" {
));
}
/// Size retries allowed in one turn. Each lowers the learned limit, so the
/// next request leaves out more of its oldest images.
const max_request_size_retries: u8 = 2;
/// A size rejection is retried without compaction only when the rejected
/// request carried images the next one can leave out. Anything else falls
/// through to context overflow recovery.
fn shouldRetryWithFewerImages(
failure: agent_stream_provider.Failure,
images_sent: usize,
retries_used: u8,
replay_safe: bool,
cancelled: bool,
) bool {
return !cancelled and
replay_safe and
images_sent > 0 and
retries_used < max_request_size_retries and
failure.kind == .request_too_large;
}
/// The size the next request must fit after `rejected_bytes` were refused.
fn loweredRequestByteLimit(rejected_bytes: usize) usize {
return rejected_bytes / 4 * 3;
}
/// Leaves the oldest images out of `request` until its serialized body fits
/// `limits`, rebuilding after each change, and returns the body to send.
/// Every pass leaves out at least one more image, so the loop ends.
fn fitRequestImages(
arena: Allocator,
provider: agent_stream_provider.Provider,
request: *agent_stream_provider.RequestData,
first_body: []const u8,
limits: request_image_budget.Limits,
tool_text_limit: usize,
trace_ctx: TraceContext,
) ![]const u8 {
var body = first_body;
while (true) {
const projection = try request_image_budget.withholdOldestImages(arena, request.messages, body.len, limits, tool_text_limit);
if (projection.withheld == 0) return body;
var smaller = request.*;
smaller.messages = projection.messages;
const rebuilt = (try provider.buildRequest(arena, smaller)) orelse return body;
debug_trace.eventf("images", "request_images_withheld", trace_ctx, "withheld={d} request_bytes_before={d} request_bytes_after={d} limit_bytes={d}", .{
projection.withheld,
body.len,
rebuilt.len,
limits.max_request_bytes,
});
request.* = smaller;
body = rebuilt;
}
}
test "size rejections retry with fewer images only while images remain" {
const too_large = agent_stream_provider.Failure{ .kind = .request_too_large };
const overflow_text = agent_stream_provider.Failure{
.kind = .invalid_request,
.detail = @constCast("AI_APICallError: input is too long"),
};
const cases = [_]struct {
failure: agent_stream_provider.Failure,
images_sent: usize = 3,
retries_used: u8 = 0,
replay_safe: bool = true,
cancelled: bool = false,
expected: bool,
}{
.{ .failure = too_large, .expected = true },
.{ .failure = too_large, .retries_used = max_request_size_retries - 1, .expected = true },
.{ .failure = too_large, .retries_used = max_request_size_retries, .expected = false },
.{ .failure = too_large, .images_sent = 0, .expected = false },
.{ .failure = too_large, .replay_safe = false, .expected = false },
.{ .failure = too_large, .cancelled = true, .expected = false },
.{ .failure = overflow_text, .expected = false },
};
for (cases) |case| try std.testing.expectEqual(case.expected, shouldRetryWithFewerImages(
case.failure,
case.images_sent,
case.retries_used,
case.replay_safe,
case.cancelled,
));
try std.testing.expectEqual(@as(usize, 30_000), loweredRequestByteLimit(40_000));
try std.testing.expect(loweredRequestByteLimit(33_829_735) < 33_829_735);
}
/// fx never asks a model to extend an assistant message, and models without
/// prefill support reject that shape. Returns `source` unchanged, or an
/// `arena` copy ending with a host continuation when history projection left
@@ -4757,7 +4844,7 @@ test "deferred retry cleanup clears status and contains sink errors" {
clear_deferred_retry_status(&deps);
try std.testing.expectEqual(@as(usize, 1), normal.route_recovery_clear_count);
var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = 0 });
var failing = std.testing.FailingAllocator.init(testing_allocator.no_resize, .{ .fail_index = 0 });
var failed = support.FakeAgentRuntimeDeps.init(failing.allocator());
defer failed.deinit();
const failed_deps = failed.deps();
@@ -5223,7 +5310,7 @@ fn prepareExternalFailureDiagnostic(
) !types.ModelFailureDiagnostic {
const trimmed = std.mem.trim(u8, raw, " \t\r\n");
const owned_source = if (trimmed.len > 0 and !hasDiagnosticIdentifier(trimmed))
try std.fmt.allocPrint(alloc, "{s}: {s}", .{ fallback, trimmed })
try alloc.print("{s}: {s}", .{ fallback, trimmed })
else
null;
defer if (owned_source) |source| alloc.free(source);
@@ -5242,9 +5329,9 @@ fn prepareExternalFailureDiagnostic(
}
fn hasDiagnosticIdentifier(text: []const u8) bool {
if (std.mem.indexOf(u8, text, ": ")) |separator| {
if (std.mem.find(u8, text, ": ")) |separator| {
if (separator == 0) return false;
return std.mem.indexOfAny(u8, text[0..separator], " \t\r\n") == null;
return std.mem.findAny(u8, text[0..separator], " \t\r\n") == null;
}
for (text) |byte| {
if (!std.ascii.isAlphanumeric(byte) and byte != '_' and byte != '-' and byte != '.') return false;
@@ -6222,7 +6309,7 @@ pub fn compactContext(
.before_summary = request.before_summary,
};
if (progress.operation_id) |id| deps.compaction_activity.?.running(deps.ctx, id, .preparation);
errdefer |err| {
return compactWithProgress(alloc, deps, request, &progress) catch |err| {
if (err == error.Cancelled) {
compactor.traceEvent(trace_ctx, .transaction_failed, "stage={s} origin={s} err={s}", .{ @tagName(progress.stage), @tagName(request.activity_origin), @errorName(err) });
} else {
@@ -6232,7 +6319,18 @@ pub fn compactContext(
deps.compaction_activity.?.settle(deps.ctx, id, compaction_activity.failure(err, progress.stage, cancel_flag.load(.seq_cst)));
if (request.failure_provenance) |out| out.* = .{ .operation_id = id, .turn_id = trace_ctx.turn_id, .err = err };
}
}
return err;
};
}
fn compactWithProgress(
alloc: Allocator,
deps: *const AgentRuntimeDeps,
request: ContextCompactionRequest,
progress: *CompactionProgress,
) !?compactor.Result {
const trace_ctx = request.compactor.trace_ctx;
const cancel_flag = request.compactor.cancel_flag;
var compactor_request = request.compactor;
compactor_request.progress = progress.interface();
var result = try compactor.compact(alloc, compactor_request) orelse {
@@ -6329,7 +6427,7 @@ test "compaction activity automatic error provenance excludes secondary finaliza
var job = fixture.job();
job.turn_id = 31;
job.model = @constCast(model);
var steps = [_]types.ToolExecutionStep{.{ .assistant = @constCast("history " ** 19_000) }};
var steps = [_]types.ToolExecutionStep{.{ .assistant = @constCast(text_utils.repeat("history ", 19_000)) }};
var history = [_]HistoryTurn{.{ .assistant = .{
.user = .{ .text = @constCast("earlier request") },
.assistant = @constCast("earlier answer"),
@@ -6447,7 +6545,7 @@ test "automatic compaction interruption persists transport cancellation and pres
var job = fixture.job();
job.turn_id = 37;
job.model = @constCast(model);
var steps = [_]types.ToolExecutionStep{.{ .assistant = @constCast("history " ** 19_000) }};
var steps = [_]types.ToolExecutionStep{.{ .assistant = @constCast(text_utils.repeat("history ", 19_000)) }};
var history = [_]HistoryTurn{.{ .assistant = .{
.user = .{ .text = @constCast("earlier request") },
.assistant = @constCast("earlier answer"),
@@ -6598,7 +6696,7 @@ test "compaction activity transaction settles only after publication and preserv
// A tool call leaves work to summarize, so the model is asked. Its
// output is larger than the kept budget, so the turn is compacted,
// while its user message still fits word for word.
const notes = "recorded notes " ** 400;
const notes = text_utils.repeat("recorded notes ", 400);
const calls = [_]types.ToolCall{.{ .id = "record-1", .name = "read_file", .arguments_json = "{\"path\":\"notes.md\"}" }};
const results = [_]types.PersistedToolResult{.{
.tool_call_id = @constCast("record-1"),
@@ -6610,7 +6708,7 @@ test "compaction activity transaction settles only after publication and preserv
}};
const steps = [_]types.ToolExecutionStep{.{ .tool_calls = @constCast(&calls), .tool_results = @constCast(&results) }};
const turns = [_]HistoryTurn{.{ .assistant = .{
.user = .{ .text = @constCast("recorded source " ** 100) },
.user = .{ .text = @constCast(text_utils.repeat("recorded source ", 100)) },
.assistant = @constCast("Recorded it."),
.execution = .{ .tool_steps = @constCast(&steps) },
} }};
@@ -7008,6 +7106,7 @@ fn processQueuedPromptLoop(
var assistant_prefill_recovery_used = false;
var skip_next_preflight_refresh = false;
var context_overflow_recovery: ContextOverflowRecoveryState = .ready;
var request_size_retries: u8 = 0;
var recovery_has_unexecuted_tool_start = false;
var successful_recovery_strategy: ?model_response_recovery.Strategy = null;
defer {
@@ -7269,7 +7368,7 @@ fn processQueuedPromptLoop(
config.first_call_tool_choice
else
.auto;
const request_data = agent_stream_provider.RequestData{
var request_data = agent_stream_provider.RequestData{
.model = gateway_model,
.instructions = gateway_instructions.items,
.messages = request_messages,
@@ -7287,11 +7386,22 @@ fn processQueuedPromptLoop(
};
var prepared_request_body: ?[]const u8 = null;
var request_cost_for_attempt: ?runtime_prompt_context.RequestCost = null;
var request_images_for_attempt: usize = 0;
if (try deps.agent_stream_provider.buildRequest(
overlay_arena,
request_data,
)) |request_body| {
)) |built_request_body| {
const request_body = try fitRequestImages(
overlay_arena,
deps.agent_stream_provider,
&request_data,
built_request_body,
.{ .max_request_bytes = agent.requestByteLimit(gateway_model) orelse request_image_budget.default_max_request_bytes },
config.max_tool_result_bytes,
step_ctx,
);
prepared_request_body = request_body;
request_images_for_attempt = image_data.countRequestImages(request_data.messages);
const measured_request_cost = try runtime_prompt_context.measureProviderRequest(std.heap.c_allocator, request_body, request_data);
const applicable_calibration = if (agent.request_token_calibration) |*calibration|
if (std.mem.eql(u8, calibration.modelSlice(), gateway_model) and calibration.cost.applies(measured_request_cost))
@@ -8102,6 +8212,32 @@ fn processQueuedPromptLoop(
else
compacted_suffix_len < within_turn_suffix.items.len;
if (response_failure) |failure| {
// A size rejection with images aboard retries with a lower
// limit, which leaves out the oldest images instead of
// compacting the conversation.
if (request_cost_for_attempt) |rejected| if (shouldRetryWithFewerImages(
failure,
request_images_for_attempt,
request_size_retries,
streamReplaySafe(&stream_ctx),
config.cancel_flag.load(.seq_cst),
)) {
const limit = loweredRequestByteLimit(rejected.serialized_bytes);
agent.lowerRequestByteLimit(gateway_model, limit);
request_size_retries += 1;
debug_trace.eventf("images", "request_size_retry", step_ctx, "model={s} rejected_bytes={d} images_sent={d} limit_bytes={d} retry={d}", .{
gateway_model,
rejected.serialized_bytes,
request_images_for_attempt,
limit,
request_size_retries,
});
_ = summary_accumulator.finishTokenRequestWithoutUsage(false);
stream_result.deinit(arena);
stream_result_set = false;
reset_stream_for_next_attempt = true;
continue;
};
if (shouldRecoverContextOverflow(
failure,
has_compactable_context,
src/core/agent/runtime/parallel_execution.zig
+6 -6
View File
@@ -1,5 +1,6 @@
const std = @import("std");
const builtin = @import("builtin");
const testing_allocator = @import("../../shared/testing_allocator.zig");
const skill_contract = @import("../../skills/skill_contract.zig");
const types = @import("../../shared/types.zig");
const tool_dispatch = @import("../../tooling/tool_dispatch.zig");
@@ -178,7 +179,7 @@ pub fn runSequentialCalls(
} else completed: {
const execution = options.execute(options.exec_ctx, alloc, call, index) catch |err| blk: {
const output = options.format_error(options.format_ctx, alloc, call.name, err) catch
try std.fmt.allocPrint(alloc, "Tool execution failed: {s}", .{@errorName(err)});
try alloc.print("Tool execution failed: {s}", .{@errorName(err)});
defer alloc.free(output);
break :blk ToolExecutionResult{
.status = .failure,
@@ -326,8 +327,7 @@ fn materializeParallelAttempt(
const execution: ToolExecutionResult = if (slot.err) |err| .{
.status = .failure,
.model_output = blk: {
const output = options.format_error(options.format_ctx, alloc, call.name, err) catch |format_err| try std.fmt.allocPrint(
alloc,
const output = options.format_error(options.format_ctx, alloc, call.name, err) catch |format_err| try alloc.print(
"Tool execution failed: {s}; additionally failed to format error: {s}",
.{ @errorName(err), @errorName(format_err) },
);
@@ -578,7 +578,7 @@ fn updateMaxInFlight(max_in_flight: *std.atomic.Value(usize), candidate: usize)
}
fn parallelTestFormatError(_: *anyopaque, alloc: Allocator, tool_name: []const u8, err: anyerror) ![]const u8 {
return std.fmt.allocPrint(alloc, "{s} failed with {s}", .{ tool_name, @errorName(err) });
return alloc.print("{s} failed with {s}", .{ tool_name, @errorName(err) });
}
fn toolCall(id: []const u8, name: []const u8, args: []const u8) ToolCall {
@@ -972,7 +972,7 @@ fn checkParallelRunAllocationFailures(alloc: Allocator) !void {
test "parallel run cleans up every allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkParallelRunAllocationFailures,
.{},
);
@@ -1043,7 +1043,7 @@ fn checkParallelResultDuplicationAllocationFailures(alloc: Allocator) !void {
test "parallel result duplication cleans up every allocation failure" {
try std.testing.checkAllAllocationFailures(
std.testing.allocator,
testing_allocator.no_resize,
checkParallelResultDuplicationAllocationFailures,
.{},
);
src/core/agent/runtime/prompt_context.zig
+58 -17
View File
@@ -1,7 +1,9 @@
const std = @import("std");
const testing_allocator = @import("../../shared/testing_allocator.zig");
const token_estimate = @import("../../shared/token_estimate.zig");
const types = @import("../../shared/types.zig");
const stream_provider = @import("../stream_provider.zig");
const text_utils = @import("../../shared/text_utils.zig");
const Allocator = std.mem.Allocator;
const ChatMessage = types.ChatMessage;
@@ -32,12 +34,27 @@ const ImagePartData = struct {
data: ?[]const u8 = null,
};
/// Responses `input_image` parts carry the URL as a string; chat-completions
/// `image_url` parts wrap it in an object.
const ImageUrl = struct {
url: []const u8,
pub fn jsonParse(alloc: Allocator, source: anytype, options: std.json.ParseOptions) !ImageUrl {
if (try source.peekNextTokenType() == .string) {
return .{ .url = try std.json.innerParse([]const u8, alloc, source, options) };
}
const Wrapped = struct { url: []const u8 };
const wrapped = try std.json.innerParse(Wrapped, alloc, source, options);
return .{ .url = wrapped.url };
}
};
const ImagePart = struct {
type: []const u8 = "",
mediaType: ?[]const u8 = null,
detail: ?[]const u8 = null,
data: ?ImagePartData = null,
image_url: ?[]const u8 = null,
image_url: ?ImageUrl = null,
};
const MessageContent = struct {
@@ -62,6 +79,7 @@ const CostMessage = struct {
const CostRequest = struct {
prompt: ?[]const CostMessage = null,
input: ?[]const CostMessage = null,
messages: ?[]const CostMessage = null,
};
pub const MeasurementError = error{ OutOfMemory, InvalidRequestMeasurement };
@@ -94,11 +112,13 @@ pub fn measureProviderRequest(alloc: Allocator, body: []const u8, request: strea
else => error.InvalidRequestMeasurement,
};
defer parsed.deinit();
if (parsed.value.prompt != null and parsed.value.input != null) return error.InvalidRequestMeasurement;
const messages = parsed.value.prompt orelse parsed.value.input orelse {
// Unrecognized envelope (e.g. chat-completions "messages" bodies):
// degrade to the conservative text estimate rather than fail the
// request over a shape this measurer does not know.
const envelopes = @as(u8, @intFromBool(parsed.value.prompt != null)) +
@intFromBool(parsed.value.input != null) +
@intFromBool(parsed.value.messages != null);
if (envelopes > 1) return error.InvalidRequestMeasurement;
const messages = parsed.value.prompt orelse parsed.value.input orelse parsed.value.messages orelse {
// Unrecognized envelope: degrade to the conservative text estimate
// rather than fail the request over a shape this measurer does not know.
const tokens = textTokens(body);
return .{ .serialized_bytes = body.len, .text_tokens = tokens, .estimated_input_tokens = tokens };
};
@@ -119,7 +139,9 @@ pub fn measureProviderRequest(alloc: Allocator, body: []const u8, request: strea
continue;
for (parts) |part| {
const payload = if (parsed.value.input != null and std.mem.eql(u8, part.type, "input_image"))
part.image_url orelse return error.InvalidRequestMeasurement
(part.image_url orelse return error.InvalidRequestMeasurement).url
else if (parsed.value.messages != null and std.mem.eql(u8, part.type, "image_url"))
(part.image_url orelse return error.InvalidRequestMeasurement).url
else if (parsed.value.prompt != null and std.mem.eql(u8, part.type, "file") and
std.mem.startsWith(u8, part.mediaType orelse "", "image/"))
payload: {
@@ -392,7 +414,7 @@ test "provider request image accounting excludes encoded payload length" {
};
inline for (cases) |case| {
const small = try measureProviderRequest(std.testing.allocator, case.prefix ++ "AAAA" ++ case.suffix, measurement_test_request(true));
const large = try measureProviderRequest(std.testing.allocator, case.prefix ++ ("AAAA" ** 1000) ++ case.suffix, measurement_test_request(true));
const large = try measureProviderRequest(std.testing.allocator, case.prefix ++ text_utils.repeat("AAAA", 1000) ++ case.suffix, measurement_test_request(true));
try std.testing.expect(large.serialized_bytes > small.serialized_bytes);
try std.testing.expectEqual(small.text_tokens, large.text_tokens);
try std.testing.expectEqual(small.estimated_input_tokens, large.estimated_input_tokens);
@@ -498,12 +520,31 @@ test "provider request measurement excludes responses-protocol tool image payloa
try std.testing.expectEqual(textTokens(without_image_payload), measured.text_tokens);
}
test "provider request measurement degrades to text estimate on unknown envelopes" {
// Chat-completions bodies carry a "messages" array this measurer does not
// parse. An image-bearing request must degrade to the conservative text
// estimate, never fail the request.
test "provider request measurement excludes chat-completions image payloads" {
// Chat-completions bodies carry user images, including retained tool
// images, as image_url parts with data URLs.
const body =
\\{"model":"fixture/model","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]}]}
\\{"model":"fixture/model","messages":[{"role":"tool","tool_call_id":"call_1","content":"capture"},{"role":"user","content":[{"type":"text","text":"The tool \"read_file\" returned 1 image(s)."},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAABBBBCCCCDDDD"}}]}]}
;
var request = measurement_test_request(false);
request.messages = &.{.{
.role = .tool,
.content = "capture",
.tool_result_memory = .{
.tool_images = &.{.{ .data = @constCast("AAAABBBBCCCCDDDD"), .mime_type = @constCast("image/png") }},
},
}};
const measured = try measureProviderRequest(std.testing.allocator, body, request);
try std.testing.expect(measured.image_identity != null);
const without_image_payload =
\\{"model":"fixture/model","messages":[{"role":"tool","tool_call_id":"call_1","content":"capture"},{"role":"user","content":[{"type":"text","text":"The tool \"read_file\" returned 1 image(s)."},{"type":"image_url","image_url":{"url":""}}]}]}
;
try std.testing.expectEqual(textTokens(without_image_payload), measured.text_tokens);
}
test "provider request measurement degrades to text estimate on unknown envelopes" {
const body =
\\{"model":"fixture/model","contents":[{"role":"user","parts":[{"inline_data":{"data":"AAAA"}}]}]}
;
const measured = try measureProviderRequest(std.testing.allocator, body, measurement_test_request(true));
try std.testing.expectEqual(textTokens(body), measured.estimated_input_tokens);
@@ -511,7 +552,7 @@ test "provider request measurement degrades to text estimate on unknown envelope
}
test "provider request image calibration uses exact usage plus text growth without compounding" {
const first = RequestCost{ .serialized_bytes = 4_000_000, .text_tokens = 100, .image_identity = [_]u8{1} ** 32, .estimated_input_tokens = 100 };
const first = RequestCost{ .serialized_bytes = 4_000_000, .text_tokens = 100, .image_identity = @as([32]u8, @splat(1)), .estimated_input_tokens = 100 };
var next = first;
next.text_tokens = 150;
next.estimated_input_tokens = 150;
@@ -526,7 +567,7 @@ test "provider request image calibration uses exact usage plus text growth witho
try std.testing.expectEqual(@as(usize, 1100), repeated.estimated_input_tokens);
try std.testing.expectEqual(@as(usize, 1000), calibrateProviderRequest(first, .{ .request = calibrated, .exact_input_tokens = 1050 }).estimated_input_tokens);
next.image_identity = [_]u8{2} ** 32;
next.image_identity = @as([32]u8, @splat(2));
try std.testing.expectEqual(next, calibrateProviderRequest(next, .{ .request = first, .exact_input_tokens = 1000 }));
next.image_identity = null;
try std.testing.expectEqual(next, calibrateProviderRequest(next, .{ .request = first, .exact_input_tokens = 1000 }));
@@ -541,7 +582,7 @@ test "provider request image accounting borrows large payloads and releases scra
const payload = try alloc.alloc(u8, 4 * 1024 * 1024);
defer alloc.free(payload);
@memset(payload, 'A');
const body = try std.fmt.allocPrint(alloc, "{{\"input\":[{{\"role\":\"user\",\"content\":[{{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,{s}\"}}]}}]}}", .{payload});
const body = try alloc.print("{{\"input\":[{{\"role\":\"user\",\"content\":[{{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,{s}\"}}]}}]}}", .{payload});
defer alloc.free(body);
var storage: [8192]u8 = undefined;
var scratch = std.heap.FixedBufferAllocator.init(&storage);
@@ -577,7 +618,7 @@ test "provider request image accounting handles allocation and malformed input f
_ = try measureProviderRequest(alloc, "{\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,AAAA\"}]}]}", measurement_test_request(true));
}
};
try std.testing.checkAllAllocationFailures(std.testing.allocator, Probe.run, .{});
try std.testing.checkAllAllocationFailures(testing_allocator.no_resize, Probe.run, .{});
for ([_][]const u8{
"{",
"{\"input\":[],\"prompt\":[]}",
src/core/agent/runtime/request_image_budget.zig
+218
View File
@@ -0,0 +1,218 @@
//! Keeps the images re-sent with every request under the provider's request
//! size limit. Each request carries every earlier image, so its body grows
//! with every screenshot while its token estimate barely moves. The oldest
//! images leave the request first, each with a note for loading it again.
//! History is never changed.
const std = @import("std");
const types = @import("../../shared/types.zig");
const image_attachments = @import("../../images/image_attachments.zig");
const utf8PrefixLength = @import("../../config/context_limits.zig").utf8PrefixLength;
const Allocator = std.mem.Allocator;
const ChatMessage = types.ChatMessage;
/// Claude routes reject request bodies over 32 MiB, the strictest limit
/// measured through AI Gateway. Gemini and OpenAI routes accept larger ones.
pub const default_max_request_bytes: usize = 30 * 1024 * 1024;
/// Anthropic accepts at most 100 images in one request.
pub const default_max_request_images: usize = 100;
/// Planning allowance for the note that replaces each withheld image.
const notice_reserve_bytes: usize = 320;
pub const Limits = struct {
max_request_bytes: usize = default_max_request_bytes,
max_images: usize = default_max_request_images,
};
/// Returns how many of the oldest images must leave a request of
/// `request_bytes` for it to fit `limits`. `image_bytes` holds each image's
/// serialized size, oldest first. Returns `image_bytes.len` when the request
/// does not fit even without images.
pub fn oldestImagesToWithhold(image_bytes: []const usize, request_bytes: usize, limits: Limits) usize {
var withheld: usize = 0;
var bytes = request_bytes;
while (withheld < image_bytes.len) : (withheld += 1) {
if (image_bytes.len - withheld <= limits.max_images and bytes <= limits.max_request_bytes) break;
bytes = (bytes -| image_bytes[withheld]) +| notice_reserve_bytes;
}
return withheld;
}
pub const Projection = struct {
messages: []const ChatMessage,
withheld: usize = 0,
};
/// Leaves the oldest images out of a request whose serialized size is
/// `request_bytes` until it fits `limits`. Returned messages are `messages`
/// itself when nothing is withheld, otherwise a copy allocated in `arena`.
/// `tool_text_limit` bounds tool result content after the note is added.
pub fn withholdOldestImages(
arena: Allocator,
messages: []const ChatMessage,
request_bytes: usize,
limits: Limits,
tool_text_limit: usize,
) Allocator.Error!Projection {
var sizes: std.ArrayList(usize) = .empty;
for (messages) |message| {
for (message.images) |image| {
// An unreadable attachment cannot be serialized either; it
// still counts toward the image limit.
try sizes.append(arena, image_attachments.attachmentEncodedBytes(image) orelse 0);
}
if (message.tool_result_memory) |memory| {
for (memory.tool_images) |image| try sizes.append(arena, image.data.len);
}
}
const withheld = oldestImagesToWithhold(sizes.items, request_bytes, limits);
if (withheld == 0) return .{ .messages = messages };
const projected = try arena.dupe(ChatMessage, messages);
var remaining = withheld;
for (projected) |*message| {
if (remaining == 0) break;
if (message.images.len > 0) {
const count = @min(remaining, message.images.len);
var notice: std.Io.Writer.Allocating = .init(arena);
for (message.images[0..count]) |image| writeAttachmentNotice(&notice.writer, image) catch return error.OutOfMemory;
message.images = message.images[count..];
message.content = try std.mem.concat(arena, u8, &.{ notice.written(), message.content orelse "" });
remaining -= count;
}
if (remaining == 0) break;
if (message.tool_result_memory) |*memory| {
if (memory.tool_images.len == 0) continue;
const count = @min(remaining, memory.tool_images.len);
const notice = try toolImagesNotice(arena, count, memory.tool_image_handle);
memory.tool_images = memory.tool_images[count..];
message.content = try prependNotice(arena, notice, message.content orelse "", tool_text_limit);
remaining -= count;
}
}
return .{ .messages = projected, .withheld = withheld };
}
fn writeAttachmentNotice(writer: *std.Io.Writer, image: types.ImageAttachment) std.Io.Writer.Error!void {
try writer.print("[Image #{d} not sent to keep the request under the provider's size limit.", .{image.id});
if (image.inline_data == null) {
if (image.snapshot_path) |path| try writer.print(" The original is saved at {s}; read_file it to see it again.", .{path});
}
try writer.writeAll("]\n");
}
fn toolImagesNotice(arena: Allocator, count: usize, handle: ?[]const u8) Allocator.Error![]u8 {
const subject = if (count == 1) "1 image" else try arena.print("{d} images", .{count});
if (handle) |value| {
const object = if (count == 1) "it" else "them";
return arena.print("[{s} from this result not sent to keep the request under the provider's size limit. Use read_tool_result with handle {s} to load {s} again.]\n", .{ subject, value, object });
}
return arena.print("[{s} from this result not sent to keep the request under the provider's size limit.]\n", .{subject});
}
fn prependNotice(arena: Allocator, notice: []const u8, content: []const u8, limit: usize) Allocator.Error![]u8 {
const notice_keep = utf8PrefixLength(notice, limit);
const keep = utf8PrefixLength(content, limit -| notice.len);
return std.mem.concat(arena, u8, &.{ notice[0..notice_keep], content[0..keep] });
}
test "oldest images leave until the request fits" {
const limits = Limits{ .max_request_bytes = 1000, .max_images = 10 };
const sizes = [_]usize{ 400, 400, 400 };
try std.testing.expectEqual(@as(usize, 0), oldestImagesToWithhold(&sizes, 1000, limits));
try std.testing.expectEqual(@as(usize, 1), oldestImagesToWithhold(&sizes, 1001, limits));
// 1300 - 400 + 320 = 1220, then 1220 - 400 + 320 = 1140, then 1060.
try std.testing.expectEqual(@as(usize, 3), oldestImagesToWithhold(&sizes, 1300, limits));
try std.testing.expectEqual(@as(usize, 3), oldestImagesToWithhold(&sizes, 50_000, limits));
try std.testing.expectEqual(@as(usize, 0), oldestImagesToWithhold(&.{}, 50_000, limits));
}
test "oldest images leave when a request carries too many" {
const limits = Limits{ .max_request_bytes = 1_000_000, .max_images = 2 };
const sizes = [_]usize{ 10, 10, 10, 10 };
try std.testing.expectEqual(@as(usize, 2), oldestImagesToWithhold(&sizes, 100, limits));
try std.testing.expectEqual(@as(usize, 0), oldestImagesToWithhold(sizes[0..2], 100, limits));
}
fn testToolImage(arena: Allocator, encoded_len: usize) !types.ToolImage {
const data = try arena.alloc(u8, encoded_len);
@memset(data, 'A');
return .{ .data = data, .mime_type = try arena.dupe(u8, "image/png") };
}
test "withholding keeps history and names how to reload each image" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
const inline_bytes = try arena.alloc(u8, 3000);
@memset(inline_bytes, 0);
const attachments = [_]types.ImageAttachment{.{
.id = 1,
.path = try arena.dupe(u8, "shot.png"),
.media_type = try arena.dupe(u8, "image/png"),
.inline_data = inline_bytes,
}};
const tool_images = [_]types.ToolImage{ try testToolImage(arena, 4000), try testToolImage(arena, 4000) };
const messages = [_]ChatMessage{
.{ .role = .user, .content = "compare these", .images = &attachments },
.{ .role = .tool, .content = "<path>a.png</path>", .tool_result_memory = .{ .tool_images = &tool_images, .tool_image_handle = "image-result-call_a" } },
};
// 12,500 bytes against 9,000: the attachment (4,000 encoded) is enough.
const first = try withholdOldestImages(arena, &messages, 12_500, .{ .max_request_bytes = 9000 }, 4096);
try std.testing.expectEqual(@as(usize, 1), first.withheld);
try std.testing.expectEqual(@as(usize, 0), first.messages[0].images.len);
try std.testing.expect(std.mem.startsWith(u8, first.messages[0].content.?, "[Image #1 not sent to keep the request under the provider's size limit.]\n"));
try std.testing.expect(std.mem.endsWith(u8, first.messages[0].content.?, "compare these"));
try std.testing.expectEqual(@as(usize, 2), first.messages[1].tool_result_memory.?.tool_images.len);
// Against 5,000 every image leaves, oldest first.
const second = try withholdOldestImages(arena, &messages, 12_500, .{ .max_request_bytes = 5000 }, 4096);
try std.testing.expectEqual(@as(usize, 3), second.withheld);
try std.testing.expectEqual(@as(usize, 0), second.messages[1].tool_result_memory.?.tool_images.len);
try std.testing.expectEqualStrings(
"[2 images from this result not sent to keep the request under the provider's size limit. Use read_tool_result with handle image-result-call_a to load them again.]\n<path>a.png</path>",
second.messages[1].content.?,
);
// The caller's messages are unchanged.
try std.testing.expectEqual(@as(usize, 1), messages[0].images.len);
try std.testing.expectEqual(@as(usize, 2), messages[1].tool_result_memory.?.tool_images.len);
try std.testing.expectEqualStrings("compare these", messages[0].content.?);
// A request that already fits is returned as is.
const unchanged = try withholdOldestImages(arena, &messages, 9000, .{ .max_request_bytes = 9000 }, 4096);
try std.testing.expectEqual(@as(usize, 0), unchanged.withheld);
try std.testing.expectEqual(@as([*]const ChatMessage, &messages), unchanged.messages.ptr);
}
test "withheld snapshot attachments point at their saved file" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
var notice: std.Io.Writer.Allocating = .init(arena);
try writeAttachmentNotice(&notice.writer, .{
.id = 3,
.path = try arena.dupe(u8, "screen.png"),
.media_type = try arena.dupe(u8, "image/png"),
.snapshot_path = try arena.dupe(u8, "/home/u/.fx/sessions/s/images/3.bin"),
});
try std.testing.expectEqualStrings(
"[Image #3 not sent to keep the request under the provider's size limit. The original is saved at /home/u/.fx/sessions/s/images/3.bin; read_file it to see it again.]\n",
notice.written(),
);
}
test "a withheld tool image without a saved handle says so plainly" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
try std.testing.expectEqualStrings(
"[1 image from this result not sent to keep the request under the provider's size limit.]\n",
try toolImagesNotice(arena, 1, null),
);
try std.testing.expectEqualStrings(
"[1 image from this result not sent to keep the request under the provider's size limit. Use read_tool_result with handle image-result-a to load it again.]\n",
try toolImagesNotice(arena, 1, "image-result-a"),
);
}
src/core/agent/runtime/response_language.zig
+2 -1
View File
@@ -1,5 +1,6 @@
const std = @import("std");
const language_script = @import("../../shared/language_script.zig");
const text_utils = @import("../../shared/text_utils.zig");
const minimum_letters: usize = 5;
const minimum_unexpected_prose_letters: usize = 8;
@@ -167,7 +168,7 @@ test "response language evidence distinguishes clear scripts" {
);
try std.testing.expectEqual(
Script.latin,
evidence("English transcript payload with sparse 界 markers. " ** 16).script.?,
evidence(text_utils.repeat("English transcript payload with sparse 界 markers. ", 16)).script.?,
);
}
Loaded 100 of 525 files, more files were not shown because too many files have changed in this diff. Show more