mirror of
https://github.com/vercel-labs/fx.git
synced 2026-10-28 10:15:17 +08:00
Bring the remaining fx TLA+ models into tests/tla
- Add models, checkers, and Bun drivers for ACP permission modes and speed, launch flags, durable libfx, the MCP-v2 engine and the MCP host, /usage, the shell tool and git admission, the shell snapshot and terminal lifecycle, the libfx provider credential edge, the indexer's grep, file picker, and snapshot reuse, and the session rollout. - Add tests/tla/lib/ and tests/tla/drivers/lib/ so every checker and driver finds TLC and Java the same way, gives each TLC run its own temp folder, runs the binary from this checkout, and keeps sound off. - Add tests/tla/mcp/rules.md, the MCP specification rules the MCP-v2 models cite by ID. - Mark the models that describe the next MCP-v2 version, and run their drivers only with --next or FX_TLA_NEXT_MCP=1. - Point the /usage and MCP trace comments in src/ at their models, and update tests/tla/README.md, tests/AGENTS.md, and CONTRIBUTING.md.
This commit is contained in:
348 files changed
+27774
-91
No files matched your search
+3
-2
@@ -114,11 +114,12 @@ python3 benchmarks/first_frame.py --binary /tmp/fx-before --binary ./zig-out/bin
|
||||
|
||||
### Check a TLA+ model
|
||||
|
||||
`tests/tla/` holds TLA+ models of the session manager, the workspace indexer, and libfx durable sessions and steering. CI runs the libfx model; run the matching checker yourself when you change behavior that one of the others describes. You need Java and `tla2tools.jar`, and `tests/tla/README.md` covers setup.
|
||||
`tests/tla/` holds TLA+ models of fx, such as the session manager, the MCP engine and host, ACP, `/usage`, the shell tool, the workspace indexer, and libfx. Most come with Bun drivers that run the real binary and have TLC check what it did. CI runs the libfx model; run the matching checker yourself when you change behavior that one of the others describes. You need Java and `tla2tools.jar`, and `tests/tla/README.md` lists every model and covers setup.
|
||||
|
||||
```bash
|
||||
tests/tla/session_manager/check.sh SessionLog
|
||||
tests/tla/indexer/check.sh
|
||||
tests/tla/mcp/check.sh --model Engine
|
||||
bun test ./tests/tla/drivers/shell_command/
|
||||
```
|
||||
|
||||
### Run the model evals
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
//! JSONL trace of core steps (format version 1).
|
||||
//!
|
||||
//! The I/O layer writes one line per `step` call when tracing is on. The Bun
|
||||
//! bridge turns a trace into a TLA+ observation and asks TLC whether it is a
|
||||
//! behavior of the machine's model. `data` carries only model projections
|
||||
//! (ids, versions, flags, counters), never tokens or user content.
|
||||
//! bridge in `tests/tla/drivers/mcp/bridge.ts` turns a trace into a TLA+
|
||||
//! observation and asks TLC whether it is a behavior of the machine's model
|
||||
//! in `tests/tla/mcp/`. `data` carries only model projections (ids, versions,
|
||||
//! flags, counters), never tokens or user content.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! When `/usage` fetches, and what one fetch asks for. Pure; mirrors
|
||||
//! `FloorClear`, `Open`, `Manual`, `Timer` and `StartFetch` in the
|
||||
//! LeanUsage TLA+ model.
|
||||
//! LeanUsage TLA+ model (`tests/tla/usage/LeanUsage.tla`).
|
||||
|
||||
const std = @import("std");
|
||||
const report = @import("report.zig");
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
//! What `/usage` shows for one period, built from a cache entry. Pure;
|
||||
//! mirrors `ViewOf` in the LeanUsage TLA+ model: today's numbers count only
|
||||
//! mirrors `ViewOf` in `tests/tla/usage/LeanUsage.tla`: today's numbers count only
|
||||
//! when they belong to the current UTC day (`TodayIsToday`).
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
+11
-8
@@ -32,10 +32,13 @@ tests/
|
||||
│ └── render-lab/ rendering evidence tool
|
||||
├── evals/ model-backed evals, run only on request
|
||||
└── tla/ TLA+ models, their checkers, and their Bun drivers
|
||||
├── session_manager/ one folder per model, named after the module it describes
|
||||
├── indexer/
|
||||
├── sdk/
|
||||
└── drivers/ Bun drivers that walk real code along a model's state graph
|
||||
├── session_manager/ one folder per model area, named after the module it describes,
|
||||
├── mcp/ each with a check.sh; tla/README.md lists them all
|
||||
├── ...
|
||||
├── lib/ tlc.sh and run-tlc.sh: find TLC and run configs with their verdicts
|
||||
└── drivers/ Bun drivers that run real code and have TLC check each run
|
||||
├── lib/ tlc.ts and context.ts, shared by every driver
|
||||
└── <area>/ one folder per model area
|
||||
```
|
||||
|
||||
Only files that end in `.test.ts` at the root of `tests/e2e/` are E2E tests. CI discovers them there and nowhere else, so a test in a subfolder never runs in CI.
|
||||
@@ -75,10 +78,10 @@ python3 -m scripts.pgso.corpus --manifest scripts/pgso/corpus.json --list # fr
|
||||
|
||||
Every TLA+ spec, its TLC checker, the Bun drivers that walk code along a spec, and the design record a spec checks live in `tests/tla/`, and nowhere else. `tests/tla/README.md` lists the requirements and commands.
|
||||
|
||||
- Name a model's folder after the module it describes: `session_manager/` for `src/core/session_manager/`, `indexer/` for `src/core/indexer/`, and `sdk/` for libfx.
|
||||
- Give each folder a `check.sh` that runs every config and exits non-zero when one misses its expected verdict. Configs that must fail, such as witnesses and planted mutations, are part of that contract, so encode their verdicts in `check.sh`.
|
||||
- Put Bun drivers in `tests/tla/drivers/`. They import the code under test by relative path and read the state graphs a model's `check.sh` writes, such as `tests/tla/sdk/graphs/`. Generated graphs are ignored by git.
|
||||
- Find TLC through `TLA2TOOLS_JAR` and Java through `JAVA_HOME`, then Homebrew's OpenJDK, then `PATH`, as the shell scripts do.
|
||||
- Name a model's folder after the module it describes: `session_manager/` for `src/core/session_manager/`, `mcp_host/` for `src/core/mcp_host/`, and `sdk/` for libfx.
|
||||
- Give each folder a `check.sh` that runs every config and exits non-zero when one misses its expected verdict. Configs that must fail, such as witnesses and planted mutations, are part of that contract, so encode their verdicts in `check.sh`. Source `../lib/tlc.sh` and list each config with its verdict for `tlc_verdicts`, rather than finding TLC yourself.
|
||||
- Put Bun drivers in `tests/tla/drivers/<area>/`. They run the code under test, from `zig-out/bin/fx` or by relative import, and have TLC check each run. Use `drivers/lib/tlc.ts` to run TLC and `drivers/lib/context.ts` for the binary, the output folder, and the `tests/e2e/` helpers. Every fx process a driver starts sets `FX_SOUND=0`. Generated output, such as `tests/tla/sdk/graphs/`, goes to the temp folder or is ignored by git.
|
||||
- A model of code that has not landed says so at the top, and the drivers that need that code run only behind an explicit flag, such as `--next` or `FX_TLA_NEXT_MCP=1`. `tests/tla/README.md` lists them. Remove the flag in the change that lands the code.
|
||||
- When a change alters behavior that a spec models, update the spec in the same change and run its checker. CI runs only the `sdk/` model and its drivers; run the others yourself.
|
||||
- A new model, checker, or driver gets a row in `tests/tla/README.md`.
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
__pycache__/
|
||||
+42
-8
@@ -4,10 +4,19 @@ TLA+ specifications for fx, checked with TLC, together with the scripts and Bun
|
||||
|
||||
| Folder | What it models | Checker |
|
||||
|---|---|---|
|
||||
| `session_manager/` | The session log, turn lifecycle, writer lock, snapshots, fork, subagents, catalog, host wiring, the side-file move, and v1 conversion. `system-design.md` is the design record the specs check and defines the `D1` to `D62` decision numbers that code comments cite. | `session_manager/check.sh` |
|
||||
| `indexer/` | Refreshing a scanned root, and when a previous listing may be reused. | `indexer/check.sh` |
|
||||
| `sdk/` | libfx durable sessions (`DurableSession.tla`), agent names and routing (`AgentRouting.tla`), and steering (`DurableSteer.tla`). `check.sh` also writes the TLC state graphs the drivers walk. | `sdk/check.sh`, then the drivers |
|
||||
| `drivers/` | The Bun drivers. `drive.mjs`, `drive-agents.mjs`, and `drive-steer.mjs` walk the real SDK code along the `sdk/` state graphs and check every step against the model. | `bun drivers/drive.mjs`, `bun drivers/drive-agents.mjs` |
|
||||
| `acp/` | The ACP permission modes, which match the CLI's `/permissions`, and the per-session Standard, Fast, and Ultrafast speed (`src/acp/`). | `acp/check.sh`, then the drivers in `drivers/acp/` |
|
||||
| `cli/` | How `fx` and `fx ask` handle `--model`, `--fast`, `--ultrafast`, and `--effort` at launch, for the current design and a proposed one that validates them. | `cli/check.sh`, `bun test drivers/cli/` |
|
||||
| `durable/` | Durable libfx sessions (`sdk/durable/`): the journal, tool replay, parallel batches, fencing, the inbox, resumption, and snapshots. | `durable/check.sh`, `durable/mutants.py` |
|
||||
| `indexer/` | When a scanned root's listing may be reused, merging parallel grep results, the file picker's loading row, and listing snapshots (`src/core/indexer/`). | `indexer/check.sh` |
|
||||
| `mcp/` | The MCP-v2 engine (`src/mcp/`): wire format, requests, stdio, version detection, the tool catalog, both HTTP transports, input-required rounds, authorization, subscriptions, elicitation, and the engine. `rules.md` lists the MCP specification rules the models cite by ID. | `mcp/check.sh`, then the drivers in `drivers/mcp/` |
|
||||
| `mcp_host/` | fx's MCP host (`src/core/mcp_host/`): host lifecycle, saved credentials, logins shared between processes, sign-in, and elicitation routing. | `mcp_host/check.sh`, `bun test drivers/mcp_host/` |
|
||||
| `sdk/` | libfx durable sessions (`DurableSession.tla`), agent names and routing (`AgentRouting.tla`), steering (`DurableSteer.tla`), and the provider credential edge (`ProviderAuth.tla`). `check.sh` also writes the TLC state graphs the drivers walk. | `sdk/check.sh`, then the drivers |
|
||||
| `session_manager/` | The session log, turn lifecycle, writer lock, snapshots, fork, subagents, catalog, host wiring, the side-file move, and v1 conversion. `rollout/` holds the models of the changes that moved fx onto the session manager. `system-design.md` is the design record the specs check and defines the `D1` to `D62` decision numbers that code comments cite. | `session_manager/check.sh`, `session_manager/rollout/run.py`, `bun test drivers/session_manager/` |
|
||||
| `shell_command/` | How an agent git command is admitted and run (`src/core/shell_command/safe_git.zig`), and one model turn through the shell tool, for the current design and a redesign. | `shell_command/check.sh`, `bun test drivers/shell_command/` |
|
||||
| `terminal/` | The in-memory shell snapshot owner (`src/core/terminal/shell_snapshot.zig`) and the lifecycle of hosted terminals. | `terminal/check.sh` |
|
||||
| `usage/` | `/usage` refreshes and the shared usage cache (`src/usage/`): `LeanUsage.tla` and `UsageModel.tla`. | `usage/check.sh`, then the drivers in `drivers/usage/` |
|
||||
| `drivers/` | The Bun drivers. They run the real fx binary or SDK code, record what happens, and have TLC check each run against its model. `drive.mjs`, `drive-agents.mjs`, and `drive-steer.mjs` walk the SDK code along the `sdk/` state graphs. | See each folder's `check.sh` |
|
||||
| `lib/` | `tlc.sh`, which finds TLC and Java and runs a table of configs with their expected verdicts, and `run-tlc.sh`, which runs TLC once for the drivers. | |
|
||||
|
||||
## Requirements
|
||||
|
||||
@@ -20,15 +29,35 @@ curl -fL -o ~/.local/share/tla/tla2tools.jar \
|
||||
https://github.com/tlaplus/tlaplus/releases/latest/download/tla2tools.jar
|
||||
```
|
||||
|
||||
Set `TLA2TOOLS_JAR` to use a jar in another location. The shell scripts look for Java in `JAVA_HOME`, then in Homebrew's OpenJDK, then on `PATH`.
|
||||
Set `TLA2TOOLS_JAR` to use a jar in another location. The scripts look for Java in `JAVA_HOME`, then in Homebrew's OpenJDK, then on `PATH`. The `mcp/`, `durable/`, and `session_manager/` runners also need Python 3.
|
||||
|
||||
The SDK drivers also need Bun, the native addon, and the durable World packages:
|
||||
The drivers need Bun. Most of them also run the binary that `zig build` puts in `zig-out/bin/fx`, and the terminal ones need tmux. They take these settings from the environment:
|
||||
|
||||
| Variable | Default | Effect |
|
||||
|---|---|---|
|
||||
| `FX_BIN` | `zig-out/bin/fx` | The binary under test |
|
||||
| `FX_WORKTREE` | this checkout | The checkout whose binary and `tests/e2e/` helpers a driver uses |
|
||||
| `FX_TLA_OUT` | a folder under the system temp folder | Where traces, TLC logs, and reports go |
|
||||
| `FX_SOUND` | `0` in every driver | Keep it at `0`; sound playback can overload a machine during a run |
|
||||
|
||||
The SDK drivers also need the native addon and the durable World packages:
|
||||
|
||||
```sh
|
||||
zig build libfx-napi -Dnapi-surface=core
|
||||
npm ci --prefix sdk/durable
|
||||
```
|
||||
|
||||
## Models of code that has not landed
|
||||
|
||||
Some models describe the next MCP-v2 version, which `src/mcp/` and `src/core/mcp_host/` do not have yet. Their model checks pass, but the real code does not match them until that version lands:
|
||||
|
||||
- In `mcp/`, `Engine`, `EngineTrace`, `HttpModern`, `HttpModernTrace`, `HttpLegacy`, and `StdioConn` say so at the top. `drivers/mcp/labs/engine_lab.zig` does not build against `src/mcp/` yet, so `drivers/mcp/driver.ts` runs its scenarios only with `--next`.
|
||||
- In `drivers/mcp_host/`, the tests marked `NEXT_MCP` run only with `FX_TLA_NEXT_MCP=1`.
|
||||
|
||||
The MCP-v2 models of the two HTTP transports, input-required rounds, authorization, subscriptions, and elicitation are model-checked only. Their traces came from a conformance adapter that is not part of fx. The `drivers/mcp_host/` sign-in tests drive real fx over HTTP against the fake server in `drivers/mcp/fakes/`.
|
||||
|
||||
The `cli/` and `shell_command/` tests also check a proposed design. Those checks run only with `FX_TLA_PROPOSED=1`, and they fail until fx implements the design.
|
||||
|
||||
## Run
|
||||
|
||||
From the repository root:
|
||||
@@ -38,11 +67,16 @@ tests/tla/session_manager/check.sh # every session spec
|
||||
tests/tla/session_manager/check.sh SessionLog Fork # selected specs
|
||||
tests/tla/session_manager/mutants.py # every planted bug must be caught
|
||||
tests/tla/session_manager/trace/selftest.sh # the trace validator must accept and reject
|
||||
tests/tla/indexer/check.sh # every refresh config and its expected verdict
|
||||
tests/tla/session_manager/rollout/run.py check # the rollout models; mutate for their planted bugs
|
||||
tests/tla/indexer/check.sh # every config and its expected verdict
|
||||
tests/tla/acp/check.sh # likewise for acp/, cli/, durable/, shell_command/, terminal/, usage/
|
||||
tests/tla/mcp/check.sh # every MCP-v2 model and planted bug
|
||||
tests/tla/mcp_host/check.sh # every MCP host model, planted bug, and finding
|
||||
tests/tla/sdk/check.sh # the SDK specs; writes the state graphs the drivers walk
|
||||
bun tests/tla/drivers/drive.mjs send --walks 40 # one durable-session graph; see .github/workflows/ci.yml for the rest
|
||||
bun tests/tla/drivers/drive-agents.mjs send --walks 60 # agent routing
|
||||
bun tests/tla/drivers/drive-steer.mjs ideal --walks 20 # steering
|
||||
bun test ./tests/tla/drivers/shell_command/ # a driver written as bun tests; see each check.sh for its own
|
||||
```
|
||||
|
||||
`Fork` and `V1Conversion` can run for many minutes. `session_manager/README.md` describes each session spec, the planted bugs, and the trace checkers.
|
||||
A `check.sh` that uses `lib/tlc.sh` takes config names to run only those, for example `tests/tla/usage/check.sh LeanUsage`. `Fork`, `V1Conversion`, and the full `LeanUsage` config can run for many minutes. `session_manager/README.md` describes each session spec, the planted bugs, and the trace checkers.
|
||||
@@ -0,0 +1,210 @@
|
||||
------------------------- MODULE AcpPermissionModes -------------------------
|
||||
(***************************************************************************)
|
||||
(* fx's ACP permission-mode contract, matching the CLI's /permissions. *)
|
||||
(* *)
|
||||
(* - ACP lists exactly the CLI's three modes, in this order: auto (the *)
|
||||
(* default), ask, full-access, named Auto, Ask, Full access. Both the *)
|
||||
(* `mode` config option and the older `modes` field list them and show *)
|
||||
(* the same current mode. *)
|
||||
(* - A session (new, resumed, or new in a fresh process) starts in the *)
|
||||
(* profile's saved permission_mode as it is when the session starts; *)
|
||||
(* an unset profile means auto. Another fx process saving a mode does *)
|
||||
(* not change a session that is already open. *)
|
||||
(* - An accepted change, through session/set_config_option or the older *)
|
||||
(* session/set_mode, switches the session and saves the mode as the *)
|
||||
(* profile's default, as /permissions does in the CLI. *)
|
||||
(* - Any other value is rejected and changes nothing. *)
|
||||
(* - A prompt is enforced by the mode the client was shown: ask asks the *)
|
||||
(* client, auto sends the action to fx's automatic reviewer, and full *)
|
||||
(* access runs it without asking. *)
|
||||
(* - Mode option values carry only ACP's fields (value, name, *)
|
||||
(* description, _meta). *)
|
||||
(* *)
|
||||
(* `last` is what the client observes for the latest step; `stored` is *)
|
||||
(* the profile's saved mode read back from settings.json after the step. *)
|
||||
(* *)
|
||||
(* Assumptions: one fx acp process at a time drives one active session; *)
|
||||
(* other fx processes may save a mode between steps (External); resume *)
|
||||
(* follows a prompt (fx saves a chat at its first turn). *)
|
||||
(***************************************************************************)
|
||||
EXTENDS Naturals, Sequences
|
||||
|
||||
CONSTANTS Profiles, \* saved modes a run may start from: Modes \cup {"unset"}
|
||||
Values \* values a client may send: Modes plus unknown ones
|
||||
|
||||
ModeSeq == <<"auto", "ask", "full-access">>
|
||||
Modes == {ModeSeq[k] : k \in 1..Len(ModeSeq)}
|
||||
ModesCsv == "auto,ask,full-access"
|
||||
NamesCsv == "Auto,Ask,Full access"
|
||||
DefaultMode == "auto"
|
||||
|
||||
Eff(p) == IF p = "unset" THEN DefaultMode ELSE p
|
||||
|
||||
\* How the prompt probe (a shell call that writes a file) is handled. The
|
||||
\* client declines permission requests and the reviewer clears the action.
|
||||
Enforce(m) ==
|
||||
CASE m = "ask" -> "asked/no-run"
|
||||
[] m = "auto" -> "reviewed/ran"
|
||||
[] m = "full-access" -> "direct/ran"
|
||||
|
||||
VARIABLES profile, cur, saved, last
|
||||
vars == <<profile, cur, saved, last>>
|
||||
|
||||
NoView == [current |-> "", legacy |-> "", listed |-> "", legacyListed |-> "",
|
||||
names |-> "", schemaOk |-> TRUE]
|
||||
|
||||
\* session/new and session/resume responses carry configOptions and modes.
|
||||
SessionView(m) == [current |-> m, legacy |-> m, listed |-> ModesCsv,
|
||||
legacyListed |-> ModesCsv, names |-> NamesCsv, schemaOk |-> TRUE]
|
||||
|
||||
\* set_config_option responses carry configOptions only.
|
||||
ConfigView(m) == [SessionView(m) EXCEPT !.legacy = "n/a", !.legacyListed = "n/a"]
|
||||
|
||||
\* `fx status --json` lists the modes and the one a new session starts in.
|
||||
StatusView(m) == ConfigView(m)
|
||||
|
||||
Obs(op, arg, ok, hasView, view, enforce, stored) ==
|
||||
[op |-> op, arg |-> arg, ok |-> ok, hasView |-> hasView, view |-> view,
|
||||
enforce |-> enforce, stored |-> stored]
|
||||
|
||||
Init ==
|
||||
/\ profile \in Profiles
|
||||
/\ cur = Eff(profile)
|
||||
/\ saved = FALSE
|
||||
/\ last = Obs("new", "none", TRUE, TRUE, SessionView(Eff(profile)), "none", profile)
|
||||
|
||||
SetConfig(v) ==
|
||||
IF v \in Modes
|
||||
THEN /\ cur' = v
|
||||
/\ profile' = v
|
||||
/\ last' = Obs("set", v, TRUE, TRUE, ConfigView(v), "none", v)
|
||||
/\ UNCHANGED saved
|
||||
ELSE /\ last' = Obs("set", v, FALSE, FALSE, NoView, "none", profile)
|
||||
/\ UNCHANGED <<profile, cur, saved>>
|
||||
|
||||
\* session/set_mode answers null, so the client sees no view.
|
||||
SetLegacy(v) ==
|
||||
IF v \in Modes
|
||||
THEN /\ cur' = v
|
||||
/\ profile' = v
|
||||
/\ last' = Obs("legacy", v, TRUE, FALSE, NoView, "none", v)
|
||||
/\ UNCHANGED saved
|
||||
ELSE /\ last' = Obs("legacy", v, FALSE, FALSE, NoView, "none", profile)
|
||||
/\ UNCHANGED <<profile, cur, saved>>
|
||||
|
||||
Prompt ==
|
||||
/\ saved' = TRUE
|
||||
/\ last' = Obs("prompt", "none", TRUE, FALSE, NoView, Enforce(cur), profile)
|
||||
/\ UNCHANGED <<profile, cur>>
|
||||
|
||||
\* A new fx acp process resumes the session.
|
||||
Resume ==
|
||||
/\ saved
|
||||
/\ LET resumed == Eff(profile) IN
|
||||
/\ cur' = resumed
|
||||
/\ last' = Obs("resume", "none", TRUE, TRUE, SessionView(resumed), "none", profile)
|
||||
/\ UNCHANGED <<profile, saved>>
|
||||
|
||||
\* A new fx acp process starts a new session.
|
||||
Fresh ==
|
||||
/\ LET started == Eff(profile) IN
|
||||
/\ cur' = started
|
||||
/\ last' = Obs("fresh", "none", TRUE, TRUE, SessionView(started), "none", profile)
|
||||
/\ saved' = FALSE
|
||||
/\ UNCHANGED profile
|
||||
|
||||
\* session/new in the running process.
|
||||
NewSession ==
|
||||
/\ LET opened == Eff(profile) IN
|
||||
/\ cur' = opened
|
||||
/\ last' = Obs("new", "none", TRUE, TRUE, SessionView(opened), "none", profile)
|
||||
/\ saved' = FALSE
|
||||
/\ UNCHANGED profile
|
||||
|
||||
\* Another fx process (the CLI's /permissions, or another ACP process)
|
||||
\* saves a mode. The open session keeps its own, like a second CLI window.
|
||||
External(v) ==
|
||||
/\ profile' = v
|
||||
/\ last' = Obs("other", v, TRUE, FALSE, NoView, "none", v)
|
||||
/\ UNCHANGED <<cur, saved>>
|
||||
|
||||
\* `fx status --json` in the workspace, run beside the ACP process.
|
||||
Status ==
|
||||
/\ last' = Obs("status", "none", TRUE, TRUE, StatusView(Eff(profile)), "none", profile)
|
||||
/\ UNCHANGED <<profile, cur, saved>>
|
||||
|
||||
Next ==
|
||||
\/ \E v \in Values : SetConfig(v) \/ SetLegacy(v)
|
||||
\/ \E v \in Modes : External(v)
|
||||
\/ Prompt
|
||||
\/ Resume
|
||||
\/ Fresh
|
||||
\/ NewSession
|
||||
\/ Status
|
||||
|
||||
Spec == Init /\ [][Next]_vars
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
(* Safety properties. EnforceTable restates the expected handling so a *)
|
||||
(* change to Enforce is caught. *)
|
||||
|
||||
EnforceTable == [m \in Modes |->
|
||||
IF m = "ask" THEN "asked/no-run"
|
||||
ELSE IF m = "auto" THEN "reviewed/ran"
|
||||
ELSE "direct/ran"]
|
||||
|
||||
SessionOps == {"new", "resume", "fresh"}
|
||||
ChangeOps == {"set", "legacy"}
|
||||
|
||||
ListsExactlyCliModes ==
|
||||
last.hasView =>
|
||||
/\ last.view.listed = ModesCsv
|
||||
/\ last.view.names = NamesCsv
|
||||
/\ last.view.legacyListed \in {ModesCsv, "n/a"}
|
||||
|
||||
SchemaClean == last.hasView => last.view.schemaOk
|
||||
|
||||
ReportsEnforcedMode ==
|
||||
(last.hasView /\ last.op # "status") => /\ last.view.current = cur
|
||||
/\ last.view.legacy \in {cur, "n/a"}
|
||||
|
||||
\* Status describes the next session, which may differ from an open one.
|
||||
StatusMatchesNextSession == last.op = "status" => last.view.current = Eff(profile)
|
||||
|
||||
SessionsStartFromProfile ==
|
||||
last.op \in SessionOps => /\ cur = Eff(profile)
|
||||
/\ last.view.legacy = cur
|
||||
/\ last.view.legacyListed = ModesCsv
|
||||
|
||||
DefaultIsAuto ==
|
||||
(last.op \in SessionOps /\ profile = "unset") => cur = "auto"
|
||||
|
||||
UnknownRejected == last.op \in ChangeOps => (last.ok <=> last.arg \in Modes)
|
||||
|
||||
ChangeIsSaved ==
|
||||
(last.op \in ChangeOps /\ last.ok) => /\ cur = last.arg
|
||||
/\ profile = last.arg
|
||||
|
||||
StoredIsProfile == last.stored = profile
|
||||
|
||||
PromptEnforcesMode == last.op = "prompt" => last.enforce = EnforceTable[cur]
|
||||
|
||||
TypeOK ==
|
||||
/\ profile \in Modes \cup {"unset"}
|
||||
/\ cur \in Modes
|
||||
/\ saved \in BOOLEAN
|
||||
|
||||
\* A rejected change leaves the session and the profile as they were.
|
||||
RejectedChangesNothing ==
|
||||
[][(last'.op \in ChangeOps /\ ~last'.ok) => UNCHANGED <<profile, cur>>]_vars
|
||||
|
||||
\* Another process's choice never changes the open session.
|
||||
OpenSessionKeepsMode == [][last'.op = "other" => cur' = cur]_vars
|
||||
|
||||
\* A session only enters full access when the user picks it, or when it
|
||||
\* starts while full access is the saved mode.
|
||||
FullAccessOnlyByChoice ==
|
||||
[][(cur' = "full-access" /\ cur # "full-access") =>
|
||||
\/ (last'.op \in ChangeOps /\ last'.arg = "full-access")
|
||||
\/ (last'.op \in SessionOps /\ profile' = "full-access")]_vars
|
||||
=============================================================================
|
||||
@@ -0,0 +1,50 @@
|
||||
--------------------- MODULE AcpPermissionModesTrace ---------------------
|
||||
(***************************************************************************)
|
||||
(* Trace validation: replays one observed run of the real `fx acp` binary *)
|
||||
(* against AcpPermissionModes. TraceLog comes from the generated *)
|
||||
(* AcpPermissionModesObs module; each element has the fields of Obs. *)
|
||||
(* *)
|
||||
(* NotDone is a sentinel: TLC reporting it violated means every *)
|
||||
(* observation matched a spec step. Exhausting the search without that *)
|
||||
(* violation means the run is not a behavior of the spec. *)
|
||||
(***************************************************************************)
|
||||
EXTENDS AcpPermissionModes, AcpPermissionModesObs, Sequences
|
||||
|
||||
VARIABLE i
|
||||
|
||||
traceVars == <<vars, i>>
|
||||
|
||||
Match(o) ==
|
||||
/\ last.op = o.op
|
||||
/\ last.arg = o.arg
|
||||
/\ last.ok = o.ok
|
||||
/\ last.hasView = o.hasView
|
||||
/\ o.hasView => (last.view = o.view)
|
||||
/\ last.enforce = o.enforce
|
||||
/\ last.stored = o.stored
|
||||
|
||||
StepFor(o) ==
|
||||
CASE o.op = "set" -> SetConfig(o.arg)
|
||||
[] o.op = "legacy" -> SetLegacy(o.arg)
|
||||
[] o.op = "prompt" -> Prompt
|
||||
[] o.op = "resume" -> Resume
|
||||
[] o.op = "fresh" -> Fresh
|
||||
[] o.op = "new" -> NewSession
|
||||
[] o.op = "other" -> External(o.arg)
|
||||
[] o.op = "status" -> Status
|
||||
[] OTHER -> FALSE
|
||||
|
||||
ASSUME Len(TraceLog) >= 1 /\ TraceLog[1].op = "new"
|
||||
|
||||
TraceInit == /\ i = 1
|
||||
/\ Init
|
||||
/\ Match(TraceLog[1])
|
||||
|
||||
TraceNext == /\ i < Len(TraceLog)
|
||||
/\ \E o \in {TraceLog[i + 1]} :
|
||||
/\ StepFor(o)
|
||||
/\ Match(o)'
|
||||
/\ i' = i + 1
|
||||
|
||||
NotDone == i < Len(TraceLog)
|
||||
=============================================================================
|
||||
@@ -0,0 +1,138 @@
|
||||
------------------------------- MODULE AcpSpeed -------------------------------
|
||||
(***************************************************************************)
|
||||
(* Per-session speed contract for fx's ACP server, mirroring the CLI: *)
|
||||
(* one Standard / Fast / Ultrafast choice per chat. *)
|
||||
(* *)
|
||||
(* - A client sees the `fast` option only for models that offer Fast and *)
|
||||
(* the `ultrafast` option only for models that offer Ultrafast. *)
|
||||
(* - Turning one lane on turns the other off. *)
|
||||
(* - Enabling a lane the model lacks is rejected; disabling always works. *)
|
||||
(* - Switching models turns off a lane the new model lacks. *)
|
||||
(* - The choice is saved with the chat and survives resume. *)
|
||||
(* - Unknown config ids are rejected instead of silently succeeding. *)
|
||||
(* - A prompt carries Ultrafast, else Fast when the model supports it. *)
|
||||
(* *)
|
||||
(* `fast` may start on for a model without Fast when the user's settings *)
|
||||
(* default enables it (CLI parity); it is then hidden and has no effect. *)
|
||||
(* Every value is a string or boolean so TLC never compares mixed types. *)
|
||||
(***************************************************************************)
|
||||
EXTENDS Naturals, TLC
|
||||
|
||||
CONSTANTS Models, SupFast, SupUltra
|
||||
|
||||
ASSUME SupFast \subseteq Models /\ SupUltra \subseteq Models
|
||||
|
||||
VARIABLES model, fast, ultra, last
|
||||
|
||||
vars == <<model, fast, ultra, last>>
|
||||
|
||||
Ops == {"new", "fast", "ultrafast", "model", "unknown", "prompt", "resume"}
|
||||
Speeds == {"none", "standard", "fast", "ultrafast"}
|
||||
|
||||
Bool(b) == IF b THEN "true" ELSE "false"
|
||||
|
||||
(* What a client reads from configOptions after a step. *)
|
||||
View == [model |-> model,
|
||||
fast |-> IF model \in SupFast THEN Bool(fast) ELSE "absent",
|
||||
ultra |-> IF model \in SupUltra THEN Bool(ultra) ELSE "absent"]
|
||||
|
||||
(* The lane a prompt sends to the Gateway. *)
|
||||
RequestSpeed == IF ultra THEN "ultrafast"
|
||||
ELSE IF fast /\ model \in SupFast THEN "fast"
|
||||
ELSE "standard"
|
||||
|
||||
Result(op, arg, ok, speed) == [op |-> op, arg |-> arg, ok |-> ok, speed |-> speed]
|
||||
|
||||
Init == /\ model \in Models
|
||||
/\ fast \in BOOLEAN
|
||||
/\ ultra = FALSE
|
||||
/\ last = Result("new", "none", TRUE, "none")
|
||||
|
||||
SetFast(b) ==
|
||||
IF b /\ model \notin SupFast
|
||||
THEN /\ last' = Result("fast", Bool(b), FALSE, "none")
|
||||
/\ UNCHANGED <<model, fast, ultra>>
|
||||
ELSE /\ fast' = b
|
||||
/\ ultra' = (ultra /\ ~b)
|
||||
/\ last' = Result("fast", Bool(b), TRUE, "none")
|
||||
/\ UNCHANGED model
|
||||
|
||||
SetUltra(b) ==
|
||||
IF b /\ model \notin SupUltra
|
||||
THEN /\ last' = Result("ultrafast", Bool(b), FALSE, "none")
|
||||
/\ UNCHANGED <<model, fast, ultra>>
|
||||
ELSE /\ ultra' = b
|
||||
/\ fast' = (fast /\ ~b)
|
||||
/\ last' = Result("ultrafast", Bool(b), TRUE, "none")
|
||||
/\ UNCHANGED model
|
||||
|
||||
SetModel(m) ==
|
||||
/\ model' = m
|
||||
/\ fast' = (fast /\ m \in SupFast)
|
||||
/\ ultra' = (ultra /\ m \in SupUltra)
|
||||
/\ last' = Result("model", m, TRUE, "none")
|
||||
|
||||
SetUnknown ==
|
||||
/\ last' = Result("unknown", "none", FALSE, "none")
|
||||
/\ UNCHANGED <<model, fast, ultra>>
|
||||
|
||||
Prompt ==
|
||||
/\ last' = Result("prompt", "none", TRUE, RequestSpeed)
|
||||
/\ UNCHANGED <<model, fast, ultra>>
|
||||
|
||||
(* A new process resumes the saved chat. Assumption: the chat has had a *)
|
||||
(* prompt; the v2 store saves a chat at its first turn, so resuming an *)
|
||||
(* untouched chat fails on main independently of speed. *)
|
||||
Resume ==
|
||||
/\ last' = Result("resume", "none", TRUE, "none")
|
||||
/\ UNCHANGED <<model, fast, ultra>>
|
||||
|
||||
Next == \/ \E b \in BOOLEAN : SetFast(b) \/ SetUltra(b)
|
||||
\/ \E m \in Models : SetModel(m)
|
||||
\/ SetUnknown
|
||||
\/ Prompt
|
||||
\/ Resume
|
||||
|
||||
Spec == Init /\ [][Next]_vars
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
(* Safety properties checked by TLC; planted mutants must violate them. *)
|
||||
|
||||
TypeOK == /\ model \in Models
|
||||
/\ fast \in BOOLEAN
|
||||
/\ ultra \in BOOLEAN
|
||||
/\ last.op \in Ops
|
||||
/\ last.ok \in BOOLEAN
|
||||
/\ last.speed \in Speeds
|
||||
|
||||
(* Fast and Ultrafast are never both on. *)
|
||||
Exclusive == ~(fast /\ ultra)
|
||||
|
||||
(* Ultrafast is never on while hidden: a hidden lane would fail prompts. *)
|
||||
UltraValid == ultra => model \in SupUltra
|
||||
|
||||
(* Fast is never left on after a switch to a model without it. *)
|
||||
SwitchClearsFast == last.op = "model" => (fast => model \in SupFast)
|
||||
|
||||
(* An accepted lane request takes effect. *)
|
||||
RequestHonored ==
|
||||
(last.ok /\ last.op = "fast") => Bool(fast) = last.arg
|
||||
RequestHonoredUltra ==
|
||||
(last.ok /\ last.op = "ultrafast") => Bool(ultra) = last.arg
|
||||
|
||||
(* Only enabling an unsupported lane is rejected. *)
|
||||
RejectOnlyUnsupported ==
|
||||
/\ (last.op = "fast" /\ ~last.ok) => (last.arg = "true" /\ model \notin SupFast)
|
||||
/\ (last.op = "ultrafast" /\ ~last.ok) => (last.arg = "true" /\ model \notin SupUltra)
|
||||
AcceptOnlySupported ==
|
||||
/\ (last.op = "fast" /\ last.ok /\ last.arg = "true") => model \in SupFast
|
||||
/\ (last.op = "ultrafast" /\ last.ok /\ last.arg = "true") => model \in SupUltra
|
||||
|
||||
(* Unknown config ids are errors. *)
|
||||
UnknownRejected == last.op = "unknown" => ~last.ok
|
||||
|
||||
(* A prompt never sends a lane the model lacks. *)
|
||||
PromptLaneSupported ==
|
||||
/\ (last.op = "prompt" /\ last.speed = "fast") => model \in SupFast
|
||||
/\ (last.op = "prompt" /\ last.speed = "ultrafast") => model \in SupUltra
|
||||
=============================================================================
|
||||
@@ -0,0 +1,52 @@
|
||||
---------------------------- MODULE AcpSpeedTrace ----------------------------
|
||||
(***************************************************************************)
|
||||
(* Trace validation: replays one observed run of the real `fx acp` binary *)
|
||||
(* against AcpSpeed. TraceLog comes from the generated AcpSpeedObs module. *)
|
||||
(* Each element is *)
|
||||
(* [op, arg, ok, speed, hasView, view] *)
|
||||
(* where view is the client-visible configOptions projection. Steps that *)
|
||||
(* return no config (errors and prompts) set hasView to FALSE and carry a *)
|
||||
(* placeholder record, so TLC never compares a record with a string. *)
|
||||
(* *)
|
||||
(* NotDone is a sentinel: TLC reporting it violated (exit 12) means every *)
|
||||
(* observation matched a spec step. Exhausting the search without that *)
|
||||
(* violation means the run is not a behavior of the spec. *)
|
||||
(***************************************************************************)
|
||||
EXTENDS AcpSpeed, AcpSpeedObs, Sequences
|
||||
|
||||
VARIABLE i
|
||||
|
||||
traceVars == <<vars, i>>
|
||||
|
||||
Match(o) ==
|
||||
/\ last.op = o.op
|
||||
/\ last.arg = o.arg
|
||||
/\ last.ok = o.ok
|
||||
/\ last.speed = o.speed
|
||||
/\ o.hasView => (View = o.view)
|
||||
|
||||
StepFor(o) ==
|
||||
CASE o.op = "fast" -> SetFast(o.arg = "true")
|
||||
[] o.op = "ultrafast" -> SetUltra(o.arg = "true")
|
||||
[] o.op = "model" -> SetModel(o.arg)
|
||||
[] o.op = "unknown" -> SetUnknown
|
||||
[] o.op = "prompt" -> Prompt
|
||||
[] o.op = "resume" -> Resume
|
||||
[] OTHER -> FALSE
|
||||
|
||||
ASSUME Len(TraceLog) >= 1 /\ TraceLog[1].op = "new"
|
||||
|
||||
TraceInit == /\ i = 1
|
||||
/\ Init
|
||||
/\ Match(TraceLog[1])
|
||||
|
||||
TraceNext == /\ i < Len(TraceLog)
|
||||
/\ \E o \in {TraceLog[i + 1]} :
|
||||
/\ StepFor(o)
|
||||
/\ Match(o)'
|
||||
/\ i' = i + 1
|
||||
|
||||
TraceSpec == TraceInit /\ [][TraceNext]_traceVars
|
||||
|
||||
NotDone == i < Len(TraceLog)
|
||||
=============================================================================
|
||||
@@ -0,0 +1,22 @@
|
||||
CONSTANTS
|
||||
Profiles <- MCProfiles
|
||||
Values <- MCValues
|
||||
INIT Init
|
||||
NEXT Next
|
||||
INVARIANTS
|
||||
ListsExactlyCliModes
|
||||
SchemaClean
|
||||
ReportsEnforcedMode
|
||||
StatusMatchesNextSession
|
||||
SessionsStartFromProfile
|
||||
DefaultIsAuto
|
||||
UnknownRejected
|
||||
ChangeIsSaved
|
||||
StoredIsProfile
|
||||
PromptEnforcesMode
|
||||
TypeOK
|
||||
PROPERTIES
|
||||
RejectedChangesNothing
|
||||
OpenSessionKeepsMode
|
||||
FullAccessOnlyByChoice
|
||||
CHECK_DEADLOCK FALSE
|
||||
@@ -0,0 +1,9 @@
|
||||
------------------------ MODULE MCAcpPermissionModes ------------------------
|
||||
(* Exhaustive model: every starting profile, the three modes plus unknown *)
|
||||
(* values a client might still send (the old "code" mode and the CLI's *)
|
||||
(* internal "yolo" name). *)
|
||||
EXTENDS AcpPermissionModes
|
||||
|
||||
MCProfiles == {"unset", "auto", "ask", "full-access"}
|
||||
MCValues == {"auto", "ask", "full-access", "code", "yolo"}
|
||||
=============================================================================
|
||||
@@ -0,0 +1,22 @@
|
||||
\* Exhaustive check of the speed contract over three catalog shapes:
|
||||
\* a model with both lanes, one with Fast only, and one with neither.
|
||||
CONSTANTS
|
||||
Models <- MCModels
|
||||
SupFast <- MCSupFast
|
||||
SupUltra <- MCSupUltra
|
||||
|
||||
SPECIFICATION Spec
|
||||
|
||||
INVARIANTS
|
||||
TypeOK
|
||||
Exclusive
|
||||
UltraValid
|
||||
SwitchClearsFast
|
||||
RequestHonored
|
||||
RequestHonoredUltra
|
||||
RejectOnlyUnsupported
|
||||
AcceptOnlySupported
|
||||
UnknownRejected
|
||||
PromptLaneSupported
|
||||
|
||||
CHECK_DEADLOCK FALSE
|
||||
@@ -0,0 +1,9 @@
|
||||
----------------------------- MODULE MCAcpSpeed -----------------------------
|
||||
(* Model-checking constants: the same three catalog shapes the runtime *)
|
||||
(* driver serves from its fake Gateway. *)
|
||||
EXTENDS AcpSpeed
|
||||
|
||||
MCModels == {"openai/gpt-6-astra", "openai/gpt-5.6-priority", "anthropic/claude-plain"}
|
||||
MCSupFast == {"openai/gpt-6-astra", "openai/gpt-5.6-priority"}
|
||||
MCSupUltra == {"openai/gpt-6-astra"}
|
||||
=============================================================================
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
# Model-check the ACP models with TLC and require each config's expected
|
||||
# verdict.
|
||||
#
|
||||
# AcpPermissionModes: the ACP permission-mode contract, which matches the CLI's
|
||||
# /permissions. AcpSpeed: the per-session Standard, Fast, and Ultrafast speed
|
||||
# contract. Both configs must pass.
|
||||
#
|
||||
# The planted bugs and the runs of the real `fx acp` binary are in
|
||||
# ../drivers/acp/:
|
||||
# bun ../drivers/acp/permission_modes/check-spec.ts
|
||||
# bun ../drivers/acp/permission_modes/driver.ts
|
||||
# bun ../drivers/acp/speed/check-spec.ts
|
||||
# bun ../drivers/acp/speed/driver.ts
|
||||
#
|
||||
# Needs Java 11+ and tla2tools.jar, as ../README.md describes.
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
# shellcheck source=../lib/tlc.sh
|
||||
. ../lib/tlc.sh
|
||||
|
||||
tlc_verdicts "$@" <<'EOF'
|
||||
MCAcpPermissionModes MCAcpPermissionModes -
|
||||
MCAcpSpeed MCAcpSpeed -
|
||||
EOF
|
||||
@@ -0,0 +1,268 @@
|
||||
---------------------------- MODULE LaunchSelection ----------------------------
|
||||
(***************************************************************************)
|
||||
(* Launch-time model, speed and effort flags for `fx` and `fx ask`. *)
|
||||
(* *)
|
||||
(* One scenario fixes what the user asked for, what the catalog load *)
|
||||
(* returns, and what the chosen model supports. The machine then runs the *)
|
||||
(* entry point: *)
|
||||
(* *)
|
||||
(* ask: AskValidate -> Send *)
|
||||
(* interactive: FirstFrame, then CatalogReady and Submit in the order *)
|
||||
(* the scenario fixes (early = submit before the catalog). *)
|
||||
(* *)
|
||||
(* Send applies the turn-time gates that exist in both designs: Fast is *)
|
||||
(* masked unless the catalog confirms it, Ultra fails the turn unless the *)
|
||||
(* catalog confirms it, and effort is dropped unless the catalog lists it. *)
|
||||
(* *)
|
||||
(* Design = "current" has no launch validation at all. Design = *)
|
||||
(* "proposed" validates at launch. Mutant plants one bug in the proposed *)
|
||||
(* design so each invariant is shown to fail when its rule is broken. *)
|
||||
(***************************************************************************)
|
||||
EXTENDS Naturals, FiniteSets, TLC
|
||||
|
||||
CONSTANTS
|
||||
Design, \* "current" | "proposed"
|
||||
Mutant, \* "none" | "reject_weak" | "no_canonical" | "block_frame" | "persist" | "silent"
|
||||
Class \* "all" | "ideal" | "nonideal": which scenarios Init may pick
|
||||
|
||||
ASSUME Design \in {"current", "proposed"}
|
||||
ASSUME Mutant \in {"none", "reject_weak", "no_canonical", "block_frame", "persist", "silent"}
|
||||
ASSUME Class \in {"all", "ideal", "nonideal"}
|
||||
|
||||
VARIABLE st
|
||||
vars == <<st>>
|
||||
|
||||
(* ----------------------------- Scenarios -------------------------------- *)
|
||||
|
||||
Paths == {"ask", "interactive"}
|
||||
(* none: no --model flag, the configured model is used. *)
|
||||
(* private_hidden: a real team model that only an authenticated catalog *)
|
||||
(* lists; the anonymous public catalog does not. *)
|
||||
ModelReqs == {"none", "exact", "partial_unique", "partial_ambiguous", "unknown", "private_hidden"}
|
||||
SpeedReqs == {"none", "fast", "ultrafast", "off"}
|
||||
EffortReqs == {"none", "supported", "unsupported"}
|
||||
(* absent: a provider without a catalog, such as a configured provider. *)
|
||||
Catalogs == {"authenticated", "anonymous", "failed", "absent"}
|
||||
|
||||
ScenarioSpace ==
|
||||
[path: Paths, model: ModelReqs, speed: SpeedReqs, effort: EffortReqs,
|
||||
catalog: Catalogs, fastCap: BOOLEAN, ultraCap: BOOLEAN, early: BOOLEAN]
|
||||
|
||||
Ideal(s) ==
|
||||
/\ s.catalog = "authenticated"
|
||||
/\ s.model \in {"none", "exact"}
|
||||
/\ s.speed = "fast" => s.fastCap
|
||||
/\ s.speed = "ultrafast" => s.ultraCap
|
||||
/\ s.effort # "unsupported"
|
||||
/\ ~s.early
|
||||
|
||||
Scenarios ==
|
||||
{ s \in ScenarioSpace :
|
||||
/\ s.path = "ask" => ~s.early
|
||||
/\ CASE Class = "all" -> TRUE
|
||||
[] Class = "ideal" -> Ideal(s)
|
||||
[] Class = "nonideal" -> ~Ideal(s) }
|
||||
|
||||
(* ------------------------- Catalog and gateway --------------------------- *)
|
||||
|
||||
Loaded(s) == s.catalog \in {"authenticated", "anonymous"}
|
||||
Authoritative(s) == s.catalog = "authenticated"
|
||||
|
||||
(* What matchCatalogModel returns for the --model text. *)
|
||||
Match(s) ==
|
||||
IF ~Loaded(s) THEN "no_catalog"
|
||||
ELSE CASE s.model = "none" -> "none"
|
||||
[] s.model = "exact" -> "matched"
|
||||
[] s.model = "partial_unique" -> "matched"
|
||||
[] s.model = "partial_ambiguous" -> "ambiguous"
|
||||
[] s.model = "unknown" -> "unknown"
|
||||
[] s.model = "private_hidden" -> IF Authoritative(s) THEN "matched" ELSE "unknown"
|
||||
|
||||
(* Model names: "raw" is the --model text, "canonical" the catalog ID, *)
|
||||
(* "configured" the saved model. An exact ID is already canonical. *)
|
||||
Name(s, m) == IF m = "raw" /\ s.model \in {"exact", "private_hidden"} THEN "canonical" ELSE m
|
||||
|
||||
(* The real gateway accepts a name only when that model exists. *)
|
||||
GatewayAccepts(s, m) == Name(s, m) \in {"canonical", "configured"}
|
||||
|
||||
(* fx knows the capabilities of the model it will send only when the loaded *)
|
||||
(* catalog lists that exact name. *)
|
||||
CapKnownFor(s, m) ==
|
||||
/\ Loaded(s)
|
||||
/\ \/ Name(s, m) = "configured"
|
||||
\/ /\ Name(s, m) = "canonical"
|
||||
/\ (s.model = "private_hidden" => Authoritative(s))
|
||||
|
||||
(* -------------------------------- State ---------------------------------- *)
|
||||
|
||||
(* Placeholder before any request; after = FALSE keeps it out of every check. *)
|
||||
NoReq ==
|
||||
[ model |-> "none", claimedFast |-> FALSE, wireFast |-> FALSE, claimedUltra |-> FALSE,
|
||||
wireUltra |-> FALSE, claimedEffort |-> "none", wireEffort |-> "none", after |-> FALSE ]
|
||||
|
||||
Fresh(s) ==
|
||||
[ sc |-> s,
|
||||
phase |-> "start", \* start | running | done
|
||||
catalogSeen |-> FALSE, \* interactive: the UI saw the load finish
|
||||
sel |-> [ model |-> IF s.model = "none" THEN "configured" ELSE "raw",
|
||||
fast |-> s.speed = "fast",
|
||||
ultra |-> s.speed = "ultrafast",
|
||||
effort |-> s.effort ],
|
||||
outcome |-> "pending",
|
||||
req |-> NoReq,
|
||||
corrections |-> {},
|
||||
notices |-> {},
|
||||
persisted |-> FALSE ]
|
||||
|
||||
Init == st \in { Fresh(s) : s \in Scenarios }
|
||||
|
||||
S == st.sc
|
||||
|
||||
(* The proposed decision, shared by both entry points. *)
|
||||
ModelRejected(s) ==
|
||||
/\ Match(s) \in {"ambiguous", "unknown"}
|
||||
/\ (Authoritative(s) \/ Mutant = "reject_weak")
|
||||
|
||||
Resolved(s, cur) ==
|
||||
IF Match(s) = "matched" /\ Mutant # "no_canonical" THEN "canonical" ELSE cur
|
||||
|
||||
(* -------------------------------- ask ------------------------------------ *)
|
||||
|
||||
AskValidate ==
|
||||
/\ S.path = "ask"
|
||||
/\ st.phase = "start"
|
||||
/\ IF Design = "current"
|
||||
THEN st' = [st EXCEPT !.phase = "running"]
|
||||
ELSE LET m == Resolved(S, st.sel.model)
|
||||
k == CapKnownFor(S, m)
|
||||
reason ==
|
||||
CASE ModelRejected(S) -> "rejected_model"
|
||||
[] st.sel.ultra /\ ~(k /\ S.ultraCap) -> "rejected_ultrafast"
|
||||
[] st.sel.fast /\ k /\ ~S.fastCap -> "rejected_fast"
|
||||
[] st.sel.effort = "unsupported" /\ k -> "rejected_effort"
|
||||
[] OTHER -> "none"
|
||||
IN st' = [st EXCEPT !.sel.model = m,
|
||||
!.phase = IF reason = "none" THEN "running" ELSE "done",
|
||||
!.outcome = IF reason = "none" THEN "pending" ELSE reason]
|
||||
|
||||
(* ----------------------------- interactive ------------------------------- *)
|
||||
|
||||
FirstFrame ==
|
||||
/\ S.path = "interactive"
|
||||
/\ st.phase = "start"
|
||||
/\ Mutant = "block_frame" => st.catalogSeen
|
||||
/\ st' = [st EXCEPT !.phase = "running"]
|
||||
|
||||
Corrected(s, sel) ==
|
||||
IF ModelRejected(s)
|
||||
THEN [ sel |-> [model |-> "configured", fast |-> FALSE, ultra |-> FALSE, effort |-> "none"],
|
||||
corrections |-> {"model"} ]
|
||||
ELSE LET m == Resolved(s, sel.model)
|
||||
k == CapKnownFor(s, m)
|
||||
fast == sel.fast /\ ~(k /\ ~s.fastCap)
|
||||
ultra == sel.ultra /\ k /\ s.ultraCap
|
||||
effort == IF sel.effort = "unsupported" /\ k THEN "none" ELSE sel.effort
|
||||
IN [ sel |-> [model |-> m, fast |-> fast, ultra |-> ultra, effort |-> effort],
|
||||
corrections |-> { c \in {"fast", "ultrafast", "effort"} :
|
||||
CASE c = "fast" -> fast # sel.fast
|
||||
[] c = "ultrafast" -> ultra # sel.ultra
|
||||
[] c = "effort" -> effort # sel.effort } ]
|
||||
|
||||
CatalogReady ==
|
||||
/\ S.path = "interactive"
|
||||
/\ ~st.catalogSeen
|
||||
/\ S.early => st.phase = "done"
|
||||
/\ IF Design = "current"
|
||||
THEN st' = [st EXCEPT !.catalogSeen = TRUE]
|
||||
ELSE LET c == Corrected(S, st.sel) IN
|
||||
st' = [st EXCEPT !.catalogSeen = TRUE,
|
||||
!.sel = c.sel,
|
||||
!.corrections = c.corrections,
|
||||
!.notices = IF Mutant = "silent" THEN {} ELSE c.corrections,
|
||||
!.persisted = Mutant = "persist" /\ c.corrections # {}]
|
||||
|
||||
(* ------------------------------- sending --------------------------------- *)
|
||||
|
||||
Send ==
|
||||
/\ st.phase = "running"
|
||||
/\ S.path = "interactive" => (S.early <=> ~st.catalogSeen)
|
||||
/\ LET m == st.sel.model
|
||||
k == CapKnownFor(S, m)
|
||||
IN IF st.sel.ultra /\ ~(k /\ S.ultraCap)
|
||||
THEN st' = [st EXCEPT !.phase = "done", !.outcome = "turn_error"]
|
||||
ELSE st' = [st EXCEPT
|
||||
!.phase = "done",
|
||||
!.outcome = IF GatewayAccepts(S, m) THEN "sent" ELSE "gateway_error",
|
||||
!.req = [ model |-> Name(S, m),
|
||||
claimedFast |-> st.sel.fast,
|
||||
wireFast |-> st.sel.fast /\ k /\ S.fastCap,
|
||||
claimedUltra |-> st.sel.ultra,
|
||||
wireUltra |-> st.sel.ultra,
|
||||
claimedEffort |-> st.sel.effort,
|
||||
wireEffort |-> IF st.sel.effort = "supported" /\ k THEN "supported" ELSE "none",
|
||||
after |-> S.path = "ask" \/ st.catalogSeen ]]
|
||||
|
||||
Finished ==
|
||||
/\ st.phase = "done"
|
||||
/\ S.path = "interactive" => st.catalogSeen
|
||||
|
||||
Next ==
|
||||
\/ AskValidate
|
||||
\/ FirstFrame
|
||||
\/ CatalogReady
|
||||
\/ Send
|
||||
\/ (Finished /\ UNCHANGED vars)
|
||||
|
||||
Spec == Init /\ [][Next]_vars
|
||||
|
||||
(* Weak fairness: an enabled step eventually happens, so every run ends. *)
|
||||
FairSpec == Spec /\ WF_vars(Next)
|
||||
|
||||
(* ------------------------------ Invariants ------------------------------- *)
|
||||
|
||||
(* A request sent after the decision point (always for ask; after the UI *)
|
||||
(* saw the catalog for interactive). *)
|
||||
Decided == st.req.after
|
||||
|
||||
(* With an authenticated catalog, fx never sends a model that does not exist. *)
|
||||
ValidModel == Decided /\ Authoritative(S) => GatewayAccepts(S, st.req.model)
|
||||
|
||||
(* When fx knows the capabilities, the speed it shows is the speed it sends. *)
|
||||
HonestSpeed ==
|
||||
Decided /\ CapKnownFor(S, st.req.model) =>
|
||||
/\ st.req.claimedFast = st.req.wireFast
|
||||
/\ st.req.claimedUltra = st.req.wireUltra
|
||||
|
||||
(* When fx knows the capabilities, the effort it claims is the effort it sends. *)
|
||||
HonestEffort ==
|
||||
Decided /\ CapKnownFor(S, st.req.model) => st.req.claimedEffort = st.req.wireEffort
|
||||
|
||||
(* fx ask fails during validation, never with a late turn error. *)
|
||||
AskFailsEarly == S.path = "ask" => st.outcome # "turn_error"
|
||||
|
||||
(* Every rejection or correction is backed by catalog evidence. *)
|
||||
NoFalseReject ==
|
||||
/\ (st.outcome = "rejected_model" \/ "model" \in st.corrections) =>
|
||||
Authoritative(S) /\ S.model \in {"partial_ambiguous", "unknown"}
|
||||
/\ (st.outcome = "rejected_fast" \/ "fast" \in st.corrections) =>
|
||||
CapKnownFor(S, st.sel.model) /\ ~S.fastCap
|
||||
/\ (st.outcome = "rejected_effort" \/ "effort" \in st.corrections) =>
|
||||
CapKnownFor(S, st.sel.model) /\ S.effort = "unsupported"
|
||||
/\ (st.outcome = "rejected_ultrafast" \/ "ultrafast" \in st.corrections) =>
|
||||
~(CapKnownFor(S, st.sel.model) /\ S.ultraCap)
|
||||
|
||||
(* A unique partial name runs on its catalog ID. *)
|
||||
Canonical == Decided /\ S.model = "partial_unique" /\ Loaded(S) => st.req.model = "canonical"
|
||||
|
||||
(* Per-launch flags never rewrite saved settings. *)
|
||||
SettingsUntouched == ~st.persisted
|
||||
|
||||
(* The interactive first frame never waits for the catalog. *)
|
||||
FirstFrameNotBlocked == (S.path = "interactive" /\ st.phase = "start") => ENABLED FirstFrame
|
||||
|
||||
(* Every automatic correction is announced. *)
|
||||
CorrectionsAnnounced == st.corrections \subseteq st.notices
|
||||
|
||||
(* Every behavior ends. *)
|
||||
Termination == <>Finished
|
||||
=============================================================================
|
||||
@@ -0,0 +1,65 @@
|
||||
---------------------------- MODULE LaunchTrace ----------------------------
|
||||
(***************************************************************************)
|
||||
(* Trace validation: replays behavior observed from the real fx binary *)
|
||||
(* through LaunchSelection's own actions. *)
|
||||
(* *)
|
||||
(* LaunchObserved is generated by bun/bridge.ts. It defines Observed, a *)
|
||||
(* sequence of records *)
|
||||
(* [sc, orderFree, outcome, notices, req] *)
|
||||
(* where sc is a LaunchSelection scenario, req is *)
|
||||
(* [present, model, indicator, wireFast, wireUltra, wireEffort] *)
|
||||
(* and indicator is "on", "off" or "unobserved" (the footer speed glyph at *)
|
||||
(* submit time; fx ask has no footer). *)
|
||||
(* *)
|
||||
(* orderFree marks the early-submit race: the prompt and the catalog land *)
|
||||
(* together, so either interleaving may explain the observation. *)
|
||||
(* *)
|
||||
(* Observations are replayed in order. The index k advances only when the *)
|
||||
(* machine has finished a scenario in exactly the observed way. The run is *)
|
||||
(* ACCEPTED when TLC violates NotAllMatched, which is reachable only after *)
|
||||
(* every observation matched. A completed search means REJECTED. *)
|
||||
(***************************************************************************)
|
||||
EXTENDS LaunchSelection, LaunchObserved, Sequences
|
||||
|
||||
VARIABLE k
|
||||
traceVars == <<st, k>>
|
||||
|
||||
Starts(o) ==
|
||||
{ Fresh([o.sc EXCEPT !.early = e]) : e \in (IF o.orderFree THEN BOOLEAN ELSE {o.sc.early}) }
|
||||
|
||||
IndicatorMatches(r, i) ==
|
||||
i = "unobserved" \/ ((r.claimedFast \/ r.claimedUltra) <=> (i = "on"))
|
||||
|
||||
ReqMatches(r, o) ==
|
||||
IF ~o.present
|
||||
THEN r.model = "none"
|
||||
ELSE /\ r.model = o.model
|
||||
/\ IndicatorMatches(r, o.indicator)
|
||||
/\ r.wireFast = o.wireFast
|
||||
/\ r.wireUltra = o.wireUltra
|
||||
/\ r.wireEffort = o.wireEffort
|
||||
|
||||
Matches(o) ==
|
||||
/\ Finished
|
||||
/\ st.outcome = o.outcome
|
||||
/\ st.notices = o.notices
|
||||
/\ ReqMatches(st.req, o.req)
|
||||
|
||||
TraceInit == k = 1 /\ st \in Starts(Observed[1])
|
||||
|
||||
Step ==
|
||||
/\ k <= Len(Observed)
|
||||
/\ (AskValidate \/ FirstFrame \/ CatalogReady \/ Send)
|
||||
/\ UNCHANGED k
|
||||
|
||||
Advance ==
|
||||
/\ k <= Len(Observed)
|
||||
/\ Matches(Observed[k])
|
||||
/\ k' = k + 1
|
||||
/\ IF k + 1 <= Len(Observed) THEN st' \in Starts(Observed[k + 1]) ELSE UNCHANGED st
|
||||
|
||||
TraceSpec == TraceInit /\ [][Step \/ Advance]_traceVars
|
||||
|
||||
(* Sentinel: violated exactly when every observation matched. *)
|
||||
NotAllMatched == k <= Len(Observed)
|
||||
=============================================================================
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
# Model-check LaunchSelection, the launch-time --model, --fast, --ultrafast,
|
||||
# and --effort handling of `fx` and `fx ask`, and require each config's
|
||||
# expected verdict.
|
||||
#
|
||||
# proposed.cfg and current-holds.cfg must pass. Each current-*.cfg shows a
|
||||
# property the design without launch validation breaks, and each mutant-*.cfg
|
||||
# plants a bug in the proposed design that its property must catch. The
|
||||
# simulate-*.cfg configs run random behaviors in ideal and non-ideal
|
||||
# conditions; SIM_RUNS sets how many (default 20000).
|
||||
#
|
||||
# Run the real binary through every modeled scenario and have TLC replay it:
|
||||
# bun test ../drivers/cli/
|
||||
#
|
||||
# Needs Java 11+ and tla2tools.jar, as ../README.md describes.
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
# shellcheck source=../lib/tlc.sh
|
||||
. ../lib/tlc.sh
|
||||
|
||||
sim="-simulate num=${SIM_RUNS:-20000} -depth 20 -seed 1"
|
||||
tlc_verdicts "$@" <<EOF
|
||||
LaunchSelection proposed -
|
||||
LaunchSelection current-holds -
|
||||
LaunchSelection current-model ValidModel
|
||||
LaunchSelection current-speed HonestSpeed
|
||||
LaunchSelection current-effort HonestEffort
|
||||
LaunchSelection current-late-error AskFailsEarly
|
||||
LaunchSelection current-canonical Canonical
|
||||
LaunchSelection mutant-reject-weak NoFalseReject
|
||||
LaunchSelection mutant-no-canonical Canonical
|
||||
LaunchSelection mutant-block-frame FirstFrameNotBlocked
|
||||
LaunchSelection mutant-persist SettingsUntouched
|
||||
LaunchSelection mutant-silent CorrectionsAnnounced
|
||||
LaunchSelection simulate-ideal simulate $sim
|
||||
LaunchSelection simulate-nonideal simulate $sim
|
||||
EOF
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "current"
|
||||
Mutant = "none"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
Canonical
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "current"
|
||||
Mutant = "none"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
HonestEffort
|
||||
@@ -0,0 +1,11 @@
|
||||
SPECIFICATION FairSpec
|
||||
CONSTANTS
|
||||
Design = "current"
|
||||
Mutant = "none"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
NoFalseReject
|
||||
SettingsUntouched
|
||||
FirstFrameNotBlocked
|
||||
CorrectionsAnnounced
|
||||
PROPERTY Termination
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "current"
|
||||
Mutant = "none"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
AskFailsEarly
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "current"
|
||||
Mutant = "none"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
ValidModel
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "current"
|
||||
Mutant = "none"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
HonestSpeed
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "block_frame"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
FirstFrameNotBlocked
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "no_canonical"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
Canonical
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "persist"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
SettingsUntouched
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "reject_weak"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
NoFalseReject
|
||||
@@ -0,0 +1,7 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "silent"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
CorrectionsAnnounced
|
||||
@@ -0,0 +1,16 @@
|
||||
SPECIFICATION FairSpec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "none"
|
||||
Class = "all"
|
||||
INVARIANTS
|
||||
ValidModel
|
||||
HonestSpeed
|
||||
HonestEffort
|
||||
AskFailsEarly
|
||||
NoFalseReject
|
||||
Canonical
|
||||
SettingsUntouched
|
||||
FirstFrameNotBlocked
|
||||
CorrectionsAnnounced
|
||||
PROPERTY Termination
|
||||
@@ -0,0 +1,15 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "none"
|
||||
Class = "ideal"
|
||||
INVARIANTS
|
||||
ValidModel
|
||||
HonestSpeed
|
||||
HonestEffort
|
||||
AskFailsEarly
|
||||
NoFalseReject
|
||||
Canonical
|
||||
SettingsUntouched
|
||||
FirstFrameNotBlocked
|
||||
CorrectionsAnnounced
|
||||
@@ -0,0 +1,15 @@
|
||||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Design = "proposed"
|
||||
Mutant = "none"
|
||||
Class = "nonideal"
|
||||
INVARIANTS
|
||||
ValidModel
|
||||
HonestSpeed
|
||||
HonestEffort
|
||||
AskFailsEarly
|
||||
NoFalseReject
|
||||
Canonical
|
||||
SettingsUntouched
|
||||
FirstFrameNotBlocked
|
||||
CorrectionsAnnounced
|
||||
@@ -0,0 +1,190 @@
|
||||
// Exhaustive TLC check of the permission-mode contract plus mutation
|
||||
// qualification: each planted mutant must violate the named property, which
|
||||
// shows the properties are strong enough to catch that mistake.
|
||||
//
|
||||
// bun tests/tla/drivers/acp/permission_modes/check-spec.ts
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { outDir } from "../../lib/tlc";
|
||||
import { SPEC_DIR, runTlc, sha256File, tlcToolchain, utcStamp, writeJson } from "./tlc";
|
||||
|
||||
type Mutant = { id: string; note: string; from: string; to: string; expect: string };
|
||||
|
||||
const MUTANTS: Mutant[] = [
|
||||
{
|
||||
id: "new-reports-ask",
|
||||
note: "a new session reports Ask whatever fx enforces (an earlier fx)",
|
||||
from: `/\\ last = Obs("new", "none", TRUE, TRUE, SessionView(Eff(profile)), "none", profile)`,
|
||||
to: `/\\ last = Obs("new", "none", TRUE, TRUE, SessionView("ask"), "none", profile)`,
|
||||
expect: "ReportsEnforcedMode",
|
||||
},
|
||||
{
|
||||
id: "lists-code",
|
||||
note: "the options still list the old code mode",
|
||||
from: `[current |-> m, legacy |-> m, listed |-> ModesCsv,`,
|
||||
to: `[current |-> m, legacy |-> m, listed |-> "code,ask",`,
|
||||
expect: "ListsExactlyCliModes",
|
||||
},
|
||||
{
|
||||
id: "extra-option-field",
|
||||
note: "option values carry a field outside ACP's schema",
|
||||
from: `names |-> NamesCsv, schemaOk |-> TRUE]`,
|
||||
to: `names |-> NamesCsv, schemaOk |-> FALSE]`,
|
||||
expect: "SchemaClean",
|
||||
},
|
||||
{
|
||||
id: "legacy-modes-stale",
|
||||
note: "the older modes field shows the default instead of the session's mode",
|
||||
from: `[current |-> m, legacy |-> m,`,
|
||||
to: `[current |-> m, legacy |-> DefaultMode,`,
|
||||
expect: "ReportsEnforcedMode",
|
||||
},
|
||||
{
|
||||
id: "resume-ignores-profile",
|
||||
note: "a resumed session starts in the default mode",
|
||||
from: `LET resumed == Eff(profile) IN`,
|
||||
to: `LET resumed == DefaultMode IN`,
|
||||
expect: "SessionsStartFromProfile",
|
||||
},
|
||||
{
|
||||
id: "fresh-ignores-profile",
|
||||
note: "a new process starts sessions in the default mode",
|
||||
from: `LET started == Eff(profile) IN`,
|
||||
to: `LET started == DefaultMode IN`,
|
||||
expect: "SessionsStartFromProfile",
|
||||
},
|
||||
{
|
||||
id: "default-ask",
|
||||
note: "an unset profile means Ask instead of Auto",
|
||||
from: `Eff(p) == IF p = "unset" THEN DefaultMode ELSE p`,
|
||||
to: `Eff(p) == IF p = "unset" THEN "ask" ELSE p`,
|
||||
expect: "DefaultIsAuto",
|
||||
},
|
||||
{
|
||||
id: "unknown-accepted",
|
||||
note: "set_config_option answers success for an unknown mode (an earlier fx)",
|
||||
from: `ELSE /\\ last' = Obs("set", v, FALSE, FALSE, NoView, "none", profile)`,
|
||||
to: `ELSE /\\ last' = Obs("set", v, TRUE, FALSE, NoView, "none", profile)`,
|
||||
expect: "UnknownRejected",
|
||||
},
|
||||
{
|
||||
id: "config-change-not-saved",
|
||||
note: "a set_config_option change is not saved to the profile (an earlier fx)",
|
||||
from: `/\\ profile' = v\n /\\ last' = Obs("set"`,
|
||||
to: `/\\ profile' = profile\n /\\ last' = Obs("set"`,
|
||||
expect: "ChangeIsSaved",
|
||||
},
|
||||
{
|
||||
id: "legacy-change-not-saved",
|
||||
note: "a session/set_mode change is not saved to the profile (an earlier fx)",
|
||||
from: `/\\ profile' = v\n /\\ last' = Obs("legacy"`,
|
||||
to: `/\\ profile' = profile\n /\\ last' = Obs("legacy"`,
|
||||
expect: "ChangeIsSaved",
|
||||
},
|
||||
{
|
||||
id: "stored-misread",
|
||||
note: "the profile's saved mode differs from what the session uses",
|
||||
from: `Enforce(cur), profile)`,
|
||||
to: `Enforce(cur), "unset")`,
|
||||
expect: "StoredIsProfile",
|
||||
},
|
||||
{
|
||||
id: "implicit-full-access",
|
||||
note: "a prompt leaves the session in full access without the user choosing it",
|
||||
from: `/\\ UNCHANGED <<profile, cur>>`,
|
||||
to: `/\\ cur' = "full-access"\n /\\ UNCHANGED profile`,
|
||||
expect: "FullAccessOnlyByChoice",
|
||||
},
|
||||
{
|
||||
id: "status-shows-open-session",
|
||||
note: "fx status reports the open session's mode instead of the next session's",
|
||||
from: `StatusView(Eff(profile))`,
|
||||
to: `StatusView(cur)`,
|
||||
expect: "StatusMatchesNextSession",
|
||||
},
|
||||
{
|
||||
id: "new-session-stale",
|
||||
note: "session/new keeps the process's mode instead of reading the saved one (the spare-process bug)",
|
||||
from: `LET opened == Eff(profile) IN`,
|
||||
to: `LET opened == cur IN`,
|
||||
expect: "SessionsStartFromProfile",
|
||||
},
|
||||
{
|
||||
id: "other-process-leaks",
|
||||
note: "another process saving a mode switches the open session",
|
||||
from: `/\\ UNCHANGED <<cur, saved>>`,
|
||||
to: `/\\ cur' = v\n /\\ UNCHANGED saved`,
|
||||
expect: "OpenSessionKeepsMode",
|
||||
},
|
||||
{
|
||||
id: "full-access-reviewed",
|
||||
note: "full access still sends actions to the automatic reviewer",
|
||||
from: `[] m = "full-access" -> "direct/ran"`,
|
||||
to: `[] m = "full-access" -> "reviewed/ran"`,
|
||||
expect: "PromptEnforcesMode",
|
||||
},
|
||||
{
|
||||
id: "rejection-changes-mode",
|
||||
note: "a rejected change still switches the session",
|
||||
from: `ELSE /\\ last' = Obs("set", v, FALSE, FALSE, NoView, "none", profile)\n /\\ UNCHANGED <<profile, cur, saved>>`,
|
||||
to: `ELSE /\\ last' = Obs("set", v, FALSE, FALSE, NoView, "none", profile)\n /\\ cur' = "ask"\n /\\ UNCHANGED <<profile, saved>>`,
|
||||
expect: "RejectedChangesNothing",
|
||||
},
|
||||
];
|
||||
|
||||
const SPEC_FILES = ["AcpPermissionModes.tla", "MCAcpPermissionModes.tla", "MCAcpPermissionModes.cfg"];
|
||||
const ACTION_PROPERTIES = new Set(["RejectedChangesNothing", "OpenSessionKeepsMode", "FullAccessOnlyByChoice"]);
|
||||
|
||||
/// Checks a mutant against its target property alone, so a kill shows that
|
||||
/// property catches the mistake regardless of which one TLC checks first.
|
||||
function targetCfg(target: string): string {
|
||||
return [
|
||||
"CONSTANTS",
|
||||
" Profiles <- MCProfiles",
|
||||
" Values <- MCValues",
|
||||
"INIT Init",
|
||||
"NEXT Next",
|
||||
`${ACTION_PROPERTIES.has(target) ? "PROPERTY" : "INVARIANT"} ${target}`,
|
||||
"CHECK_DEADLOCK FALSE",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tool = tlcToolchain();
|
||||
const reportDir = outDir("acp-permission-modes-spec");
|
||||
mkdirSync(reportDir, { recursive: true });
|
||||
|
||||
const base = await runTlc(join(reportDir, "base"), "MCAcpPermissionModes.tla", "MCAcpPermissionModes.cfg", SPEC_FILES);
|
||||
const basePassed = base.exitCode === 0 && /No error has been found/.test(base.output);
|
||||
console.log(`base spec: ${basePassed ? "PASS" : "FAIL"} (exit ${base.exitCode}, ${base.distinctStates} distinct states, depth ${base.depth})`);
|
||||
|
||||
const source = readFileSync(join(SPEC_DIR, "AcpPermissionModes.tla"), "utf8");
|
||||
const mutants = [];
|
||||
for (const mutant of MUTANTS) {
|
||||
const count = source.split(mutant.from).length - 1;
|
||||
if (count !== 1) throw new Error(`mutant ${mutant.id}: pattern matched ${count} times`);
|
||||
const dir = join(reportDir, "mutants", mutant.id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, "AcpPermissionModes.tla"), source.replace(mutant.from, mutant.to));
|
||||
writeFileSync(join(dir, "Target.cfg"), targetCfg(mutant.expect));
|
||||
const run = await runTlc(dir, "MCAcpPermissionModes.tla", "Target.cfg", ["MCAcpPermissionModes.tla"]);
|
||||
const killed = run.exitCode !== 0 && run.violated === mutant.expect;
|
||||
console.log(`mutant ${mutant.id}: ${killed ? "KILLED" : "SURVIVED"} (exit ${run.exitCode}, violated ${run.violated ?? "none"}, expected ${mutant.expect})`);
|
||||
mutants.push({ ...mutant, exitCode: run.exitCode, violated: run.violated, killed });
|
||||
}
|
||||
|
||||
const summary = {
|
||||
toolchain: tool,
|
||||
spec_sha256: Object.fromEntries(SPEC_FILES.map((name) => [name, sha256File(join(SPEC_DIR, name))])),
|
||||
base: { passed: basePassed, exitCode: base.exitCode, distinctStates: base.distinctStates, depth: base.depth, argv: base.argv },
|
||||
mutants,
|
||||
allKilled: mutants.every((mutant) => mutant.killed),
|
||||
};
|
||||
writeJson(join(reportDir, "summary.json"), summary);
|
||||
console.log(`mutants killed: ${mutants.filter((mutant) => mutant.killed).length}/${mutants.length}`);
|
||||
console.log(`report: ${reportDir}`);
|
||||
if (!basePassed || !summary.allKilled) process.exit(1);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,331 @@
|
||||
// Runtime driver: runs the real `fx acp` binary from a worktree through
|
||||
// directed and seeded-random permission-mode plans, records what a client
|
||||
// observes (mode option and older modes field, JSON-RPC errors, the
|
||||
// profile's saved mode, and how each prompt's shell call was enforced), and
|
||||
// asks TLC whether each observed run is a behavior of
|
||||
// tests/tla/acp/AcpPermissionModes.tla.
|
||||
//
|
||||
// bun tests/tla/drivers/acp/permission_modes/driver.ts --label main # from the repository root, after zig build
|
||||
//
|
||||
// No model calls: completions and reviewer answers come from the fake
|
||||
// Gateway in tests/e2e/tmux-helpers.ts. --worktree runs another checkout's build.
|
||||
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { Fixture, NO_VIEW, type Observation, loadHelpers, viewOf } from "./harness";
|
||||
import { REPO_ROOT, outDir } from "../../lib/tlc";
|
||||
import { sha256File, sha256Text, tlcToolchain, utcStamp, writeJson } from "./tlc";
|
||||
import { validate } from "./trace";
|
||||
|
||||
// ------------------------------------------------------------------ plans --
|
||||
|
||||
const MODES = ["auto", "ask", "full-access"];
|
||||
const UNKNOWN = ["code", "yolo"];
|
||||
const PROFILES = ["unset", ...MODES];
|
||||
|
||||
type PlanOp =
|
||||
| { op: "set" | "legacy" | "other"; value: string }
|
||||
| { op: "prompt" }
|
||||
| { op: "resume" }
|
||||
| { op: "fresh" }
|
||||
| { op: "new" }
|
||||
| { op: "status" };
|
||||
|
||||
type Plan = { name: string; profile: string; ops: PlanOp[] };
|
||||
|
||||
const set = (value: string): PlanOp => ({ op: "set", value });
|
||||
const legacy = (value: string): PlanOp => ({ op: "legacy", value });
|
||||
const other = (value: string): PlanOp => ({ op: "other", value });
|
||||
const P: PlanOp = { op: "prompt" };
|
||||
const R: PlanOp = { op: "resume" };
|
||||
const F: PlanOp = { op: "fresh" };
|
||||
const N: PlanOp = { op: "new" };
|
||||
const S: PlanOp = { op: "status" };
|
||||
|
||||
const DIRECTED: Plan[] = [
|
||||
{ name: "d1-unset-is-auto", profile: "unset",
|
||||
ops: [P, set("ask"), P, set("full-access"), P, set("auto"), P, R, P] },
|
||||
{ name: "d2-profile-ask", profile: "ask",
|
||||
ops: [P, F, legacy("full-access"), P, F, P, R, set("auto"), F, P] },
|
||||
{ name: "d3-unknown-rejected", profile: "auto",
|
||||
ops: [set("code"), legacy("code"), set("yolo"), legacy("yolo"), P, set("ask"), set("code"), P] },
|
||||
{ name: "d4-profile-full-access", profile: "full-access",
|
||||
ops: [P, set("ask"), P, R, P, F, P] },
|
||||
{ name: "d5-legacy-only", profile: "unset",
|
||||
ops: [legacy("ask"), P, legacy("auto"), P, legacy("full-access"), P, R, P, F] },
|
||||
// A warm process (like fx-plus's spare) must not start new sessions in a
|
||||
// mode another process has since replaced.
|
||||
{ name: "d6-other-process", profile: "full-access",
|
||||
ops: [P, other("ask"), P, N, P, other("full-access"), P, R, P, set("auto"), N, P] },
|
||||
// fx status reports the next session's mode, which an open session may not share.
|
||||
{ name: "d7-status", profile: "unset",
|
||||
ops: [S, set("full-access"), S, other("ask"), S, P, N, S, legacy("auto"), F, S] },
|
||||
];
|
||||
|
||||
function mulberry32(seed: number) {
|
||||
let a = seed >>> 0;
|
||||
return () => {
|
||||
a = (a + 0x6d2b79f5) >>> 0;
|
||||
let t = a;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
function randomPlans(count: number, steps: number, seed: number): Plan[] {
|
||||
const rand = mulberry32(seed);
|
||||
const pick = <T>(items: T[]): T => items[Math.floor(rand() * items.length)]!;
|
||||
const weighted: Array<() => PlanOp> = [
|
||||
() => set(pick(MODES)), () => set(pick(MODES)), () => set(pick(MODES)), () => set(pick(UNKNOWN)),
|
||||
() => legacy(pick(MODES)), () => legacy(pick(MODES)), () => legacy(pick(UNKNOWN)),
|
||||
() => other(pick(MODES)), () => N, () => S,
|
||||
() => P, () => P, () => P, () => R, () => F,
|
||||
];
|
||||
// fx saves a chat at its first turn, so resume only follows a prompt in
|
||||
// the current session (an explicit assumption in the spec).
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const name = `r${String(index + 1).padStart(2, "0")}-seed${seed}`;
|
||||
const profile = pick(PROFILES);
|
||||
let prompted = false;
|
||||
const ops = Array.from({ length: steps }, () => {
|
||||
const op = pick(weighted)();
|
||||
const chosen = op.op === "resume" && !prompted ? P : op;
|
||||
if (chosen.op === "prompt") prompted = true;
|
||||
if (chosen.op === "fresh" || chosen.op === "new") prompted = false;
|
||||
return chosen;
|
||||
});
|
||||
return { name, profile, ops };
|
||||
});
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ runner --
|
||||
|
||||
async function runPlan(plan: Plan, worktree: string, helpers: Awaited<ReturnType<typeof loadHelpers>>, dir: string) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeJson(join(dir, "plan.json"), plan);
|
||||
const fixture = new Fixture(worktree, helpers, plan.profile, join(dir, "fx-trace.log"));
|
||||
const observations: Observation[] = [];
|
||||
let error: string | null = null;
|
||||
const sessionStep = (op: string, response: any) => {
|
||||
const view = response.error ? null : viewOf(response.result);
|
||||
observations.push({
|
||||
op, arg: "none", ok: !response.error && view !== null, hasView: view !== null, view: view ?? NO_VIEW,
|
||||
enforce: "none", stored: fixture.stored(),
|
||||
detail: response.error ? { error: response.error } : view ? undefined : { result: response.result },
|
||||
});
|
||||
return response.result?.sessionId as string | undefined;
|
||||
};
|
||||
try {
|
||||
let acp = await fixture.start();
|
||||
let sessionId = sessionStep("new", await acp.call("session/new", { cwd: fixture.workspace, mcpServers: [] }));
|
||||
if (!sessionId) throw new Error("session/new returned no session");
|
||||
|
||||
for (const step of plan.ops) {
|
||||
switch (step.op) {
|
||||
case "set": {
|
||||
const response = await acp.call("session/set_config_option", { sessionId, configId: "mode", value: step.value });
|
||||
const view = response.error ? null : viewOf(response.result);
|
||||
observations.push({
|
||||
op: "set", arg: step.value, ok: !response.error, hasView: view !== null, view: view ?? NO_VIEW,
|
||||
enforce: "none", stored: fixture.stored(), detail: response.error ? { error: response.error } : undefined,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "legacy": {
|
||||
const response = await acp.call("session/set_mode", { sessionId, modeId: step.value });
|
||||
observations.push({
|
||||
op: "legacy", arg: step.value, ok: !response.error, hasView: false, view: NO_VIEW,
|
||||
enforce: "none", stored: fixture.stored(),
|
||||
detail: response.error ? { error: response.error } : { result: response.result ?? null },
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "prompt": {
|
||||
const probe = await fixture.probePrompt(sessionId);
|
||||
observations.push({
|
||||
op: "prompt", arg: "none", ok: probe.ok, hasView: false, view: NO_VIEW,
|
||||
enforce: probe.enforce, stored: fixture.stored(), detail: probe.detail,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "resume": {
|
||||
acp = await fixture.restart();
|
||||
sessionStep("resume", await acp.call("session/resume", { sessionId, cwd: fixture.workspace, mcpServers: [] }));
|
||||
break;
|
||||
}
|
||||
case "fresh": {
|
||||
acp = await fixture.restart();
|
||||
const next = sessionStep("fresh", await acp.call("session/new", { cwd: fixture.workspace, mcpServers: [] }));
|
||||
if (!next) throw new Error("session/new in a fresh process returned no session");
|
||||
sessionId = next;
|
||||
break;
|
||||
}
|
||||
case "new": {
|
||||
const next = sessionStep("new", await acp.call("session/new", { cwd: fixture.workspace, mcpServers: [] }));
|
||||
if (!next) throw new Error("session/new returned no session");
|
||||
sessionId = next;
|
||||
break;
|
||||
}
|
||||
case "status": {
|
||||
const status = fixture.status();
|
||||
observations.push({
|
||||
op: "status", arg: "none", ok: status.ok, hasView: status.view !== null, view: status.view ?? NO_VIEW,
|
||||
enforce: "none", stored: fixture.stored(), detail: status.detail,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "other": {
|
||||
fixture.saveFromOtherProcess(step.value);
|
||||
observations.push({
|
||||
op: "other", arg: step.value, ok: true, hasView: false, view: NO_VIEW,
|
||||
enforce: "none", stored: fixture.stored(),
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (caught) {
|
||||
error = caught instanceof Error ? caught.message : String(caught);
|
||||
} finally {
|
||||
await fixture.dispose();
|
||||
}
|
||||
|
||||
writeJson(join(dir, "observations.json"), observations);
|
||||
writeFileSync(join(dir, "acp.jsonl"), fixture.acpLog.map((entry) => JSON.stringify(entry)).join("\n") + "\n");
|
||||
writeFileSync(join(dir, "gateway.jsonl"), fixture.gatewayLog.map((entry) => JSON.stringify(entry)).join("\n") + "\n");
|
||||
writeJson(join(dir, "processes.json"), fixture.processes);
|
||||
return { observations, processes: fixture.processes, error };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- controls --
|
||||
|
||||
function syntheticTrace(): Observation[] {
|
||||
const session = (current: string) => ({
|
||||
current, legacy: current, listed: "auto,ask,full-access", legacyListed: "auto,ask,full-access",
|
||||
names: "Auto,Ask,Full access", schemaOk: true,
|
||||
});
|
||||
const config = (current: string) => ({ ...session(current), legacy: "n/a", legacyListed: "n/a" });
|
||||
const quiet = { hasView: false, view: NO_VIEW };
|
||||
return [
|
||||
{ op: "new", arg: "none", ok: true, hasView: true, view: session("auto"), enforce: "none", stored: "unset" },
|
||||
{ op: "prompt", arg: "none", ok: true, ...quiet, enforce: "reviewed/ran", stored: "unset" },
|
||||
{ op: "set", arg: "ask", ok: true, hasView: true, view: config("ask"), enforce: "none", stored: "ask" },
|
||||
{ op: "prompt", arg: "none", ok: true, ...quiet, enforce: "asked/no-run", stored: "ask" },
|
||||
{ op: "set", arg: "code", ok: false, ...quiet, enforce: "none", stored: "ask" },
|
||||
{ op: "legacy", arg: "full-access", ok: true, ...quiet, enforce: "none", stored: "full-access" },
|
||||
{ op: "prompt", arg: "none", ok: true, ...quiet, enforce: "direct/ran", stored: "full-access" },
|
||||
{ op: "resume", arg: "none", ok: true, hasView: true, view: session("full-access"), enforce: "none", stored: "full-access" },
|
||||
{ op: "other", arg: "ask", ok: true, ...quiet, enforce: "none", stored: "ask" },
|
||||
{ op: "status", arg: "none", ok: true, hasView: true, view: config("ask"), enforce: "none", stored: "ask" },
|
||||
{ op: "prompt", arg: "none", ok: true, ...quiet, enforce: "direct/ran", stored: "ask" },
|
||||
{ op: "new", arg: "none", ok: true, hasView: true, view: session("ask"), enforce: "none", stored: "ask" },
|
||||
{ op: "prompt", arg: "none", ok: true, ...quiet, enforce: "asked/no-run", stored: "ask" },
|
||||
];
|
||||
}
|
||||
|
||||
/// Flips the latest prompt's enforcement (or a shown mode), so a conforming
|
||||
/// trace stops conforming.
|
||||
function tamper(observations: Observation[]): Observation[] | null {
|
||||
const copy: Observation[] = structuredClone(observations);
|
||||
for (let index = copy.length - 1; index >= 0; index -= 1) {
|
||||
const step = copy[index]!;
|
||||
if (step.op === "prompt" && step.ok) {
|
||||
step.enforce = step.enforce === "direct/ran" ? "asked/no-run" : "direct/ran";
|
||||
return copy;
|
||||
}
|
||||
if (step.hasView) {
|
||||
step.view.current = step.view.current === "ask" ? "auto" : "ask";
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- main --
|
||||
|
||||
function argValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(name);
|
||||
return index === -1 ? undefined : process.argv[index + 1];
|
||||
}
|
||||
|
||||
function git(worktree: string, ...args: string[]): string {
|
||||
const result = Bun.spawnSync(["git", "-C", worktree, ...args], { stdout: "pipe", stderr: "pipe" });
|
||||
return new TextDecoder().decode(result.stdout).trim();
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const worktree = resolve(argValue("--worktree") ?? REPO_ROOT);
|
||||
const fxBin = join(worktree, "zig-out", "bin", "fx");
|
||||
if (!existsSync(fxBin)) throw new Error(`missing ${fxBin}; run zig build first`);
|
||||
const label = argValue("--label") ?? "run";
|
||||
const seed = Number(argValue("--seed") ?? "20261007");
|
||||
const randomCount = Number(argValue("--random") ?? "20");
|
||||
const steps = Number(argValue("--steps") ?? "12");
|
||||
|
||||
const reportDir = outDir(`acp-permission-modes-${label}`);
|
||||
mkdirSync(reportDir, { recursive: true });
|
||||
const status = git(worktree, "status", "--porcelain");
|
||||
const manifest = {
|
||||
label,
|
||||
worktree,
|
||||
head: git(worktree, "rev-parse", "HEAD"),
|
||||
dirty: status.length > 0,
|
||||
status: status.split("\n").filter(Boolean),
|
||||
diff_sha256: status ? sha256Text(git(worktree, "diff", "HEAD")) : null,
|
||||
fx_binary_sha256: sha256File(fxBin),
|
||||
toolchain: tlcToolchain(),
|
||||
bun: Bun.version,
|
||||
seed,
|
||||
randomCount,
|
||||
steps,
|
||||
argv: process.argv.slice(1),
|
||||
};
|
||||
writeJson(join(reportDir, "manifest.json"), manifest);
|
||||
if (status) writeFileSync(join(reportDir, "worktree.diff"), git(worktree, "diff", "HEAD") + "\n");
|
||||
|
||||
const helpers = await loadHelpers(worktree);
|
||||
const only = argValue("--only");
|
||||
const plans = [...DIRECTED, ...randomPlans(randomCount, steps, seed)]
|
||||
.filter((plan) => !only || new RegExp(only).test(plan.name));
|
||||
const results = [];
|
||||
for (const plan of plans) {
|
||||
const dir = join(reportDir, "traces", plan.name);
|
||||
const run = await runPlan(plan, worktree, helpers, dir);
|
||||
const processFailed = run.processes.some((proc) => proc.exit !== 0);
|
||||
const checked = run.error ? null : await validate(run.observations, dir, plan.profile);
|
||||
const verdict = run.error ? "RUN_ERROR" : processFailed ? "PROCESS_ERROR" : checked!.verdict;
|
||||
results.push({ name: plan.name, profile: plan.profile, verdict, steps: run.observations.length, error: run.error, firstDivergence: checked?.firstDivergence ?? null, processes: run.processes.map((proc) => proc.exit) });
|
||||
const divergence = checked?.firstDivergence;
|
||||
const where = divergence
|
||||
? ` first divergence at step ${divergence.step}: ${JSON.stringify({ ...divergence.observation, detail: undefined })}`
|
||||
: "";
|
||||
console.log(`${plan.name} [${plan.profile}]: ${verdict} (${run.observations.length} steps)${run.error ? ` ${run.error}` : where}`);
|
||||
}
|
||||
|
||||
// Controls show the bridge separates conforming from non-conforming runs.
|
||||
const controls = [];
|
||||
const synthetic = syntheticTrace();
|
||||
controls.push({ name: "synthetic-conforming", expect: "ACCEPTED", got: (await validate(synthetic, join(reportDir, "controls", "synthetic-conforming"), "unset")).verdict });
|
||||
controls.push({ name: "synthetic-tampered", expect: "REJECTED", got: (await validate(tamper(synthetic)!, join(reportDir, "controls", "synthetic-tampered"), "unset")).verdict });
|
||||
controls.push({ name: "empty-trace", expect: "TOOL_ERROR", got: (await validate([], join(reportDir, "controls", "empty-trace"), "unset")).verdict });
|
||||
const firstAccepted = results.find((result) => result.verdict === "ACCEPTED");
|
||||
if (firstAccepted) {
|
||||
const observed = (await Bun.file(join(reportDir, "traces", firstAccepted.name, "observations.json")).json()) as Observation[];
|
||||
const tampered = tamper(observed);
|
||||
if (tampered) {
|
||||
controls.push({ name: `tampered-${firstAccepted.name}`, expect: "REJECTED", got: (await validate(tampered, join(reportDir, "controls", `tampered-${firstAccepted.name}`), firstAccepted.profile)).verdict });
|
||||
}
|
||||
}
|
||||
for (const control of controls) console.log(`control ${control.name}: ${control.got} (expected ${control.expect})`);
|
||||
|
||||
const counts = results.reduce<Record<string, number>>((acc, result) => ({ ...acc, [result.verdict]: (acc[result.verdict] ?? 0) + 1 }), {});
|
||||
const controlsPassed = controls.every((control) => control.got === control.expect);
|
||||
const summary = { manifest, counts, controlsPassed, controls, results };
|
||||
writeJson(join(reportDir, "summary.json"), summary);
|
||||
console.log(`verdicts: ${JSON.stringify(counts)}; controls ${controlsPassed ? "passed" : "FAILED"}`);
|
||||
console.log(`report: ${reportDir}`);
|
||||
const allAccepted = results.every((result) => result.verdict === "ACCEPTED");
|
||||
process.exit(controlsPassed && allAccepted ? 0 : 1);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,340 @@
|
||||
// Shared runtime bridge for the permission-mode bundle: an ACP stdio client
|
||||
// that declines permission requests, an isolated fx home and fake Gateway,
|
||||
// the client-visible projection of the mode option, the profile's saved
|
||||
// mode, and a prompt probe that shows how fx enforced the current mode.
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
// ------------------------------------------------------------ observation --
|
||||
|
||||
/// What a client sees of the mode setting in one response. `legacy` and
|
||||
/// `legacyListed` come from the older `modes` field; responses without it
|
||||
/// (set_config_option) report "n/a".
|
||||
export type View = {
|
||||
current: string;
|
||||
legacy: string;
|
||||
listed: string;
|
||||
legacyListed: string;
|
||||
names: string;
|
||||
schemaOk: boolean;
|
||||
};
|
||||
|
||||
export type Observation = {
|
||||
op: string;
|
||||
arg: string;
|
||||
ok: boolean;
|
||||
hasView: boolean;
|
||||
view: View;
|
||||
enforce: string;
|
||||
stored: string;
|
||||
detail?: unknown;
|
||||
};
|
||||
|
||||
export const NO_VIEW: View = { current: "", legacy: "", listed: "", legacyListed: "", names: "", schemaOk: true };
|
||||
|
||||
// The ACP schema's fields for a select value (session config options).
|
||||
const VALUE_KEYS = new Set(["value", "name", "description", "_meta"]);
|
||||
|
||||
function flatValues(options: unknown): any[] {
|
||||
if (!Array.isArray(options)) return [];
|
||||
return options.flatMap((entry: any) => (Array.isArray(entry?.options) ? entry.options : [entry]));
|
||||
}
|
||||
|
||||
export function viewOf(result: any): View | null {
|
||||
const configOptions = result?.configOptions;
|
||||
if (!Array.isArray(configOptions)) return null;
|
||||
const mode = configOptions.find((option: any) => option?.id === "mode");
|
||||
if (!mode || typeof mode.currentValue !== "string") return null;
|
||||
const values = flatValues(mode.options);
|
||||
const modes = result?.modes;
|
||||
return {
|
||||
current: mode.currentValue,
|
||||
legacy: modes === undefined ? "n/a" : String(modes?.currentModeId ?? "missing"),
|
||||
listed: values.map((value) => String(value?.value)).join(","),
|
||||
legacyListed: modes === undefined ? "n/a" : (modes?.availableModes ?? []).map((entry: any) => String(entry?.id)).join(","),
|
||||
names: values.map((value) => String(value?.name)).join(","),
|
||||
schemaOk:
|
||||
mode.type === "select" &&
|
||||
mode.category === "mode" &&
|
||||
values.every((value) => value && Object.keys(value).every((key) => VALUE_KEYS.has(key))),
|
||||
};
|
||||
}
|
||||
|
||||
/// The profile's saved permission mode, in the CLI's names.
|
||||
export function storedMode(home: string): string {
|
||||
const path = join(home, ".fx", "settings.json");
|
||||
if (!existsSync(path)) return "unset";
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = JSON.parse(readFileSync(path, "utf8"))?.permission_mode;
|
||||
} catch {
|
||||
return "unreadable";
|
||||
}
|
||||
if (raw === undefined || raw === null) return "unset";
|
||||
const value = String(raw).toLowerCase();
|
||||
if (value === "ask" || value === "auto") return value;
|
||||
if (value === "yolo" || value === "full-access" || value === "full access") return "full-access";
|
||||
return `raw:${value}`;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------- ACP client --
|
||||
|
||||
export class Acp {
|
||||
private buffer = "";
|
||||
private lines: string[] = [];
|
||||
private waiters: Array<(line: string) => void> = [];
|
||||
private closed = false;
|
||||
private nextId = 1;
|
||||
stderr = "";
|
||||
permissionRequests = 0;
|
||||
readonly exited: Promise<number>;
|
||||
|
||||
constructor(readonly proc: ReturnType<typeof Bun.spawn>, private readonly log: (entry: object) => void) {
|
||||
this.exited = (async () => {
|
||||
const code = await proc.exited;
|
||||
this.closed = true;
|
||||
for (const waiter of this.waiters.splice(0)) waiter("");
|
||||
return code;
|
||||
})();
|
||||
(async () => {
|
||||
const decoder = new TextDecoder();
|
||||
for await (const chunk of proc.stdout as ReadableStream<Uint8Array>) {
|
||||
this.buffer += decoder.decode(chunk);
|
||||
const parts = this.buffer.split("\n");
|
||||
this.buffer = parts.pop() ?? "";
|
||||
for (const line of parts) {
|
||||
if (!line.trim()) continue;
|
||||
const waiter = this.waiters.shift();
|
||||
if (waiter) waiter(line);
|
||||
else this.lines.push(line);
|
||||
}
|
||||
}
|
||||
})();
|
||||
(async () => {
|
||||
const decoder = new TextDecoder();
|
||||
for await (const chunk of proc.stderr as ReadableStream<Uint8Array>) this.stderr += decoder.decode(chunk);
|
||||
})();
|
||||
}
|
||||
|
||||
private async readLine(): Promise<any> {
|
||||
const buffered = this.lines.shift();
|
||||
if (buffered) return JSON.parse(buffered);
|
||||
if (this.closed) throw new Error(`fx acp exited: ${this.stderr.slice(-400)}`);
|
||||
const line = await new Promise<string>((resolveLine, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`fx acp read timeout: ${this.stderr.slice(-400)}`)), 30_000);
|
||||
this.waiters.push((value) => {
|
||||
clearTimeout(timer);
|
||||
resolveLine(value);
|
||||
});
|
||||
});
|
||||
if (!line) throw new Error(`fx acp exited: ${this.stderr.slice(-400)}`);
|
||||
return JSON.parse(line);
|
||||
}
|
||||
|
||||
private send(message: object) {
|
||||
(this.proc.stdin as any).write(`${JSON.stringify(message)}\n`);
|
||||
(this.proc.stdin as any).flush?.();
|
||||
}
|
||||
|
||||
async call(method: string, params: Record<string, unknown>): Promise<any> {
|
||||
const id = this.nextId++;
|
||||
this.log({ dir: "out", id, method, params });
|
||||
this.send({ jsonrpc: "2.0", id, method, params });
|
||||
while (true) {
|
||||
const message = await this.readLine();
|
||||
if ("method" in message && "id" in message) {
|
||||
// Agent-to-client request. Permission requests are declined with the
|
||||
// reject option fx offers, so an "ask" turn never runs the tool.
|
||||
if (message.method === "session/request_permission") {
|
||||
this.permissionRequests += 1;
|
||||
const options: any[] = message.params?.options ?? [];
|
||||
const reject = options.find((option) => String(option?.kind ?? "").startsWith("reject"));
|
||||
this.log({ dir: "in-request", method: message.method, options: options.map((option) => option?.kind), chose: reject?.optionId ?? "cancelled" });
|
||||
this.send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: { outcome: reject ? { outcome: "selected", optionId: reject.optionId } : { outcome: "cancelled" } },
|
||||
});
|
||||
} else {
|
||||
this.log({ dir: "in-request", method: message.method, answered: "error" });
|
||||
this.send({ jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "Method not found" } });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (message.id === id) {
|
||||
this.log({ dir: "in", id, method, error: message.error ?? null, result: message.result ?? null });
|
||||
return message;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<{ exit: number; stderr: string }> {
|
||||
if (!this.closed) (this.proc.stdin as any).end();
|
||||
const timer = setTimeout(() => this.proc.kill("SIGKILL"), 5_000);
|
||||
const exit = await this.exited;
|
||||
clearTimeout(timer);
|
||||
return { exit, stderr: this.stderr };
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------- fx fixture --
|
||||
|
||||
type Helpers = {
|
||||
startDynamicFakeGateway: (respond: (body: string) => Response, options: object) => any;
|
||||
fakeGatewayFinalText: (text: string) => Response;
|
||||
fakeShellRun: (id: string, command: string, options?: Record<string, unknown>) => Response;
|
||||
};
|
||||
|
||||
export async function loadHelpers(worktree: string): Promise<Helpers> {
|
||||
return (await import(join(worktree, "tests", "e2e", "tmux-helpers.ts"))) as Helpers;
|
||||
}
|
||||
|
||||
// Process overrides that would mask what the profile and ACP decide.
|
||||
const STRIPPED_ENV = [
|
||||
"VERCEL_OIDC_TOKEN", "FX_PROVIDER", "FX_MODEL", "FX_FAST", "FX_ULTRAFAST",
|
||||
"FX_PERMISSION_MODE", "FX_DEBUG_RECORD", "FX_SESSIONS_V2",
|
||||
];
|
||||
|
||||
/// An isolated fx home, workspace and fake Gateway for one run. `profile` is
|
||||
/// the saved permission_mode the run starts from ("unset" leaves it out).
|
||||
export class Fixture {
|
||||
readonly root: string;
|
||||
readonly home: string;
|
||||
readonly workspace: string;
|
||||
readonly acpLog: object[] = [];
|
||||
readonly gatewayLog: object[] = [];
|
||||
readonly processes: Array<{ exit: number; stderr: string }> = [];
|
||||
private readonly gateway: any;
|
||||
private probe: { id: string; marker: string } | null = null;
|
||||
private probeCount = 0;
|
||||
acp: Acp | null = null;
|
||||
|
||||
constructor(private readonly worktree: string, private readonly helpers: Helpers, profile: string, traceLog?: string) {
|
||||
this.root = realpathSync(mkdtempSync(join(tmpdir(), "fx-acp-modes-")));
|
||||
this.home = join(this.root, "home");
|
||||
this.workspace = join(this.root, "workspace");
|
||||
mkdirSync(join(this.home, ".fx"), { recursive: true, mode: 0o700 });
|
||||
mkdirSync(this.workspace, { recursive: true, mode: 0o700 });
|
||||
const settings: Record<string, unknown> = { session_titles: false };
|
||||
if (profile !== "unset") settings.permission_mode = profile;
|
||||
writeFileSync(join(this.home, ".fx", "settings.json"), `${JSON.stringify(settings)}\n`, { mode: 0o600 });
|
||||
|
||||
this.gateway = helpers.startDynamicFakeGateway((body: string) => {
|
||||
const probe = this.probe;
|
||||
const kind = !probe ? "no-probe" : body.includes(probe.id) ? "final" : "tool-call";
|
||||
this.gatewayLog.push({ n: this.gatewayLog.length + 1, probe: probe?.id ?? null, kind });
|
||||
if (probe && kind === "tool-call") return helpers.fakeShellRun(probe.id, `printf ran > ${JSON.stringify(probe.marker)}`);
|
||||
return helpers.fakeGatewayFinalText("done");
|
||||
}, {});
|
||||
this.traceLog = traceLog;
|
||||
}
|
||||
|
||||
private traceLog?: string;
|
||||
|
||||
env(): Record<string, string> {
|
||||
const env: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries({
|
||||
...process.env,
|
||||
NO_COLOR: "1",
|
||||
HOME: this.home,
|
||||
AI_GATEWAY_API_KEY: "acp-modes-fake-key",
|
||||
FX_DISABLE_KEYCHAIN: "1",
|
||||
FX_SKIP_ONBOARDING: "1",
|
||||
FX_AUTO_UPGRADE: "0",
|
||||
FX_SOUND: "0",
|
||||
FX_GATEWAY_BASE_URL: this.gateway.baseUrl,
|
||||
FX_GATEWAY_CHAT_URL: this.gateway.chatUrl,
|
||||
FX_E2E_GATEWAY_MODELS_URL: `${this.gateway.baseUrl}/coding-agent/v1/models`,
|
||||
FX_E2E_GATEWAY_CHAT_URL: this.gateway.chatUrl,
|
||||
...(this.traceLog ? { FX_TRACE_LOG: this.traceLog, FX_TRACE_SCOPES: "acp,permission,host_config" } : {}),
|
||||
})) {
|
||||
if (value === undefined || STRIPPED_ENV.includes(key)) continue;
|
||||
env[key] = value;
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
async start(): Promise<Acp> {
|
||||
const fxBin = join(this.worktree, "zig-out", "bin", "fx");
|
||||
const processNumber = this.processes.length + 1;
|
||||
this.acp = new Acp(
|
||||
Bun.spawn([fxBin, "acp"], { cwd: this.workspace, env: this.env(), stdin: "pipe", stdout: "pipe", stderr: "pipe" }),
|
||||
(entry) => this.acpLog.push({ process: processNumber, ...entry }),
|
||||
);
|
||||
await this.acp.call("initialize", { protocolVersion: 1 });
|
||||
return this.acp;
|
||||
}
|
||||
|
||||
async restart(): Promise<Acp> {
|
||||
if (this.acp) this.processes.push(await this.acp.close());
|
||||
return this.start();
|
||||
}
|
||||
|
||||
/// Runs one turn whose model asks to write a marker file with the shell
|
||||
/// tool, and reports how fx handled it: "asked" (a permission request
|
||||
/// reached the client), "reviewed" (fx's automatic reviewer was called),
|
||||
/// or "direct", plus whether the command ran.
|
||||
async probePrompt(sessionId: string): Promise<{ ok: boolean; enforce: string; detail: object }> {
|
||||
const acp = this.acp!;
|
||||
this.probeCount += 1;
|
||||
const marker = join(this.workspace, `probe-${this.probeCount}.txt`);
|
||||
this.probe = { id: `probe_call_${this.probeCount}`, marker };
|
||||
const asked0 = acp.permissionRequests;
|
||||
const reviewed0 = this.gateway.classifierRequests.length + this.gateway.evaluationRequests.length;
|
||||
const response = await acp.call("session/prompt", { sessionId, prompt: [{ type: "text", text: "write the probe marker" }] });
|
||||
this.probe = null;
|
||||
const asked = acp.permissionRequests - asked0;
|
||||
const reviewed = this.gateway.classifierRequests.length + this.gateway.evaluationRequests.length - reviewed0;
|
||||
const ran = existsSync(marker);
|
||||
const ok = !response.error && response.result?.stopReason === "end_turn";
|
||||
const how = asked > 0 && reviewed > 0 ? "asked+reviewed" : asked > 0 ? "asked" : reviewed > 0 ? "reviewed" : "direct";
|
||||
return {
|
||||
ok,
|
||||
enforce: ok ? `${how}/${ran ? "ran" : "no-run"}` : "none",
|
||||
detail: { asked, reviewed, ran, stopReason: response.result?.stopReason ?? null, error: response.error ?? null },
|
||||
};
|
||||
}
|
||||
|
||||
stored(): string {
|
||||
return storedMode(this.home);
|
||||
}
|
||||
|
||||
/// Runs `fx status --json` in the workspace and projects its modes the way
|
||||
/// a client would read them before a session exists.
|
||||
status(): { ok: boolean; view: View | null; detail: object } {
|
||||
const fxBin = join(this.worktree, "zig-out", "bin", "fx");
|
||||
const run = Bun.spawnSync([fxBin, "status", "--json"], { cwd: this.workspace, env: this.env(), stdout: "pipe", stderr: "pipe" });
|
||||
const text = new TextDecoder().decode(run.stdout).trim();
|
||||
let parsed: any = null;
|
||||
try { parsed = JSON.parse(text); } catch {}
|
||||
const modes: any[] | null = Array.isArray(parsed?.modes) ? parsed.modes : null;
|
||||
const ok = run.exitCode === 0 && typeof parsed?.mode === "string" && modes !== null;
|
||||
const statusKeys = new Set(["id", "name", "description"]);
|
||||
const view: View | null = ok ? {
|
||||
current: parsed.mode,
|
||||
legacy: "n/a",
|
||||
listed: modes!.map((mode) => String(mode?.id)).join(","),
|
||||
legacyListed: "n/a",
|
||||
names: modes!.map((mode) => String(mode?.name)).join(","),
|
||||
schemaOk: modes!.every((mode) => typeof mode?.id === "string" && typeof mode?.name === "string" && Object.keys(mode).every((key) => statusKeys.has(key))),
|
||||
} : null;
|
||||
return { ok, view, detail: { exit: run.exitCode, stderr: new TextDecoder().decode(run.stderr).slice(0, 300), permission_mode: parsed?.permission_mode ?? null } };
|
||||
}
|
||||
|
||||
/// Saves a mode the way another fx process would (the CLI's /permissions
|
||||
/// or another ACP process): fx persists full access as "yolo".
|
||||
saveFromOtherProcess(mode: string) {
|
||||
const path = join(this.home, ".fx", "settings.json");
|
||||
const settings = JSON.parse(readFileSync(path, "utf8"));
|
||||
settings.permission_mode = mode === "full-access" ? "yolo" : mode;
|
||||
writeFileSync(path, `${JSON.stringify(settings)}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
async dispose() {
|
||||
if (this.acp) this.processes.push(await this.acp.close());
|
||||
this.acp = null;
|
||||
this.gateway.stop();
|
||||
rmSync(this.root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// TLC helpers for the ACP permission-mode driver: the shared runner in
|
||||
// ../../lib/tlc.ts, pointed at the specs in tests/tla/acp/.
|
||||
import { join } from "node:path";
|
||||
import { TLA_ROOT, runTlc as runShared } from "../../lib/tlc";
|
||||
|
||||
export { sha256File, sha256Text, tlcToolchain, utcStamp, writeJson } from "../../lib/tlc";
|
||||
export type { TlcRun } from "../../lib/tlc";
|
||||
|
||||
export const SPEC_DIR = join(TLA_ROOT, "acp");
|
||||
|
||||
export function runTlc(dir: string, module: string, cfg: string, specs: string[]) {
|
||||
return runShared(dir, module, cfg, specs, SPEC_DIR);
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// TLC trace validation: turns one observed run into the generated
|
||||
// AcpPermissionModesObs module and asks TLC whether it is a behavior of
|
||||
// tests/tla/acp/AcpPermissionModes.tla (through tests/tla/acp/AcpPermissionModesTrace.tla).
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { Observation } from "./harness";
|
||||
import { runTlc, sha256Text, writeJson } from "./tlc";
|
||||
|
||||
const tla = (value: string) => JSON.stringify(value);
|
||||
const bool = (value: boolean) => (value ? "TRUE" : "FALSE");
|
||||
|
||||
export function obsModule(observations: Observation[], profile: string, values: string[]): string {
|
||||
const record = (o: Observation) => {
|
||||
const v = o.view;
|
||||
const view = `[current |-> ${tla(v.current)}, legacy |-> ${tla(v.legacy)}, listed |-> ${tla(v.listed)}, legacyListed |-> ${tla(v.legacyListed)}, names |-> ${tla(v.names)}, schemaOk |-> ${bool(v.schemaOk)}]`;
|
||||
return `[op |-> ${tla(o.op)}, arg |-> ${tla(o.arg)}, ok |-> ${bool(o.ok)}, hasView |-> ${bool(o.hasView)}, view |-> ${view}, enforce |-> ${tla(o.enforce)}, stored |-> ${tla(o.stored)}]`;
|
||||
};
|
||||
return [
|
||||
"--------------------- MODULE AcpPermissionModesObs ---------------------",
|
||||
"(* Generated from one observed fx acp run. *)",
|
||||
`TProfiles == {${tla(profile)}}`,
|
||||
`TValues == {${[...new Set(values)].map(tla).join(", ")}}`,
|
||||
"TraceLog == <<",
|
||||
observations.map((o) => ` ${record(o)}`).join(",\n"),
|
||||
">>",
|
||||
"=============================================================================",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
const TRACE_CFG = [
|
||||
"CONSTANTS",
|
||||
" Profiles <- TProfiles",
|
||||
" Values <- TValues",
|
||||
"INIT TraceInit",
|
||||
"NEXT TraceNext",
|
||||
"INVARIANT NotDone",
|
||||
"CHECK_DEADLOCK FALSE",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
export type Verdict = "ACCEPTED" | "REJECTED" | "TOOL_ERROR";
|
||||
|
||||
export async function validate(observations: Observation[], dir: string, profile: string) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
if (observations.length === 0) return { verdict: "TOOL_ERROR" as Verdict, reason: "empty trace", matched: 0, firstDivergence: null };
|
||||
const values = ["auto", "ask", "full-access", ...observations.filter((o) => o.op === "set" || o.op === "legacy").map((o) => o.arg)];
|
||||
const obs = obsModule(observations, profile, values);
|
||||
writeFileSync(join(dir, "AcpPermissionModesObs.tla"), obs);
|
||||
writeFileSync(join(dir, "AcpPermissionModesTrace.cfg"), TRACE_CFG);
|
||||
const run = await runTlc(dir, "AcpPermissionModesTrace.tla", "AcpPermissionModesTrace.cfg", ["AcpPermissionModes.tla", "AcpPermissionModesTrace.tla"]);
|
||||
const finalIndex = [...run.output.matchAll(/\/\\ i = (\d+)/g)].map((match) => Number(match[1])).at(-1) ?? null;
|
||||
let verdict: Verdict;
|
||||
let matched: number;
|
||||
if (run.exitCode === 12 && run.violated === "NotDone" && finalIndex === observations.length) {
|
||||
verdict = "ACCEPTED";
|
||||
matched = observations.length;
|
||||
} else if (run.exitCode === 0 && /No error has been found/.test(run.output)) {
|
||||
verdict = "REJECTED";
|
||||
// The trace's states are i = 1..k, so the search depth is how many steps matched.
|
||||
matched = run.distinctStates === 0 ? 0 : run.depth ?? 0;
|
||||
} else {
|
||||
verdict = "TOOL_ERROR";
|
||||
matched = 0;
|
||||
}
|
||||
const result = {
|
||||
verdict,
|
||||
matched,
|
||||
length: observations.length,
|
||||
firstDivergence: verdict === "REJECTED" ? { step: matched + 1, observation: observations[matched] ?? null } : null,
|
||||
tlc: { exitCode: run.exitCode, violated: run.violated, finalIndex, depth: run.depth, durationMs: run.durationMs },
|
||||
obs_sha256: sha256Text(obs),
|
||||
};
|
||||
writeJson(join(dir, "verdict.json"), result);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Exhaustive TLC check of the speed contract plus mutation qualification:
|
||||
// each planted mutant must violate the named invariant, which shows the
|
||||
// invariants are not vacuous. Several mutants match how an earlier fx behaved.
|
||||
//
|
||||
// bun tests/tla/drivers/acp/speed/check-spec.ts
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { outDir } from "../../lib/tlc";
|
||||
import { SPEC_DIR, runTlc, sha256File, tlcToolchain, utcStamp, writeJson } from "./tlc";
|
||||
|
||||
type Mutant = { id: string; note: string; from: string; to: string; expect: string };
|
||||
|
||||
const MUTANTS: Mutant[] = [
|
||||
{
|
||||
id: "fast-ignored",
|
||||
note: "an earlier fx: set fast succeeds but changes nothing",
|
||||
from: " ELSE /\\ fast' = b\n /\\ ultra' = (ultra /\\ ~b)",
|
||||
to: " ELSE /\\ fast' = fast\n /\\ ultra' = ultra",
|
||||
expect: "RequestHonored",
|
||||
},
|
||||
{
|
||||
id: "unknown-accepted",
|
||||
note: "an earlier fx: unknown config ids succeed",
|
||||
from: 'Result("unknown", "none", FALSE, "none")',
|
||||
to: 'Result("unknown", "none", TRUE, "none")',
|
||||
expect: "UnknownRejected",
|
||||
},
|
||||
{
|
||||
id: "switch-keeps-ultra",
|
||||
note: "an earlier fx: Ultrafast stays on, hidden, after switching to a model without it",
|
||||
from: " /\\ ultra' = (ultra /\\ m \\in SupUltra)",
|
||||
to: " /\\ ultra' = ultra",
|
||||
expect: "UltraValid",
|
||||
},
|
||||
{
|
||||
id: "switch-keeps-fast",
|
||||
note: "an earlier fx: Fast stays on, hidden, after switching to a model without it",
|
||||
from: " /\\ fast' = (fast /\\ m \\in SupFast)",
|
||||
to: " /\\ fast' = fast",
|
||||
expect: "SwitchClearsFast",
|
||||
},
|
||||
{
|
||||
id: "fast-not-exclusive",
|
||||
note: "enabling Fast leaves Ultrafast on",
|
||||
from: " /\\ ultra' = (ultra /\\ ~b)",
|
||||
to: " /\\ ultra' = ultra",
|
||||
expect: "Exclusive",
|
||||
},
|
||||
{
|
||||
id: "ultra-not-exclusive",
|
||||
note: "enabling Ultrafast leaves Fast on",
|
||||
from: " /\\ fast' = (fast /\\ ~b)",
|
||||
to: " /\\ fast' = fast",
|
||||
expect: "Exclusive",
|
||||
},
|
||||
{
|
||||
id: "fast-unsupported-accepted",
|
||||
note: "Fast can be enabled on a model without it",
|
||||
from: " IF b /\\ model \\notin SupFast",
|
||||
to: " IF FALSE",
|
||||
expect: "AcceptOnlySupported",
|
||||
},
|
||||
{
|
||||
id: "fast-disable-rejected",
|
||||
note: "turning Fast off is rejected on a model without it",
|
||||
from: " IF b /\\ model \\notin SupFast",
|
||||
to: " IF model \\notin SupFast",
|
||||
expect: "RejectOnlyUnsupported",
|
||||
},
|
||||
{
|
||||
id: "prompt-hidden-fast",
|
||||
note: "a prompt sends Fast for a model without it",
|
||||
from: "ELSE IF fast /\\ model \\in SupFast THEN \"fast\"",
|
||||
to: "ELSE IF fast THEN \"fast\"",
|
||||
expect: "PromptLaneSupported",
|
||||
},
|
||||
];
|
||||
|
||||
async function main() {
|
||||
const tool = tlcToolchain();
|
||||
const reportDir = outDir("acp-speed-spec");
|
||||
mkdirSync(reportDir, { recursive: true });
|
||||
const specFiles = ["AcpSpeed.tla", "MCAcpSpeed.tla", "MCAcpSpeed.cfg"];
|
||||
|
||||
const base = await runTlc(join(reportDir, "base"), "MCAcpSpeed.tla", "MCAcpSpeed.cfg", specFiles);
|
||||
const basePassed = base.exitCode === 0 && /No error has been found/.test(base.output);
|
||||
console.log(`base spec: ${basePassed ? "PASS" : "FAIL"} (exit ${base.exitCode}, ${base.distinctStates} distinct states, depth ${base.depth})`);
|
||||
|
||||
const source = readFileSync(join(SPEC_DIR, "AcpSpeed.tla"), "utf8");
|
||||
const mutants = [];
|
||||
for (const mutant of MUTANTS) {
|
||||
const count = source.split(mutant.from).length - 1;
|
||||
if (count !== 1) throw new Error(`mutant ${mutant.id}: pattern matched ${count} times`);
|
||||
const dir = join(reportDir, "mutants", mutant.id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, "AcpSpeed.tla"), source.replace(mutant.from, mutant.to));
|
||||
const mutated = await runTlc(dir, "MCAcpSpeed.tla", "MCAcpSpeed.cfg", ["MCAcpSpeed.tla", "MCAcpSpeed.cfg"]);
|
||||
const killed = mutated.exitCode === 12 && mutated.violated === mutant.expect;
|
||||
console.log(`mutant ${mutant.id}: ${killed ? "KILLED" : "SURVIVED"} (exit ${mutated.exitCode}, violated ${mutated.violated ?? "none"}, expected ${mutant.expect})`);
|
||||
mutants.push({ ...mutant, exitCode: mutated.exitCode, violated: mutated.violated, killed });
|
||||
}
|
||||
|
||||
const summary = {
|
||||
toolchain: tool,
|
||||
spec_sha256: Object.fromEntries(specFiles.map((name) => [name, sha256File(join(SPEC_DIR, name))])),
|
||||
base: { passed: basePassed, exitCode: base.exitCode, distinctStates: base.distinctStates, depth: base.depth, argv: base.argv },
|
||||
mutants,
|
||||
allKilled: mutants.every((mutant) => mutant.killed),
|
||||
};
|
||||
writeJson(join(reportDir, "summary.json"), summary);
|
||||
console.log(`report: ${reportDir}`);
|
||||
if (!basePassed || !summary.allKilled) process.exit(1);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,370 @@
|
||||
// Runtime driver: drives the real `fx acp` binary from a worktree through
|
||||
// directed and seeded-random speed scenarios, records what a client observes
|
||||
// (configOptions projections, JSON-RPC errors, and the lane each prompt sends
|
||||
// to a local fake Gateway), and asks TLC whether each observed run is a
|
||||
// behavior of tests/tla/acp/AcpSpeed.tla (trace validation via tests/tla/acp/AcpSpeedTrace.tla).
|
||||
//
|
||||
// bun tests/tla/drivers/acp/speed/driver.ts --label main # from the repository root, after zig build
|
||||
//
|
||||
// No live model calls: completions come from the fake Gateway in the
|
||||
// worktree's own tests/e2e/tmux-helpers.ts.
|
||||
import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { REPO_ROOT, outDir } from "../../lib/tlc";
|
||||
import { sha256File, sha256Text, tlcToolchain, utcStamp, writeJson } from "./tlc";
|
||||
import { Acp, NO_VIEW, type Observation, type View, laneOf, validate, viewOf } from "./trace";
|
||||
|
||||
// ---------------------------------------------------------------- catalog --
|
||||
|
||||
const BOTH = "openai/gpt-6-astra"; // Fast (priority tier) and Ultrafast
|
||||
const FAST_ONLY = "openai/gpt-5.6-priority"; // Fast only
|
||||
const PLAIN = "anthropic/claude-plain"; // neither
|
||||
const MODELS = [BOTH, FAST_ONLY, PLAIN];
|
||||
const SUP_FAST = [BOTH, FAST_ONLY];
|
||||
const SUP_ULTRA = [BOTH];
|
||||
const TRACE_CATALOG = { models: MODELS, supFast: SUP_FAST, supUltra: SUP_ULTRA };
|
||||
|
||||
const CATALOG = [
|
||||
{
|
||||
id: BOTH, name: "GPT-6 Astra", type: "language", owned_by: "openai", tags: ["tool-use", "reasoning"],
|
||||
reasoning_options: [{ type: "effort", values: ["high", "xhigh"] }],
|
||||
pricing: {
|
||||
input_cache_read: "0.000001",
|
||||
service_tiers: {
|
||||
priority: { input: "0.00003", output: "0.00015", input_cache_read: "0.000003" },
|
||||
ultrafast: { input: "0.00006", output: "0.0003", input_cache_read: "0.000006" },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: FAST_ONLY, name: "GPT-5.6 Priority", type: "language", owned_by: "openai", tags: ["tool-use"],
|
||||
pricing: { service_tiers: { priority: { input: "0.00003", output: "0.00015" } } },
|
||||
},
|
||||
{ id: PLAIN, name: "Claude Plain", type: "language", owned_by: "anthropic", tags: ["tool-use"] },
|
||||
];
|
||||
|
||||
// ------------------------------------------------------------------ plans --
|
||||
|
||||
type PlanOp =
|
||||
| { op: "fast" | "ultrafast"; value: boolean }
|
||||
| { op: "model"; model: string }
|
||||
| { op: "unknown" }
|
||||
| { op: "prompt" }
|
||||
| { op: "resume" };
|
||||
|
||||
type Plan = { name: string; initialModel: string; defaultFast: boolean; ops: PlanOp[] };
|
||||
|
||||
const on = (op: "fast" | "ultrafast"): PlanOp => ({ op, value: true });
|
||||
const off = (op: "fast" | "ultrafast"): PlanOp => ({ op, value: false });
|
||||
const to = (model: string): PlanOp => ({ op: "model", model });
|
||||
const P: PlanOp = { op: "prompt" };
|
||||
const R: PlanOp = { op: "resume" };
|
||||
const U: PlanOp = { op: "unknown" };
|
||||
|
||||
const DIRECTED: Plan[] = [
|
||||
{ name: "d1-lanes-exclusive", initialModel: BOTH, defaultFast: false,
|
||||
ops: [P, on("fast"), P, on("ultrafast"), P, on("fast"), P, off("fast"), P, U, R, P] },
|
||||
{ name: "d2-switch-drops-ultrafast", initialModel: BOTH, defaultFast: false,
|
||||
ops: [on("ultrafast"), to(PLAIN), P, to(BOTH), P, on("ultrafast"), R, P, to(FAST_ONLY), P] },
|
||||
{ name: "d3-fast-only-model", initialModel: FAST_ONLY, defaultFast: true,
|
||||
ops: [P, on("ultrafast"), off("fast"), P, on("fast"), to(PLAIN), P, to(FAST_ONLY), P, R, P] },
|
||||
{ name: "d4-plain-model", initialModel: PLAIN, defaultFast: true,
|
||||
ops: [P, on("fast"), on("ultrafast"), off("fast"), off("ultrafast"), to(FAST_ONLY), P, R, P] },
|
||||
{ name: "d5-hidden-default-fast", initialModel: PLAIN, defaultFast: true,
|
||||
ops: [P, to(BOTH), P, R, P, on("ultrafast"), to(FAST_ONLY), P] },
|
||||
];
|
||||
|
||||
function mulberry32(seed: number) {
|
||||
let a = seed >>> 0;
|
||||
return () => {
|
||||
a = (a + 0x6d2b79f5) >>> 0;
|
||||
let t = a;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
function randomPlans(count: number, steps: number, seed: number): Plan[] {
|
||||
const rand = mulberry32(seed);
|
||||
const pick = <T>(items: T[]): T => items[Math.floor(rand() * items.length)]!;
|
||||
const weighted: Array<() => PlanOp> = [
|
||||
() => on("fast"), () => off("fast"), () => on("fast"),
|
||||
() => on("ultrafast"), () => off("ultrafast"), () => on("ultrafast"),
|
||||
() => to(pick(MODELS)), () => to(pick(MODELS)), () => to(pick(MODELS)),
|
||||
() => P, () => P, () => U, () => R,
|
||||
];
|
||||
// The v2 session store saves a chat at its first turn, so resuming earlier
|
||||
// fails with "Session not found" on main too. Resume therefore only follows
|
||||
// a prompt (an explicit assumption in tests/tla/acp/AcpSpeed.tla).
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const name = `r${String(index + 1).padStart(2, "0")}-seed${seed}`;
|
||||
const initialModel = pick(MODELS);
|
||||
const defaultFast = rand() < 0.5;
|
||||
let prompted = false;
|
||||
const ops = Array.from({ length: steps }, () => {
|
||||
const op = pick(weighted)();
|
||||
const chosen = op.op === "resume" && !prompted ? P : op;
|
||||
if (chosen.op === "prompt") prompted = true;
|
||||
return chosen;
|
||||
});
|
||||
return { name, initialModel, defaultFast, ops };
|
||||
});
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ runner --
|
||||
|
||||
type Helpers = {
|
||||
startDynamicFakeGateway: (respond: (body: string) => Response, options: object) => any;
|
||||
fakeGatewayFinalText: (text: string) => Response;
|
||||
};
|
||||
|
||||
async function runPlan(plan: Plan, worktree: string, helpers: Helpers, dir: string) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeJson(join(dir, "plan.json"), plan);
|
||||
const fxBin = join(worktree, "zig-out", "bin", "fx");
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "fx-acp-speed-")));
|
||||
const home = join(root, "home");
|
||||
const workspace = join(root, "workspace");
|
||||
mkdirSync(join(home, ".fx"), { recursive: true, mode: 0o700 });
|
||||
mkdirSync(workspace, { recursive: true, mode: 0o700 });
|
||||
writeFileSync(
|
||||
join(home, ".fx", "settings.json"),
|
||||
`${JSON.stringify({ session_titles: false, fast_mode: plan.defaultFast, models: { gateway: plan.initialModel } })}\n`,
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
|
||||
const acpLog: object[] = [];
|
||||
const gatewayLog: object[] = [];
|
||||
let completions: string[] = [];
|
||||
const gateway = helpers.startDynamicFakeGateway((body: string) => {
|
||||
completions.push(body);
|
||||
gatewayLog.push({ n: gatewayLog.length + 1, lane: laneOf(body), model: (() => { try { return JSON.parse(body).model; } catch { return null; } })() });
|
||||
return helpers.fakeGatewayFinalText(`reply ${gatewayLog.length}`);
|
||||
}, { models: () => CATALOG });
|
||||
|
||||
const env: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries({
|
||||
...process.env,
|
||||
NO_COLOR: "1",
|
||||
HOME: home,
|
||||
AI_GATEWAY_API_KEY: "acp-speed-fake-key",
|
||||
FX_DISABLE_KEYCHAIN: "1",
|
||||
FX_SKIP_ONBOARDING: "1",
|
||||
FX_AUTO_UPGRADE: "0",
|
||||
FX_SOUND: "0",
|
||||
FX_GATEWAY_BASE_URL: gateway.baseUrl,
|
||||
FX_GATEWAY_CHAT_URL: gateway.chatUrl,
|
||||
FX_E2E_GATEWAY_MODELS_URL: `${gateway.baseUrl}/coding-agent/v1/models`,
|
||||
FX_E2E_GATEWAY_CHAT_URL: gateway.chatUrl,
|
||||
FX_TRACE_LOG: join(dir, "fx-trace.log"),
|
||||
FX_TRACE_SCOPES: "acp,catalog",
|
||||
})) {
|
||||
if (value === undefined) continue;
|
||||
// FX_PROVIDER and FX_MODEL are process overrides: fx then resumes every
|
||||
// chat on the process's model instead of the chat's saved one, which no
|
||||
// real ACP host (fx-plus included) does. The Gateway is already the default.
|
||||
if (["VERCEL_OIDC_TOKEN", "FX_PROVIDER", "FX_MODEL", "FX_FAST", "FX_ULTRAFAST", "FX_SESSIONS_V2", "FX_PERMISSION_MODE", "FX_DEBUG_RECORD"].includes(key)) continue;
|
||||
env[key] = value;
|
||||
}
|
||||
if (SESSIONS_V2 !== undefined) env.FX_SESSIONS_V2 = SESSIONS_V2;
|
||||
|
||||
const processes: Array<{ exit: number; stderr: string }> = [];
|
||||
const spawn = () => new Acp(
|
||||
Bun.spawn([fxBin, "acp"], { cwd: workspace, env, stdin: "pipe", stdout: "pipe", stderr: "pipe" }),
|
||||
(entry) => acpLog.push({ process: processes.length + 1, ...entry }),
|
||||
);
|
||||
const observations: Observation[] = [];
|
||||
let acp = spawn();
|
||||
let error: string | null = null;
|
||||
try {
|
||||
await acp.call("initialize", { protocolVersion: 1 });
|
||||
const created = await acp.call("session/new", { cwd: workspace, mcpServers: [] });
|
||||
const sessionId = created.result?.sessionId as string | undefined;
|
||||
const view = viewOf(created.result?.configOptions);
|
||||
if (!sessionId || !view) throw new Error(`session/new failed: ${JSON.stringify(created.error ?? created.result)}`);
|
||||
observations.push({ op: "new", arg: "none", ok: true, speed: "none", hasView: true, view });
|
||||
|
||||
const configStep = async (op: string, arg: string, configId: string, value: string) => {
|
||||
const response = await acp.call("session/set_config_option", { sessionId, configId, value });
|
||||
const stepView = response.error ? null : viewOf(response.result?.configOptions);
|
||||
observations.push({
|
||||
op, arg, ok: !response.error, speed: "none",
|
||||
hasView: stepView !== null, view: stepView ?? NO_VIEW,
|
||||
detail: response.error ? { error: response.error } : undefined,
|
||||
});
|
||||
};
|
||||
|
||||
for (const step of plan.ops) {
|
||||
switch (step.op) {
|
||||
case "fast":
|
||||
case "ultrafast":
|
||||
await configStep(step.op, String(step.value), step.op, String(step.value));
|
||||
break;
|
||||
case "model":
|
||||
await configStep("model", step.model, "model", step.model);
|
||||
break;
|
||||
case "unknown":
|
||||
await configStep("unknown", "none", "turbo", "on");
|
||||
break;
|
||||
case "prompt": {
|
||||
completions = [];
|
||||
const response = await acp.call("session/prompt", { sessionId, prompt: [{ type: "text", text: "speed check" }] });
|
||||
const ok = !response.error && response.result?.stopReason === "end_turn";
|
||||
const lanes = completions.map(laneOf);
|
||||
const speed = !ok ? "none" : lanes.length === 1 ? lanes[0]! : `requests:${lanes.join(",") || "0"}`;
|
||||
observations.push({
|
||||
op: "prompt", arg: "none", ok, speed, hasView: false, view: NO_VIEW,
|
||||
detail: { lanes, error: response.error ?? null, stopReason: response.result?.stopReason ?? null },
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "resume": {
|
||||
processes.push(await acp.close());
|
||||
acp = spawn();
|
||||
await acp.call("initialize", { protocolVersion: 1 });
|
||||
const resumed = await acp.call("session/resume", { sessionId, cwd: workspace, mcpServers: [] });
|
||||
const resumedView = resumed.error ? null : viewOf(resumed.result?.configOptions);
|
||||
observations.push({
|
||||
op: "resume", arg: "none", ok: !resumed.error, speed: "none",
|
||||
hasView: resumedView !== null, view: resumedView ?? NO_VIEW,
|
||||
detail: resumed.error ? { error: resumed.error } : undefined,
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (caught) {
|
||||
error = caught instanceof Error ? caught.message : String(caught);
|
||||
} finally {
|
||||
processes.push(await acp.close());
|
||||
gateway.stop();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
writeJson(join(dir, "observations.json"), observations);
|
||||
writeFileSync(join(dir, "acp.jsonl"), acpLog.map((entry) => JSON.stringify(entry)).join("\n") + "\n");
|
||||
writeFileSync(join(dir, "gateway.jsonl"), gatewayLog.map((entry) => JSON.stringify(entry)).join("\n") + "\n");
|
||||
writeJson(join(dir, "processes.json"), processes);
|
||||
return { observations, processes, error };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- controls --
|
||||
|
||||
function syntheticTrace(): Observation[] {
|
||||
const v = (fast: string, ultra: string): View => ({ model: BOTH, fast, ultra });
|
||||
return [
|
||||
{ op: "new", arg: "none", ok: true, speed: "none", hasView: true, view: v("false", "false") },
|
||||
{ op: "fast", arg: "true", ok: true, speed: "none", hasView: true, view: v("true", "false") },
|
||||
{ op: "prompt", arg: "none", ok: true, speed: "fast", hasView: false, view: NO_VIEW },
|
||||
{ op: "ultrafast", arg: "true", ok: true, speed: "none", hasView: true, view: v("false", "true") },
|
||||
{ op: "prompt", arg: "none", ok: true, speed: "ultrafast", hasView: false, view: NO_VIEW },
|
||||
];
|
||||
}
|
||||
|
||||
function tamper(observations: Observation[]): Observation[] | null {
|
||||
const copy: Observation[] = structuredClone(observations);
|
||||
for (let index = copy.length - 1; index >= 0; index -= 1) {
|
||||
const step = copy[index]!;
|
||||
if (step.op === "prompt" && step.ok) {
|
||||
step.speed = step.speed === "fast" ? "standard" : "fast";
|
||||
return copy;
|
||||
}
|
||||
if (step.hasView && step.view.fast !== "absent") {
|
||||
step.view.fast = step.view.fast === "true" ? "false" : "true";
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- main --
|
||||
|
||||
function argValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(name);
|
||||
return index === -1 ? undefined : process.argv[index + 1];
|
||||
}
|
||||
|
||||
// Session store backend under test ("0" = v1 event log, "1" = v2); unset uses fx's default.
|
||||
const SESSIONS_V2 = argValue("--sessions-v2");
|
||||
|
||||
function git(worktree: string, ...args: string[]): string {
|
||||
const result = Bun.spawnSync(["git", "-C", worktree, ...args], { stdout: "pipe", stderr: "pipe" });
|
||||
return new TextDecoder().decode(result.stdout).trim();
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const worktree = resolve(argValue("--worktree") ?? REPO_ROOT);
|
||||
const fxBin = join(worktree, "zig-out", "bin", "fx");
|
||||
if (!existsSync(fxBin)) throw new Error(`missing ${fxBin}; run zig build first`);
|
||||
const label = argValue("--label") ?? "run";
|
||||
const seed = Number(argValue("--seed") ?? "20261006");
|
||||
const randomCount = Number(argValue("--random") ?? "24");
|
||||
const steps = Number(argValue("--steps") ?? "12");
|
||||
|
||||
const reportDir = outDir(`acp-speed-${label}`);
|
||||
mkdirSync(reportDir, { recursive: true });
|
||||
const status = git(worktree, "status", "--porcelain");
|
||||
const manifest = {
|
||||
label,
|
||||
worktree,
|
||||
head: git(worktree, "rev-parse", "HEAD"),
|
||||
dirty: status.length > 0,
|
||||
status: status.split("\n").filter(Boolean),
|
||||
diff_sha256: status ? sha256Text(git(worktree, "diff", "HEAD")) : null,
|
||||
fx_binary_sha256: sha256File(fxBin),
|
||||
toolchain: tlcToolchain(),
|
||||
bun: Bun.version,
|
||||
seed,
|
||||
randomCount,
|
||||
steps,
|
||||
sessionsV2: SESSIONS_V2 ?? "default",
|
||||
argv: process.argv.slice(1),
|
||||
};
|
||||
writeJson(join(reportDir, "manifest.json"), manifest);
|
||||
if (status) writeFileSync(join(reportDir, "worktree.diff"), git(worktree, "diff", "HEAD") + "\n");
|
||||
|
||||
const helpers = await import(join(worktree, "tests", "e2e", "tmux-helpers.ts")) as Helpers;
|
||||
const only = argValue("--only");
|
||||
const plans = [...DIRECTED, ...randomPlans(randomCount, steps, seed)]
|
||||
.filter((plan) => !only || new RegExp(only).test(plan.name));
|
||||
const results = [];
|
||||
for (const plan of plans) {
|
||||
const dir = join(reportDir, "traces", plan.name);
|
||||
const run = await runPlan(plan, worktree, helpers, dir);
|
||||
const processFailed = run.processes.some((proc) => proc.exit !== 0);
|
||||
const checked = run.error ? null : await validate(run.observations, dir, TRACE_CATALOG);
|
||||
const verdict = run.error ? "RUN_ERROR" : processFailed ? "PROCESS_ERROR" : checked!.verdict;
|
||||
results.push({ name: plan.name, verdict, steps: run.observations.length, error: run.error, firstDivergence: checked?.firstDivergence ?? null, processes: run.processes.map((proc) => proc.exit) });
|
||||
const where = checked?.firstDivergence ? ` first divergence at step ${checked.firstDivergence.step}: ${JSON.stringify(checked.firstDivergence.observation)}` : "";
|
||||
console.log(`${plan.name}: ${verdict} (${run.observations.length} steps)${run.error ? ` ${run.error}` : where}`);
|
||||
}
|
||||
|
||||
// Controls show the bridge separates conforming from non-conforming runs.
|
||||
const controls = [];
|
||||
const synthetic = syntheticTrace();
|
||||
controls.push({ name: "synthetic-conforming", expect: "ACCEPTED", got: (await validate(synthetic, join(reportDir, "controls", "synthetic-conforming"), TRACE_CATALOG)).verdict });
|
||||
controls.push({ name: "synthetic-tampered", expect: "REJECTED", got: (await validate(tamper(synthetic)!, join(reportDir, "controls", "synthetic-tampered"), TRACE_CATALOG)).verdict });
|
||||
controls.push({ name: "empty-trace", expect: "TOOL_ERROR", got: (await validate([], join(reportDir, "controls", "empty-trace"), TRACE_CATALOG)).verdict });
|
||||
const firstAccepted = results.find((result) => result.verdict === "ACCEPTED");
|
||||
if (firstAccepted) {
|
||||
const observed = (await Bun.file(join(reportDir, "traces", firstAccepted.name, "observations.json")).json()) as Observation[];
|
||||
const tampered = tamper(observed);
|
||||
if (tampered) {
|
||||
controls.push({ name: `tampered-${firstAccepted.name}`, expect: "REJECTED", got: (await validate(tampered, join(reportDir, "controls", `tampered-${firstAccepted.name}`), TRACE_CATALOG)).verdict });
|
||||
}
|
||||
}
|
||||
for (const control of controls) console.log(`control ${control.name}: ${control.got} (expected ${control.expect})`);
|
||||
|
||||
const counts = results.reduce<Record<string, number>>((acc, result) => ({ ...acc, [result.verdict]: (acc[result.verdict] ?? 0) + 1 }), {});
|
||||
const controlsPassed = controls.every((control) => control.got === control.expect);
|
||||
const summary = { manifest, counts, controlsPassed, controls, results };
|
||||
writeJson(join(reportDir, "summary.json"), summary);
|
||||
console.log(`verdicts: ${JSON.stringify(counts)}; controls ${controlsPassed ? "passed" : "FAILED"}`);
|
||||
console.log(`report: ${reportDir}`);
|
||||
const allAccepted = results.every((result) => result.verdict === "ACCEPTED");
|
||||
process.exit(controlsPassed && allAccepted ? 0 : 1);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,55 @@
|
||||
// Diagnostic only: prints each observed step beside the state the spec
|
||||
// predicts, to explain a TLC rejection. TLC remains the oracle; this mirrors
|
||||
// tests/tla/acp/AcpSpeed.tla for readability and assumes a deterministic start.
|
||||
//
|
||||
// bun tests/tla/drivers/acp/speed/explain.ts <report>/traces/<name>
|
||||
import { join } from "node:path";
|
||||
|
||||
const SUP_FAST = new Set(["openai/gpt-6-astra", "openai/gpt-5.6-priority"]);
|
||||
const SUP_ULTRA = new Set(["openai/gpt-6-astra"]);
|
||||
|
||||
const dir = process.argv[2];
|
||||
if (!dir) throw new Error("usage: bun tests/tla/drivers/acp/speed/explain.ts <trace dir>");
|
||||
const plan = await Bun.file(join(dir, "plan.json")).json();
|
||||
const observations = await Bun.file(join(dir, "observations.json")).json();
|
||||
|
||||
let model: string = plan.initialModel;
|
||||
let fast: boolean = plan.defaultFast;
|
||||
let ultra = false;
|
||||
let prompted = false;
|
||||
const short = (m: string) => m.split("/").pop();
|
||||
const view = () => `${short(model)} fast=${SUP_FAST.has(model) ? fast : "absent"} ultra=${SUP_ULTRA.has(model) ? ultra : "absent"}`;
|
||||
|
||||
console.log(`init ${short(plan.initialModel)} defaultFast=${plan.defaultFast}`);
|
||||
for (const [index, o] of observations.entries()) {
|
||||
let expectOk = true;
|
||||
let expectSpeed = "none";
|
||||
switch (o.op) {
|
||||
case "fast": {
|
||||
const b = o.arg === "true";
|
||||
if (b && !SUP_FAST.has(model)) expectOk = false;
|
||||
else { fast = b; if (b) ultra = false; }
|
||||
break;
|
||||
}
|
||||
case "ultrafast": {
|
||||
const b = o.arg === "true";
|
||||
if (b && !SUP_ULTRA.has(model)) expectOk = false;
|
||||
else { ultra = b; if (b) fast = false; }
|
||||
break;
|
||||
}
|
||||
case "model":
|
||||
model = o.arg; fast = fast && SUP_FAST.has(model); ultra = ultra && SUP_ULTRA.has(model);
|
||||
break;
|
||||
case "unknown":
|
||||
expectOk = false;
|
||||
break;
|
||||
case "prompt":
|
||||
prompted = true;
|
||||
expectSpeed = ultra ? "ultrafast" : fast && SUP_FAST.has(model) ? "fast" : "standard";
|
||||
break;
|
||||
}
|
||||
const seen = o.hasView ? `${short(o.view.model)} fast=${o.view.fast} ultra=${o.view.ultra}` : "";
|
||||
const want = o.hasView ? view() : "";
|
||||
const mismatch = o.ok !== expectOk || (o.op === "prompt" && o.speed !== expectSpeed) || (o.hasView && seen.replace(/"/g, "") !== want);
|
||||
console.log(`${mismatch ? "!!" : " "} ${String(index + 1).padStart(2)} ${o.op.padEnd(9)} ${String(o.arg).split("/").pop()!.padEnd(16)} ok=${o.ok} speed=${o.speed.padEnd(9)} seen[${seen}]${mismatch ? ` want[ok=${expectOk} speed=${expectSpeed} ${want}]` : ""}${prompted ? "" : " (before first prompt)"}`);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// TLC helpers for the ACP speed driver: the shared runner in
|
||||
// ../../lib/tlc.ts, pointed at the specs in tests/tla/acp/.
|
||||
import { join } from "node:path";
|
||||
import { TLA_ROOT, runTlc as runShared } from "../../lib/tlc";
|
||||
|
||||
export { sha256File, sha256Text, tlcToolchain, utcStamp, writeJson } from "../../lib/tlc";
|
||||
export type { TlcRun } from "../../lib/tlc";
|
||||
|
||||
export const SPEC_DIR = join(TLA_ROOT, "acp");
|
||||
|
||||
export function runTlc(dir: string, module: string, cfg: string, specs: string[]) {
|
||||
return runShared(dir, module, cfg, specs, SPEC_DIR);
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
// Shared pieces of the runtime bridge: the ACP stdio client, the
|
||||
// client-visible projection of configOptions, the lane a Gateway request
|
||||
// asks for, and TLC trace validation against tests/tla/acp/AcpSpeedTrace.tla.
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { runTlc, sha256Text, writeJson } from "./tlc";
|
||||
|
||||
// ------------------------------------------------------------ observation --
|
||||
|
||||
export type View = { model: string; fast: string; ultra: string };
|
||||
export type Observation = { op: string; arg: string; ok: boolean; speed: string; hasView: boolean; view: View; detail?: unknown };
|
||||
export const NO_VIEW: View = { model: "", fast: "", ultra: "" };
|
||||
|
||||
export function viewOf(configOptions: unknown): View | null {
|
||||
if (!Array.isArray(configOptions)) return null;
|
||||
const byId = new Map(configOptions.map((option: any) => [option?.id, option]));
|
||||
const model = byId.get("model")?.currentValue;
|
||||
if (typeof model !== "string") return null;
|
||||
const lane = (id: string) => {
|
||||
const option = byId.get(id);
|
||||
if (!option) return "absent";
|
||||
return typeof option.currentValue === "string" ? option.currentValue : "invalid";
|
||||
};
|
||||
return { model, fast: lane("fast"), ultra: lane("ultrafast") };
|
||||
}
|
||||
|
||||
export function laneOf(body: string): string {
|
||||
try {
|
||||
const providerOptions = JSON.parse(body).providerOptions ?? {};
|
||||
if (providerOptions.openai?.serviceTier === "ultrafast") return "ultrafast";
|
||||
if (providerOptions.gateway?.speed === "fast") return "fast";
|
||||
return "standard";
|
||||
} catch {
|
||||
return "unparseable";
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------- ACP client --
|
||||
|
||||
export class Acp {
|
||||
private buffer = "";
|
||||
private lines: string[] = [];
|
||||
private waiters: Array<(line: string) => void> = [];
|
||||
private closed = false;
|
||||
private nextId = 1;
|
||||
stderr = "";
|
||||
readonly exited: Promise<number>;
|
||||
|
||||
constructor(readonly proc: ReturnType<typeof Bun.spawn>, private readonly log: (entry: object) => void) {
|
||||
this.exited = (async () => {
|
||||
const code = await proc.exited;
|
||||
this.closed = true;
|
||||
for (const waiter of this.waiters.splice(0)) waiter("");
|
||||
return code;
|
||||
})();
|
||||
(async () => {
|
||||
const decoder = new TextDecoder();
|
||||
for await (const chunk of proc.stdout as ReadableStream<Uint8Array>) {
|
||||
this.buffer += decoder.decode(chunk);
|
||||
const parts = this.buffer.split("\n");
|
||||
this.buffer = parts.pop() ?? "";
|
||||
for (const line of parts) {
|
||||
if (!line.trim()) continue;
|
||||
const waiter = this.waiters.shift();
|
||||
if (waiter) waiter(line);
|
||||
else this.lines.push(line);
|
||||
}
|
||||
}
|
||||
})();
|
||||
(async () => {
|
||||
const decoder = new TextDecoder();
|
||||
for await (const chunk of proc.stderr as ReadableStream<Uint8Array>) this.stderr += decoder.decode(chunk);
|
||||
})();
|
||||
}
|
||||
|
||||
private async readLine(): Promise<any> {
|
||||
const buffered = this.lines.shift();
|
||||
if (buffered) return JSON.parse(buffered);
|
||||
if (this.closed) throw new Error(`fx acp exited: ${this.stderr}`);
|
||||
const line = await new Promise<string>((resolveLine, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`fx acp read timeout: ${this.stderr}`)), 20_000);
|
||||
this.waiters.push((value) => { clearTimeout(timer); resolveLine(value); });
|
||||
});
|
||||
if (!line) throw new Error(`fx acp exited: ${this.stderr}`);
|
||||
return JSON.parse(line);
|
||||
}
|
||||
|
||||
async call(method: string, params: Record<string, unknown>): Promise<any> {
|
||||
const id = this.nextId++;
|
||||
this.log({ dir: "out", id, method, params });
|
||||
(this.proc.stdin as any).write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
|
||||
(this.proc.stdin as any).flush?.();
|
||||
while (true) {
|
||||
const message = await this.readLine();
|
||||
if ("method" in message && "id" in message) {
|
||||
this.log({ dir: "in-request", method: message.method });
|
||||
(this.proc.stdin as any).write(`${JSON.stringify({ jsonrpc: "2.0", id: message.id, result: { outcome: { outcome: "cancelled" } } })}\n`);
|
||||
continue;
|
||||
}
|
||||
if (message.id === id) {
|
||||
this.log({ dir: "in", id, method, error: message.error ?? null, result: message.result ?? null });
|
||||
return message;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<{ exit: number; stderr: string }> {
|
||||
if (!this.closed) (this.proc.stdin as any).end();
|
||||
const timer = setTimeout(() => this.proc.kill("SIGKILL"), 5_000);
|
||||
const exit = await this.exited;
|
||||
clearTimeout(timer);
|
||||
return { exit, stderr: this.stderr };
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- TLC side --
|
||||
|
||||
const tla = (value: string) => JSON.stringify(value);
|
||||
const tlaSet = (values: string[]) => `{${values.map(tla).join(", ")}}`;
|
||||
|
||||
export type TraceCatalog = { models: string[]; supFast: string[]; supUltra: string[] };
|
||||
|
||||
export function obsModule(observations: Observation[], catalog: TraceCatalog): string {
|
||||
const record = (o: Observation) => {
|
||||
const view = `[model |-> ${tla(o.view.model)}, fast |-> ${tla(o.view.fast)}, ultra |-> ${tla(o.view.ultra)}]`;
|
||||
return `[op |-> ${tla(o.op)}, arg |-> ${tla(o.arg)}, ok |-> ${o.ok ? "TRUE" : "FALSE"}, speed |-> ${tla(o.speed)}, hasView |-> ${o.hasView ? "TRUE" : "FALSE"}, view |-> ${view}]`;
|
||||
};
|
||||
return [
|
||||
"---------------------------- MODULE AcpSpeedObs ----------------------------",
|
||||
"(* Generated from one observed fx acp run. *)",
|
||||
`TModels == ${tlaSet(catalog.models)}`,
|
||||
`TSupFast == ${tlaSet(catalog.supFast)}`,
|
||||
`TSupUltra == ${tlaSet(catalog.supUltra)}`,
|
||||
"TraceLog == <<",
|
||||
observations.map((o) => ` ${record(o)}`).join(",\n"),
|
||||
">>",
|
||||
"=============================================================================",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
const TRACE_CFG = [
|
||||
"CONSTANTS",
|
||||
" Models <- TModels",
|
||||
" SupFast <- TSupFast",
|
||||
" SupUltra <- TSupUltra",
|
||||
"INIT TraceInit",
|
||||
"NEXT TraceNext",
|
||||
"INVARIANT NotDone",
|
||||
"CHECK_DEADLOCK FALSE",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
export type Verdict = "ACCEPTED" | "REJECTED" | "TOOL_ERROR";
|
||||
|
||||
export async function validate(observations: Observation[], dir: string, catalog: TraceCatalog) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
if (observations.length === 0) return { verdict: "TOOL_ERROR" as Verdict, reason: "empty trace", matched: 0 };
|
||||
const obs = obsModule(observations, catalog);
|
||||
writeFileSync(join(dir, "AcpSpeedObs.tla"), obs);
|
||||
writeFileSync(join(dir, "AcpSpeedTrace.cfg"), TRACE_CFG);
|
||||
const run = await runTlc(dir, "AcpSpeedTrace.tla", "AcpSpeedTrace.cfg", ["AcpSpeed.tla", "AcpSpeedTrace.tla"]);
|
||||
const finalIndex = [...run.output.matchAll(/\/\\ i = (\d+)/g)].map((match) => Number(match[1])).at(-1) ?? null;
|
||||
let verdict: Verdict;
|
||||
let matched: number;
|
||||
if (run.exitCode === 12 && run.violated === "NotDone" && finalIndex === observations.length) {
|
||||
verdict = "ACCEPTED";
|
||||
matched = observations.length;
|
||||
} else if (run.exitCode === 0 && /No error has been found/.test(run.output)) {
|
||||
verdict = "REJECTED";
|
||||
matched = run.distinctStates === 0 ? 0 : run.depth ?? 0;
|
||||
} else {
|
||||
verdict = "TOOL_ERROR";
|
||||
matched = 0;
|
||||
}
|
||||
const result = {
|
||||
verdict,
|
||||
matched,
|
||||
length: observations.length,
|
||||
firstDivergence: verdict === "REJECTED" ? { step: matched + 1, observation: observations[matched] ?? null } : null,
|
||||
tlc: { exitCode: run.exitCode, violated: run.violated, finalIndex, depth: run.depth, durationMs: run.durationMs },
|
||||
obs_sha256: sha256Text(obs),
|
||||
};
|
||||
writeJson(join(dir, "verdict.json"), result);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// Connects behavior observed from the real fx binary to TLC.
|
||||
//
|
||||
// The observations become LaunchObserved.tla. LaunchTrace replays them through
|
||||
// LaunchSelection's own actions for one design. ACCEPTED requires TLC exit 12
|
||||
// on the NotAllMatched sentinel, reachable only after every observation
|
||||
// matched. A completed search is REJECTED; then each observation is replayed
|
||||
// alone to name the first one the design cannot produce.
|
||||
//
|
||||
// Strict verdicts: unreadable or missing evidence is never acceptance.
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { TLA_ROOT, tlcToolchain } from "../lib/tlc";
|
||||
import type { Notice, Outcome, Scenario } from "./scenarios";
|
||||
|
||||
export type Design = "current" | "proposed";
|
||||
export type Verdict = "ACCEPTED" | "REJECTED" | "TOOL_ERROR";
|
||||
|
||||
export interface ObservedRequest {
|
||||
present: boolean;
|
||||
model: string;
|
||||
indicator: "on" | "off" | "unobserved";
|
||||
wireFast: boolean;
|
||||
wireUltra: boolean;
|
||||
wireEffort: "supported" | "none" | "other";
|
||||
}
|
||||
|
||||
export interface Observation {
|
||||
id: string;
|
||||
scenario: Scenario;
|
||||
/** The early-submit race: either interleaving may explain the result. */
|
||||
orderFree: boolean;
|
||||
outcome: Outcome;
|
||||
notices: Notice[];
|
||||
req: ObservedRequest;
|
||||
evidence: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface TraceResult {
|
||||
design: Design;
|
||||
verdict: Verdict;
|
||||
detail: string;
|
||||
firstMismatch?: string;
|
||||
hashes: Record<string, string>;
|
||||
}
|
||||
|
||||
const tla = join(TLA_ROOT, "cli");
|
||||
|
||||
const str = (value: string) => JSON.stringify(value);
|
||||
const bool = (value: boolean) => (value ? "TRUE" : "FALSE");
|
||||
|
||||
function scenarioValue(s: Scenario): string {
|
||||
return `[path |-> ${str(s.path)}, model |-> ${str(s.model)}, speed |-> ${str(s.speed)}, ` +
|
||||
`effort |-> ${str(s.effort)}, catalog |-> ${str(s.catalog)}, fastCap |-> ${bool(s.fastCap)}, ` +
|
||||
`ultraCap |-> ${bool(s.ultraCap)}, early |-> ${bool(s.early)}]`;
|
||||
}
|
||||
|
||||
function observationValue(o: Observation): string {
|
||||
const r = o.req;
|
||||
return `[sc |-> ${scenarioValue(o.scenario)}, orderFree |-> ${bool(o.orderFree)}, ` +
|
||||
`outcome |-> ${str(o.outcome)}, notices |-> {${o.notices.map(str).join(", ")}}, ` +
|
||||
`req |-> [present |-> ${bool(r.present)}, model |-> ${str(r.model)}, indicator |-> ${str(r.indicator)}, ` +
|
||||
`wireFast |-> ${bool(r.wireFast)}, wireUltra |-> ${bool(r.wireUltra)}, wireEffort |-> ${str(r.wireEffort)}]]`;
|
||||
}
|
||||
|
||||
export function observedModule(observations: Observation[]): string {
|
||||
return [
|
||||
"---------------------------- MODULE LaunchObserved ----------------------------",
|
||||
"(* Generated by bun/bridge.ts from runtime observations. Do not edit. *)",
|
||||
`Observed == << ${observations.map(observationValue).join(",\n ")} >>`,
|
||||
"=============================================================================",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
const sha = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
|
||||
async function runTrace(observations: Observation[], design: Design, parent: string): Promise<{ verdict: Verdict; log: string; dir: string }> {
|
||||
const dir = mkdtempSync(join(parent, `trace-${design}-`));
|
||||
for (const file of ["LaunchSelection.tla", "LaunchTrace.tla"]) copyFileSync(join(tla, file), join(dir, file));
|
||||
writeFileSync(join(dir, "LaunchObserved.tla"), observedModule(observations));
|
||||
writeFileSync(join(dir, "trace.cfg"), [
|
||||
"SPECIFICATION TraceSpec",
|
||||
"CONSTANTS",
|
||||
` Design = ${str(design)}`,
|
||||
' Mutant = "none"',
|
||||
' Class = "all"',
|
||||
"INVARIANT NotAllMatched",
|
||||
"CHECK_DEADLOCK FALSE",
|
||||
"",
|
||||
].join("\n"));
|
||||
const { java, jar } = tlcToolchain();
|
||||
mkdirSync(join(dir, ".tlc-tmp"), { recursive: true });
|
||||
const child = Bun.spawn([java, "-XX:+UseParallelGC", `-Djava.io.tmpdir=${join(dir, ".tlc-tmp")}`, "-cp", jar, "tlc2.TLC", "-workers", "1",
|
||||
"-metadir", join(dir, ".tlc-states"), "-config", "trace.cfg", "LaunchTrace.tla"], {
|
||||
cwd: dir, stdin: "ignore", stdout: "pipe", stderr: "pipe",
|
||||
});
|
||||
const [stdout, stderr, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]);
|
||||
const log = `${stdout}\n${stderr}`;
|
||||
writeFileSync(join(dir, "tlc.log"), log);
|
||||
if (code === 12 && log.includes("Invariant NotAllMatched is violated")) return { verdict: "ACCEPTED", log, dir };
|
||||
if (code === 0 && log.includes("Model checking completed. No error has been found.")) return { verdict: "REJECTED", log, dir };
|
||||
return { verdict: "TOOL_ERROR", log, dir };
|
||||
}
|
||||
|
||||
/** Replays every observation through one design. `parent` receives the TLC work. */
|
||||
export async function checkTrace(observations: Observation[], design: Design, parent: string): Promise<TraceResult> {
|
||||
const hashes = {
|
||||
"LaunchSelection.tla": sha(join(tla, "LaunchSelection.tla")),
|
||||
"LaunchTrace.tla": sha(join(tla, "LaunchTrace.tla")),
|
||||
observations: createHash("sha256").update(JSON.stringify(observations)).digest("hex"),
|
||||
};
|
||||
if (observations.length === 0) return { design, verdict: "TOOL_ERROR", detail: "no observations", hashes };
|
||||
const whole = await runTrace(observations, design, parent);
|
||||
if (whole.verdict !== "REJECTED") {
|
||||
return { design, verdict: whole.verdict, detail: `${observations.length} observations; TLC work ${whole.dir}`, hashes };
|
||||
}
|
||||
for (const observation of observations) {
|
||||
const single = await runTrace([observation], design, parent);
|
||||
if (single.verdict !== "ACCEPTED") {
|
||||
return {
|
||||
design, verdict: "REJECTED", hashes, firstMismatch: observation.id,
|
||||
detail: `${design} design cannot produce observation ${observation.id} ` +
|
||||
`(outcome=${observation.outcome}, notices=[${observation.notices}], req=${JSON.stringify(observation.req)})`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { design, verdict: "REJECTED", hashes, detail: "each observation matches alone, but not in sequence" };
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
// Runs the real fx binary through every launch-flag scenario the
|
||||
// LaunchSelection TLA+ model covers (tests/tla/cli/LaunchSelection.tla), then has TLC
|
||||
// replay what fx did through the model. No model calls: a local fake Gateway
|
||||
// serves the catalog and accepts only models that exist.
|
||||
//
|
||||
// 1. The binary must conform to exactly one modeled design, current or proposed.
|
||||
// 2. Controls: tampered and empty evidence must not be accepted.
|
||||
// 3. With FX_TLA_PROPOSED=1: the binary must conform to the proposed design.
|
||||
// This fails until fx validates launch flags.
|
||||
import { test } from "bun:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { artifacts, binary, shellQuote, targetHelpers } from "../lib/context";
|
||||
import { checkTrace, type Design, type Observation, type ObservedRequest, type TraceResult } from "./bridge";
|
||||
import {
|
||||
AUTHENTICATED_CATALOG, CONFIGURED, PUBLIC_CATALOG, SCENARIOS, SUPPORTED_EFFORT, TRUTH,
|
||||
askRejection, flagArgs, nameClass, noticesIn, type Outcome, type Scenario,
|
||||
} from "./scenarios";
|
||||
|
||||
const { startDynamicFakeGateway, fakeGatewayFinalText, TmuxSession } = await targetHelpers();
|
||||
const reports = join(artifacts, "tlc");
|
||||
mkdirSync(reports, { recursive: true });
|
||||
const TIMEOUT = 900_000;
|
||||
const DONE_WORD = "MFV_DONE_7";
|
||||
const PROMPT = "Reply with the done word.";
|
||||
const SLOW_CATALOG_MS = 4_000;
|
||||
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
async function waitUntil(condition: () => boolean, timeoutMs: number): Promise<boolean> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (condition()) return true;
|
||||
await sleep(50);
|
||||
}
|
||||
return condition();
|
||||
}
|
||||
|
||||
interface Fixture {
|
||||
root: string; home: string; workspace: string;
|
||||
gateway: any;
|
||||
env: Record<string, string>;
|
||||
catalogRequests: Array<{ authorized: boolean; status: number }>;
|
||||
catalogServed: () => boolean;
|
||||
rejectedModels: string[];
|
||||
}
|
||||
|
||||
function fixture(s: Scenario): Fixture {
|
||||
const root = join(artifacts, s.id);
|
||||
const home = join(root, "home");
|
||||
const workspace = join(root, "workspace");
|
||||
mkdirSync(join(home, ".fx"), { recursive: true, mode: 0o700 });
|
||||
mkdirSync(workspace, { recursive: true });
|
||||
writeFileSync(join(home, ".fx/settings.json"), JSON.stringify({ auto_upgrade: false, model: CONFIGURED }));
|
||||
const catalogRequests: Fixture["catalogRequests"] = [];
|
||||
const rejectedModels: string[] = [];
|
||||
let served = false;
|
||||
let gateway: any;
|
||||
gateway = startDynamicFakeGateway(() => {
|
||||
const model = gateway.requests.at(-1)?.headers.get("ai-language-model-id") ?? "";
|
||||
if (!TRUTH.has(model)) {
|
||||
rejectedModels.push(model);
|
||||
return Response.json({ error: { type: "model_not_found", message: `Model '${model}' not found` } }, { status: 404 });
|
||||
}
|
||||
return fakeGatewayFinalText(DONE_WORD);
|
||||
}, {
|
||||
models: async (request: Request) => {
|
||||
const authorized = Boolean(request.headers.get("authorization"));
|
||||
let response: Response;
|
||||
if (s.catalog === "failed") {
|
||||
response = new Response("catalog unavailable", { status: 500 });
|
||||
} else if (s.catalog === "anonymous" && authorized) {
|
||||
response = Response.json({ error: { message: "unauthorized" } }, { status: 401 });
|
||||
} else {
|
||||
if (s.early) await sleep(SLOW_CATALOG_MS);
|
||||
response = Response.json({ data: s.catalog === "anonymous" ? PUBLIC_CATALOG : AUTHENTICATED_CATALOG });
|
||||
}
|
||||
catalogRequests.push({ authorized, status: response.status });
|
||||
if (response.status === 200 || s.catalog === "failed") served = true;
|
||||
return response;
|
||||
},
|
||||
});
|
||||
const env: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(process.env)) if (value !== undefined && key !== "FX_MODEL") env[key] = value;
|
||||
Object.assign(env, {
|
||||
HOME: home, TMPDIR: root,
|
||||
AI_GATEWAY_API_KEY: "fake-mfv-key", FX_PROVIDER: "gateway",
|
||||
FX_GATEWAY_BASE_URL: gateway.baseUrl, FX_GATEWAY_CHAT_URL: gateway.chatUrl,
|
||||
FX_E2E_GATEWAY_CHAT_URL: gateway.chatUrl,
|
||||
FX_E2E_GATEWAY_MODELS_URL: `${gateway.baseUrl}/coding-agent/v1/models`,
|
||||
FX_SOUND: "0", FX_AUTO_UPGRADE: "0", FX_DISABLE_KEYCHAIN: "1", FX_E2E_DISABLE_DOTENV: "1",
|
||||
// Kept with the scenario evidence so an unexpected outcome can be traced.
|
||||
FX_TRACE_LOG: join(root, "trace.log"), FX_TRACE_SCOPES: "agent,core,gateway,auth,session",
|
||||
});
|
||||
return { root, home, workspace, gateway, env, catalogRequests, catalogServed: () => served, rejectedModels };
|
||||
}
|
||||
|
||||
function wireOf(s: Scenario, f: Fixture, indicator: ObservedRequest["indicator"]): ObservedRequest {
|
||||
const first = f.gateway.requests[0];
|
||||
if (!first) return { present: false, model: "none", indicator: "unobserved", wireFast: false, wireUltra: false, wireEffort: "none" };
|
||||
const sent = first.headers.get("ai-language-model-id") ?? "";
|
||||
let body: any = {};
|
||||
try { body = JSON.parse(first.body); } catch {}
|
||||
const reasoning = body.reasoning;
|
||||
return {
|
||||
present: true,
|
||||
model: nameClass(s, sent),
|
||||
indicator,
|
||||
wireFast: body.providerOptions?.gateway?.speed === "fast",
|
||||
wireUltra: body.providerOptions?.openai?.serviceTier === "ultrafast",
|
||||
wireEffort: reasoning === undefined ? "none" : reasoning === SUPPORTED_EFFORT ? "supported" : "other",
|
||||
};
|
||||
}
|
||||
|
||||
function requestEvidence(f: Fixture) {
|
||||
const first = f.gateway.requests[0];
|
||||
let body: any = {};
|
||||
try { body = JSON.parse(first?.body ?? "{}"); } catch {}
|
||||
return {
|
||||
mainRequests: f.gateway.requests.length,
|
||||
sentModel: first?.headers.get("ai-language-model-id") ?? null,
|
||||
reasoning: body.reasoning ?? null,
|
||||
providerOptions: body.providerOptions ?? null,
|
||||
rejectedModels: f.rejectedModels,
|
||||
catalogRequests: f.catalogRequests,
|
||||
};
|
||||
}
|
||||
|
||||
async function observeAsk(s: Scenario): Promise<Observation> {
|
||||
const f = fixture(s);
|
||||
try {
|
||||
const argv = ["ask", "--no-save", "--auto", ...flagArgs(s), PROMPT];
|
||||
const child = Bun.spawn([binary, ...argv], {
|
||||
cwd: f.workspace, env: f.env, stdin: "ignore", stdout: "pipe", stderr: "pipe",
|
||||
});
|
||||
const timer = setTimeout(() => child.kill(), 60_000);
|
||||
const [stdout, stderr, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]);
|
||||
clearTimeout(timer);
|
||||
let outcome: Outcome = "unknown";
|
||||
const requests = f.gateway.requests.length;
|
||||
if (requests > 0) {
|
||||
if (f.rejectedModels.length > 0) outcome = "gateway_error";
|
||||
else if (code === 0 && stdout.includes(DONE_WORD)) outcome = "sent";
|
||||
} else {
|
||||
outcome = askRejection(stderr) ?? (code !== 0 ? "turn_error" : "unknown");
|
||||
}
|
||||
return {
|
||||
id: s.id, scenario: s, orderFree: false, outcome, notices: [], req: wireOf(s, f, "unobserved"),
|
||||
evidence: { argv, exitCode: code, stdout: stdout.slice(-2000), stderr: stderr.slice(-2000), ...requestEvidence(f) },
|
||||
};
|
||||
} finally {
|
||||
f.gateway.stop();
|
||||
}
|
||||
}
|
||||
|
||||
function footerLine(pane: string): string {
|
||||
return pane.split("\n").map(line => line.trimEnd()).filter(Boolean).at(-1) ?? "";
|
||||
}
|
||||
|
||||
async function observeInteractive(s: Scenario): Promise<Observation> {
|
||||
const f = fixture(s);
|
||||
const cmd = [binary, ...flagArgs(s)].map(shellQuote).join(" ");
|
||||
const session = await TmuxSession.create({ cmd, cwd: f.workspace, env: f.env, width: 140, height: 40, isolated: true, startupWaitMs: 300 });
|
||||
try {
|
||||
await session.waitForComposer(20_000);
|
||||
if (!s.early) {
|
||||
await waitUntil(f.catalogServed, 15_000);
|
||||
await sleep(1_500);
|
||||
}
|
||||
const footer = footerLine(await session.capturePane());
|
||||
const indicator = footer.includes("⚡") ? "on" : "off";
|
||||
const catalogSeenAtSubmit = f.catalogServed();
|
||||
await session.sendText(PROMPT);
|
||||
let outcome: Outcome = "unknown";
|
||||
const sentRequest = await waitUntil(() => f.gateway.requests.length > 0, s.early ? SLOW_CATALOG_MS + 10_000 : 10_000);
|
||||
if (sentRequest) {
|
||||
if (await waitUntil(() => f.rejectedModels.length > 0, 1_500)) {
|
||||
outcome = "gateway_error";
|
||||
} else {
|
||||
const done = await session.waitForText(DONE_WORD, 15_000).then(() => true, () => false);
|
||||
if (done) outcome = "sent";
|
||||
}
|
||||
} else {
|
||||
outcome = "turn_error";
|
||||
}
|
||||
if (s.early) {
|
||||
await waitUntil(f.catalogServed, 15_000);
|
||||
await sleep(2_000);
|
||||
} else {
|
||||
await sleep(500);
|
||||
}
|
||||
const scrollback = await session.captureFullScrollback();
|
||||
return {
|
||||
id: s.id, scenario: s, orderFree: s.early, outcome, notices: noticesIn(scrollback), req: wireOf(s, f, indicator),
|
||||
evidence: { cmd, footerAtSubmit: footer, catalogSeenAtSubmit, scrollbackTail: scrollback.split("\n").slice(-40).join("\n"), ...requestEvidence(f) },
|
||||
};
|
||||
} finally {
|
||||
await session.kill();
|
||||
f.gateway.stop();
|
||||
}
|
||||
}
|
||||
|
||||
let observed: Promise<Observation[]> | undefined;
|
||||
function observations(): Promise<Observation[]> {
|
||||
observed ??= (async () => {
|
||||
const out: Observation[] = [];
|
||||
for (const s of SCENARIOS) {
|
||||
const observation = s.path === "ask" ? await observeAsk(s) : await observeInteractive(s);
|
||||
console.log(`${s.kind.padEnd(8)} ${s.id.padEnd(34)} outcome=${observation.outcome} notices=[${observation.notices}] req=${JSON.stringify(observation.req)}`);
|
||||
out.push(observation);
|
||||
}
|
||||
writeFileSync(join(artifacts, "observations.json"), JSON.stringify(out, null, 2));
|
||||
return out;
|
||||
})();
|
||||
return observed;
|
||||
}
|
||||
|
||||
const verdicts = new Map<Design, Promise<TraceResult>>();
|
||||
function verdict(design: Design): Promise<TraceResult> {
|
||||
if (!verdicts.has(design)) {
|
||||
verdicts.set(design, observations().then(async list => {
|
||||
const result = await checkTrace(list, design, reports);
|
||||
writeFileSync(join(artifacts, `trace-${design}.json`), JSON.stringify(result, null, 2));
|
||||
console.log(`trace ${design}: ${result.verdict} ${result.firstMismatch ?? ""} ${result.detail}`);
|
||||
return result;
|
||||
}));
|
||||
}
|
||||
return verdicts.get(design)!;
|
||||
}
|
||||
|
||||
test("real fx conforms to exactly one modeled design", async () => {
|
||||
const [current, proposed] = await Promise.all([verdict("current"), verdict("proposed")]);
|
||||
const accepted = [current, proposed].filter(result => result.verdict === "ACCEPTED").map(result => result.design);
|
||||
writeFileSync(join(artifacts, "summary.json"), JSON.stringify({ binary, conforms: accepted, current, proposed }, null, 2));
|
||||
assert.equal(accepted.length, 1, `expected exactly one conforming design, got [${accepted}]: current=${current.detail}; proposed=${proposed.detail}`);
|
||||
}, TIMEOUT);
|
||||
|
||||
test("bridge rejects tampered and empty evidence", async () => {
|
||||
const list = await observations();
|
||||
const conforming = (await verdict("current")).verdict === "ACCEPTED" ? "current" : "proposed";
|
||||
const tampered = list.map(item => ({ ...item, req: { ...item.req } }));
|
||||
const target = tampered.find(item => item.req.present)!;
|
||||
target.req.wireFast = !target.req.wireFast;
|
||||
const result = await checkTrace(tampered, conforming, reports);
|
||||
assert.equal(result.verdict, "REJECTED", `tampered evidence for ${target.id} was ${result.verdict}`);
|
||||
assert.equal(result.firstMismatch, target.id);
|
||||
assert.equal((await checkTrace([], conforming, reports)).verdict, "TOOL_ERROR");
|
||||
}, TIMEOUT);
|
||||
|
||||
test.skipIf(process.env.FX_TLA_PROPOSED !== "1")("real fx meets the proposed design", async () => {
|
||||
const proposed = await verdict("proposed");
|
||||
assert.equal(proposed.verdict, "ACCEPTED", proposed.detail);
|
||||
assert.equal((await verdict("current")).verdict, "REJECTED");
|
||||
}, TIMEOUT);
|
||||
@@ -0,0 +1,136 @@
|
||||
// Runtime scenarios for the launch-flag contract, and how each one maps onto
|
||||
// a LaunchSelection scenario. The fake Gateway serves CATALOG; the chat route
|
||||
// accepts exactly the models in TRUTH, like the real Gateway.
|
||||
|
||||
export type Path = "ask" | "interactive";
|
||||
export type ModelReq = "none" | "exact" | "partial_unique" | "partial_ambiguous" | "unknown" | "private_hidden";
|
||||
export type SpeedReq = "none" | "fast" | "ultrafast" | "off";
|
||||
export type EffortReq = "none" | "supported" | "unsupported";
|
||||
export type CatalogMode = "authenticated" | "anonymous" | "failed";
|
||||
export type Outcome =
|
||||
| "sent" | "gateway_error" | "turn_error"
|
||||
| "rejected_model" | "rejected_fast" | "rejected_ultrafast" | "rejected_effort"
|
||||
| "unknown";
|
||||
export type Notice = "model" | "fast" | "ultrafast" | "effort";
|
||||
|
||||
export const CONFIGURED = "openai/mfv-configured";
|
||||
export const SUPPORTED_EFFORT = "high";
|
||||
export const UNSUPPORTED_EFFORT = "minimal";
|
||||
|
||||
const efforts = (values: string[]) => [{ type: "effort", values }];
|
||||
const price = { input: "0.000001", output: "0.000002" };
|
||||
|
||||
/** Fast, Ultra and three efforts. */
|
||||
const FULL = {
|
||||
id: "openai/mfv-full", type: "language", owned_by: "openai", tags: ["tool-use", "reasoning"],
|
||||
fast_options: [{ type: "toggle" }], reasoning_options: efforts(["low", "medium", SUPPORTED_EFFORT]),
|
||||
pricing: { ...price, service_tiers: { priority: { input: "0.000002", output: "0.000004" }, ultrafast: { input: "0.00001", output: "0.00002" } } },
|
||||
};
|
||||
/** No Fast, no Ultra, two efforts. */
|
||||
const plain = (id: string, owned_by: string) => ({
|
||||
id, type: "language", owned_by, tags: ["tool-use", "reasoning"],
|
||||
reasoning_options: efforts(["low", SUPPORTED_EFFORT]), pricing: { ...price },
|
||||
});
|
||||
const PLAIN = plain("anthropic/mfv-plain", "anthropic");
|
||||
const TERRA_ONE = plain("openai/mfv-terra-one", "openai");
|
||||
const TERRA_TWO = plain("openai/mfv-terra-two", "openai");
|
||||
const CONFIGURED_ENTRY = plain(CONFIGURED, "openai");
|
||||
/** Listed only to an authenticated catalog request. */
|
||||
const PRIVATE = plain("team/mfv-private", "team");
|
||||
|
||||
export const PUBLIC_CATALOG = [FULL, PLAIN, TERRA_ONE, TERRA_TWO, CONFIGURED_ENTRY];
|
||||
export const AUTHENTICATED_CATALOG = [...PUBLIC_CATALOG, PRIVATE];
|
||||
export const TRUTH = new Set(AUTHENTICATED_CATALOG.map(entry => entry.id));
|
||||
|
||||
export interface Scenario {
|
||||
id: string;
|
||||
kind: "ideal" | "nonideal";
|
||||
path: Path;
|
||||
model: ModelReq;
|
||||
/** The --model text, when model is not "none". */
|
||||
raw?: string;
|
||||
/** The catalog ID the raw text should resolve to. */
|
||||
canonical?: string;
|
||||
speed: SpeedReq;
|
||||
effort: EffortReq;
|
||||
catalog: CatalogMode;
|
||||
/** Capabilities of the model fx would send. */
|
||||
fastCap: boolean;
|
||||
ultraCap: boolean;
|
||||
/** Interactive only: submit before the catalog arrives (slow catalog). */
|
||||
early: boolean;
|
||||
}
|
||||
|
||||
const full = { fastCap: true, ultraCap: true };
|
||||
const none = { fastCap: false, ultraCap: false };
|
||||
const base = { speed: "none", effort: "none", catalog: "authenticated", early: false } as const;
|
||||
|
||||
export const SCENARIOS: Scenario[] = [
|
||||
// fx ask, ideal
|
||||
{ id: "ask-exact", kind: "ideal", path: "ask", model: "exact", raw: FULL.id, canonical: FULL.id, ...base, ...full },
|
||||
{ id: "ask-exact-fast", kind: "ideal", path: "ask", model: "exact", raw: FULL.id, canonical: FULL.id, ...base, speed: "fast", ...full },
|
||||
{ id: "ask-exact-ultrafast", kind: "ideal", path: "ask", model: "exact", raw: FULL.id, canonical: FULL.id, ...base, speed: "ultrafast", ...full },
|
||||
{ id: "ask-exact-effort", kind: "ideal", path: "ask", model: "exact", raw: FULL.id, canonical: FULL.id, ...base, effort: "supported", ...full },
|
||||
// fx ask, non-ideal
|
||||
{ id: "ask-unknown", kind: "nonideal", path: "ask", model: "unknown", raw: "totally/not-a-real-model", ...base, ...none },
|
||||
{ id: "ask-partial-unique", kind: "nonideal", path: "ask", model: "partial_unique", raw: "mfv-plain", canonical: PLAIN.id, ...base, ...none },
|
||||
{ id: "ask-partial-ambiguous", kind: "nonideal", path: "ask", model: "partial_ambiguous", raw: "mfv-terra", ...base, ...none },
|
||||
{ id: "ask-fast-unsupported", kind: "nonideal", path: "ask", model: "exact", raw: PLAIN.id, canonical: PLAIN.id, ...base, speed: "fast", ...none },
|
||||
{ id: "ask-ultrafast-unsupported", kind: "nonideal", path: "ask", model: "exact", raw: PLAIN.id, canonical: PLAIN.id, ...base, speed: "ultrafast", ...none },
|
||||
{ id: "ask-effort-unsupported", kind: "nonideal", path: "ask", model: "exact", raw: PLAIN.id, canonical: PLAIN.id, ...base, effort: "unsupported", ...none },
|
||||
{ id: "ask-configured-fast-unsupported", kind: "nonideal", path: "ask", model: "none", ...base, speed: "fast", ...none },
|
||||
{ id: "ask-private-anonymous", kind: "nonideal", path: "ask", model: "private_hidden", raw: PRIVATE.id, canonical: PRIVATE.id, ...base, catalog: "anonymous", ...none },
|
||||
{ id: "ask-unknown-catalog-failed", kind: "nonideal", path: "ask", model: "unknown", raw: "totally/not-a-real-model", ...base, catalog: "failed", ...none },
|
||||
{ id: "ask-fast-catalog-failed", kind: "nonideal", path: "ask", model: "exact", raw: FULL.id, canonical: FULL.id, ...base, speed: "fast", catalog: "failed", ...full },
|
||||
// interactive, ideal
|
||||
{ id: "tui-exact-fast", kind: "ideal", path: "interactive", model: "exact", raw: FULL.id, canonical: FULL.id, ...base, speed: "fast", ...full },
|
||||
// interactive, non-ideal
|
||||
{ id: "tui-unknown-fast", kind: "nonideal", path: "interactive", model: "unknown", raw: "totally/not-a-real-model", ...base, speed: "fast", ...none },
|
||||
{ id: "tui-partial-unique", kind: "nonideal", path: "interactive", model: "partial_unique", raw: "mfv-plain", canonical: PLAIN.id, ...base, ...none },
|
||||
{ id: "tui-ultrafast-unsupported", kind: "nonideal", path: "interactive", model: "exact", raw: PLAIN.id, canonical: PLAIN.id, ...base, speed: "ultrafast", ...none },
|
||||
{ id: "tui-effort-unsupported", kind: "nonideal", path: "interactive", model: "exact", raw: PLAIN.id, canonical: PLAIN.id, ...base, effort: "unsupported", ...none },
|
||||
{ id: "tui-private-anonymous", kind: "nonideal", path: "interactive", model: "private_hidden", raw: PRIVATE.id, canonical: PRIVATE.id, ...base, catalog: "anonymous", ...none },
|
||||
{ id: "tui-fast-unsupported-early", kind: "nonideal", path: "interactive", model: "exact", raw: PLAIN.id, canonical: PLAIN.id, ...base, speed: "fast", early: true, ...none },
|
||||
];
|
||||
|
||||
export function flagArgs(s: Scenario): string[] {
|
||||
const args: string[] = [];
|
||||
if (s.model !== "none") args.push("--model", s.raw!);
|
||||
if (s.speed === "fast") args.push("--fast");
|
||||
if (s.speed === "ultrafast") args.push("--ultrafast");
|
||||
if (s.speed === "off") args.push("--no-fast");
|
||||
if (s.effort === "supported") args.push("--effort", SUPPORTED_EFFORT);
|
||||
if (s.effort === "unsupported") args.push("--effort", UNSUPPORTED_EFFORT);
|
||||
return args;
|
||||
}
|
||||
|
||||
/** LaunchSelection's model-name class for a model ID fx actually sent. */
|
||||
/** No scenario passes CONFIGURED as --model, so the order below is unambiguous. */
|
||||
export function nameClass(s: Scenario, sent: string): "configured" | "canonical" | "raw" | "other" {
|
||||
if (sent === CONFIGURED) return "configured";
|
||||
if (s.canonical && sent === s.canonical) return "canonical";
|
||||
if (s.raw && sent === s.raw) return "raw";
|
||||
return "other";
|
||||
}
|
||||
|
||||
/**
|
||||
* Classifies an `fx ask` failure that sent no request from its stderr. Launch
|
||||
* rejections name the model; the turn-time Ultrafast failure does not.
|
||||
*/
|
||||
export function askRejection(stderr: string): Outcome | null {
|
||||
if (/^fx ask: (Unknown model "|Model "[^"]*" matches several catalog models)/m.test(stderr)) return "rejected_model";
|
||||
if (/^fx ask: Fast mode is not available for model "/m.test(stderr)) return "rejected_fast";
|
||||
if (/^fx ask: Ultrafast mode is not available for model "/m.test(stderr)) return "rejected_ultrafast";
|
||||
if (/^fx ask: Reasoning effort (is unavailable|"[^"]*" is not available)/m.test(stderr)) return "rejected_effort";
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Notice markers printed by interactive launch validation. */
|
||||
export function noticesIn(text: string): Notice[] {
|
||||
const found = new Set<Notice>();
|
||||
if (/Unknown model "|matches several catalog models/.test(text)) found.add("model");
|
||||
if (/(^|[^a-z])Fast mode is not available for model/.test(text)) found.add("fast");
|
||||
if (/Ultrafast mode is not available for model/.test(text)) found.add("ultrafast");
|
||||
if (/Reasoning effort "[^"]*" is not available for model|Reasoning effort is unavailable for the active model/.test(text)) found.add("effort");
|
||||
return [...found].sort();
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Run context for the Bun drivers written as `bun test` files: which fx
|
||||
// checkout and binary they drive, where their output goes, and the E2E helpers
|
||||
// they borrow from tests/e2e/.
|
||||
//
|
||||
// FX_WORKTREE checkout under test (default: this one)
|
||||
// FX_BIN binary under test (default: <checkout>/zig-out/bin/fx)
|
||||
// FX_TLA_OUT output root (default: a folder under the system temp dir)
|
||||
// FX_TLA_REPEAT how many times a repeatable check runs (default: 1)
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { REPO_ROOT, outDir } from "./tlc";
|
||||
|
||||
export const worktree = resolve(process.env.FX_WORKTREE ?? REPO_ROOT);
|
||||
export const binary = process.env.FX_BIN ?? join(worktree, "zig-out", "bin", "fx");
|
||||
export const artifacts = outDir(process.env.FX_TLA_CASE ?? "driver");
|
||||
export const repetitions = Number(process.env.FX_TLA_REPEAT ?? "1");
|
||||
|
||||
/// The checkout's tests/e2e/tmux-helpers.ts: tmux sessions and fake gateways.
|
||||
export async function targetHelpers() {
|
||||
const path = join(worktree, "tests", "e2e", "tmux-helpers.ts");
|
||||
if (!existsSync(path)) throw new Error(`missing ${path}`);
|
||||
return await import(pathToFileURL(path).href);
|
||||
}
|
||||
|
||||
export function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", "'\\''")}'`;
|
||||
}
|
||||
|
||||
export function hashFile(path: string): string {
|
||||
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
}
|
||||
|
||||
/// A minimal environment for an fx process under test: no inherited
|
||||
/// credentials, sound, keychain, upgrade check, or dotenv.
|
||||
export function testEnvironment(values: Record<string, string>, inherited = process.env): Record<string, string> {
|
||||
const result: Record<string, string> = {};
|
||||
for (const key of ["PATH", "TERM", "COLORTERM", "LANG", "LC_ALL", "TMPDIR"]) {
|
||||
if (inherited[key]) result[key] = inherited[key]!;
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
FX_SOUND: "0",
|
||||
FX_DISABLE_KEYCHAIN: "1",
|
||||
FX_AUTO_UPGRADE: "0",
|
||||
FX_E2E_DISABLE_DOTENV: "1",
|
||||
...values,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Shared helpers for the Bun drivers under tests/tla/drivers/: find TLC and
|
||||
// Java the same way tests/tla/lib/tlc.sh does, run TLC on a model in a
|
||||
// scratch folder, and locate the fx binary under test.
|
||||
//
|
||||
// TLC comes from TLA2TOOLS_JAR or ~/.local/share/tla/tla2tools.jar. Java comes
|
||||
// from JAVA_HOME, Homebrew's OpenJDK, or PATH. The binary is FX_BIN, or
|
||||
// zig-out/bin/fx in this checkout. Output goes to FX_TLA_OUT, or a fresh folder
|
||||
// under the system temp directory.
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
|
||||
export const REPO_ROOT = resolve(import.meta.dirname, "../../../..");
|
||||
export const TLA_ROOT = join(REPO_ROOT, "tests", "tla");
|
||||
|
||||
export function sha256File(path: string): string {
|
||||
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
}
|
||||
|
||||
export function sha256Text(text: string): string {
|
||||
return createHash("sha256").update(text).digest("hex");
|
||||
}
|
||||
|
||||
/// The fx binary under test. Throws when it has not been built.
|
||||
export function fxBin(): string {
|
||||
const bin = process.env.FX_BIN ?? join(REPO_ROOT, "zig-out", "bin", "fx");
|
||||
if (!existsSync(bin)) throw new Error(`missing ${bin}; run zig build first or set FX_BIN`);
|
||||
return bin;
|
||||
}
|
||||
|
||||
/// A fresh output folder for one driver run. The caller owns its contents.
|
||||
export function outDir(name: string): string {
|
||||
const base = process.env.FX_TLA_OUT ?? join(tmpdir(), "fx-tla");
|
||||
const dir = join(base, name, utcStamp());
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
export type Toolchain = { java: string; javaVersion: string; jar: string; jarSha256: string };
|
||||
|
||||
let toolchain: Toolchain | null = null;
|
||||
|
||||
export function tlcToolchain(): Toolchain {
|
||||
if (toolchain) return toolchain;
|
||||
const jar = process.env.TLA2TOOLS_JAR ?? join(homedir(), ".local", "share", "tla", "tla2tools.jar");
|
||||
if (!existsSync(jar)) throw new Error(`tla2tools.jar not found at ${jar} (set TLA2TOOLS_JAR)`);
|
||||
const which = Bun.spawnSync(["sh", "-c", "command -v java"], { stdout: "pipe" });
|
||||
const candidates = [
|
||||
process.env.JAVA_HOME ? join(process.env.JAVA_HOME, "bin", "java") : "",
|
||||
"/opt/homebrew/opt/openjdk/bin/java",
|
||||
"/usr/local/opt/openjdk/bin/java",
|
||||
new TextDecoder().decode(which.stdout).trim(),
|
||||
];
|
||||
for (const java of candidates) {
|
||||
if (!java || !existsSync(java)) continue;
|
||||
// /usr/bin/java on macOS is a stub that fails -version without a runtime.
|
||||
const probe = Bun.spawnSync([java, "-version"], { stdout: "pipe", stderr: "pipe" });
|
||||
if (probe.exitCode !== 0) continue;
|
||||
const javaVersion = new TextDecoder().decode(probe.stderr).split("\n")[0]?.trim() ?? "";
|
||||
toolchain = { java, javaVersion, jar, jarSha256: sha256File(jar) };
|
||||
return toolchain;
|
||||
}
|
||||
throw new Error("no working Java runtime (set JAVA_HOME or brew install openjdk)");
|
||||
}
|
||||
|
||||
export type TlcRun = {
|
||||
exitCode: number;
|
||||
output: string;
|
||||
violated: string | null;
|
||||
distinctStates: number | null;
|
||||
depth: number | null;
|
||||
argv: string[];
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
/// Runs TLC on `module` with `cfg` from `dir`, after copying `specs` from
|
||||
/// `specDir` into it, so each run keeps the exact files it checked. TLC's
|
||||
/// output is also written to `dir/tlc.log`.
|
||||
export async function runTlc(
|
||||
dir: string,
|
||||
module: string,
|
||||
cfg: string,
|
||||
specs: string[],
|
||||
specDir: string,
|
||||
extraArgs: string[] = [],
|
||||
): Promise<TlcRun> {
|
||||
const { java, jar } = tlcToolchain();
|
||||
mkdirSync(dir, { recursive: true });
|
||||
for (const name of specs) copyFileSync(join(specDir, name), join(dir, name));
|
||||
// A private java.io.tmpdir per run: TLC unpacks its standard modules there,
|
||||
// and concurrent runs sharing one folder can lose them.
|
||||
const javaTmp = join(dir, ".tlc-tmp");
|
||||
mkdirSync(javaTmp, { recursive: true });
|
||||
const argv = [
|
||||
java, "-XX:+UseParallelGC", "-Xss4m", `-Djava.io.tmpdir=${javaTmp}`, "-cp", jar, "tlc2.TLC",
|
||||
"-config", cfg, "-workers", "1", "-metadir", join(dir, ".tlc-states"), "-cleanup", "-nowarning",
|
||||
...extraArgs, module,
|
||||
];
|
||||
const started = performance.now();
|
||||
const proc = Bun.spawn(argv, { cwd: dir, stdout: "pipe", stderr: "pipe" });
|
||||
const [stdout, stderr, exitCode] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
const output = stdout + (stderr ? `\n[stderr]\n${stderr}` : "");
|
||||
writeFileSync(join(dir, "tlc.log"), output);
|
||||
const violated = /Error: (?:Invariant|Action property) (\w+) is violated/.exec(output)?.[1] ?? null;
|
||||
const distinct = /(\d+) distinct states found/.exec(output)?.[1];
|
||||
const depth = /The depth of the complete state graph search is (\d+)/.exec(output)?.[1];
|
||||
return {
|
||||
exitCode,
|
||||
output,
|
||||
violated,
|
||||
distinctStates: distinct ? Number(distinct) : null,
|
||||
depth: depth ? Number(depth) : null,
|
||||
argv,
|
||||
durationMs: Math.round(performance.now() - started),
|
||||
};
|
||||
}
|
||||
|
||||
export function writeJson(path: string, value: unknown) {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
export function utcStamp(): string {
|
||||
return new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z");
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
// Bridge self-test. Real Zig traces from labs/trace_demo.zig plus hand-written
|
||||
// controls, all checked by real TLC against tests/tla/mcp/selftest/Selftest.tla.
|
||||
//
|
||||
// bun test ./tests/tla/drivers/mcp/bridge.test.ts
|
||||
|
||||
import { beforeAll, describe, expect, test } from "bun:test";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { checkTrace, type TraceResult } from "./bridge";
|
||||
|
||||
const ROOT = resolve(import.meta.dir, "../../../..");
|
||||
const SPEC_DIR = join(ROOT, "tests/tla/mcp/selftest");
|
||||
const TIMEOUT = 60_000;
|
||||
|
||||
let work: string;
|
||||
let demo: string;
|
||||
let counter = 0;
|
||||
|
||||
beforeAll(() => {
|
||||
work = mkdtempSync(join(tmpdir(), "mcpv2-bridge-test-"));
|
||||
demo = join(work, "trace_demo");
|
||||
const build = Bun.spawnSync(
|
||||
["zig", "build-exe", "--dep", "mcp", "-Mroot=tests/tla/drivers/mcp/labs/trace_demo.zig", "-Mmcp=src/mcp/mcp.zig", "--name", "trace_demo", `-femit-bin=${demo}`],
|
||||
{ cwd: ROOT, stdout: "pipe", stderr: "pipe" },
|
||||
);
|
||||
if (build.exitCode !== 0) throw new Error(`zig build-exe failed: ${build.stderr.toString()}`);
|
||||
});
|
||||
|
||||
function check(trace: string, machine = "selftest", wrapper = "SelftestTrace"): TraceResult {
|
||||
counter += 1;
|
||||
return checkTrace({ trace, machine, specDir: SPEC_DIR, wrapper, reportDir: join(work, `report-${counter}`) });
|
||||
}
|
||||
|
||||
function zigTrace(...args: string[]): string {
|
||||
counter += 1;
|
||||
const path = join(work, `zig-${counter}.jsonl`);
|
||||
const run = Bun.spawnSync([demo, path, ...args], { stdout: "pipe", stderr: "pipe" });
|
||||
if (run.exitCode !== 0) throw new Error(`trace_demo failed: ${run.stderr.toString()}`);
|
||||
return path;
|
||||
}
|
||||
|
||||
function line(seq: number, event: string, from: string, to: string, outcomes: number): string {
|
||||
return JSON.stringify({ v: 1, seq, machine: "selftest", inst: "demo", event, from, to, effects: [], data: { outcomes } });
|
||||
}
|
||||
|
||||
function handTrace(contents: string): string {
|
||||
counter += 1;
|
||||
const path = join(work, `hand-${counter}.jsonl`);
|
||||
writeFileSync(path, contents);
|
||||
return path;
|
||||
}
|
||||
|
||||
describe("real Zig traces", () => {
|
||||
test("a run that follows the model is ACCEPTED", () => {
|
||||
const result = check(zigTrace("cancel_requested"));
|
||||
expect(result.verdict).toBe("ACCEPTED");
|
||||
expect(result.matched).toBe(1);
|
||||
}, TIMEOUT);
|
||||
|
||||
test("events the machine refuses leave no step, and the rest is ACCEPTED", () => {
|
||||
const result = check(zigTrace("response_received", "cancel_requested"));
|
||||
expect(result.verdict).toBe("ACCEPTED");
|
||||
expect(result.steps).toBe(1);
|
||||
}, TIMEOUT);
|
||||
|
||||
test("the planted late-delivery bug is REJECTED at the bad step", () => {
|
||||
const result = check(zigTrace("--allow-late-delivery", "cancel_requested", "response_received"));
|
||||
expect(result.verdict).toBe("REJECTED");
|
||||
expect(result.steps).toBe(2);
|
||||
expect(result.matched).toBe(1);
|
||||
expect(result.reason).toContain("step 2 (response_received: cancelled -> done)");
|
||||
}, TIMEOUT);
|
||||
});
|
||||
|
||||
describe("controls", () => {
|
||||
test("an empty trace is INCONCLUSIVE", () => {
|
||||
expect(check(handTrace("")).verdict).toBe("INCONCLUSIVE");
|
||||
});
|
||||
|
||||
test("a sequence gap is INCONCLUSIVE", () => {
|
||||
const result = check(handTrace(`${line(1, "cancel_requested", "pending", "cancelled", 1)}\n${line(3, "response_received", "cancelled", "done", 2)}\n`));
|
||||
expect(result.verdict).toBe("INCONCLUSIVE");
|
||||
expect(result.reason).toContain("seq 3");
|
||||
});
|
||||
|
||||
test("a torn last line is INCONCLUSIVE", () => {
|
||||
expect(check(handTrace(line(1, "cancel_requested", "pending", "cancelled", 1))).verdict).toBe("INCONCLUSIVE");
|
||||
});
|
||||
|
||||
test("a trace with no steps for the machine is INCONCLUSIVE", () => {
|
||||
expect(check(handTrace(`${line(1, "cancel_requested", "pending", "cancelled", 1)}\n`), "other").verdict).toBe("INCONCLUSIVE");
|
||||
});
|
||||
|
||||
test("a wrong projected value is REJECTED", () => {
|
||||
const result = check(handTrace(`${line(1, "cancel_requested", "pending", "cancelled", 5)}\n`));
|
||||
expect(result.verdict).toBe("REJECTED");
|
||||
expect(result.matched).toBe(0);
|
||||
}, TIMEOUT);
|
||||
|
||||
test("an unknown event is REJECTED", () => {
|
||||
expect(check(handTrace(`${line(1, "teleported", "pending", "done", 1)}\n`)).verdict).toBe("REJECTED");
|
||||
}, TIMEOUT);
|
||||
|
||||
test("a missing trace wrapper is TOOL_ERROR", () => {
|
||||
expect(check(handTrace(`${line(1, "cancel_requested", "pending", "cancelled", 1)}\n`), "selftest", "NoSuchTrace").verdict).toBe("TOOL_ERROR");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,295 @@
|
||||
// Trace bridge for the MCP-v2 engine models in tests/tla/mcp/.
|
||||
//
|
||||
// Turns a JSONL step trace into a generated TLA+ module (TraceObs) and asks TLC
|
||||
// whether it is a behavior of the machine's model, using a hand-written trace
|
||||
// wrapper that EXTENDS the model and TraceObs. Acceptance uses the sentinel
|
||||
// technique from sessions-v2: the wrapper's NotDone invariant is violated only
|
||||
// once every observed step has been matched, and the matched count must equal
|
||||
// the trace length.
|
||||
//
|
||||
// CLI: bun tests/tla/drivers/mcp/bridge.ts --trace <file> --machine <name> [--instance <id>]
|
||||
// --spec-dir <dir> --wrapper <Module> --report-dir <dir>
|
||||
// Exit: 0 ACCEPTED, 1 REJECTED, 2 INCONCLUSIVE, 3 TOOL_ERROR.
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
|
||||
export type Verdict = "ACCEPTED" | "REJECTED" | "INCONCLUSIVE" | "TOOL_ERROR";
|
||||
|
||||
export const verdictExit: Record<Verdict, number> = {
|
||||
ACCEPTED: 0,
|
||||
REJECTED: 1,
|
||||
INCONCLUSIVE: 2,
|
||||
TOOL_ERROR: 3,
|
||||
};
|
||||
|
||||
export interface TraceRequest {
|
||||
trace: string;
|
||||
machine: string;
|
||||
instance?: string;
|
||||
specDir: string;
|
||||
wrapper: string;
|
||||
reportDir: string;
|
||||
/** Extra definitions for TraceObs, such as model constants the trace can't show. */
|
||||
constants?: { [name: string]: number };
|
||||
}
|
||||
|
||||
export interface TraceResult {
|
||||
verdict: Verdict;
|
||||
reason: string;
|
||||
steps: number;
|
||||
matched: number | null;
|
||||
reportDir: string;
|
||||
}
|
||||
|
||||
type DataValue = number | boolean | string;
|
||||
|
||||
interface TraceLine {
|
||||
v: number;
|
||||
seq: number;
|
||||
machine: string;
|
||||
inst: string;
|
||||
event: string;
|
||||
from: string;
|
||||
to: string;
|
||||
effects: string[];
|
||||
data: { [name: string]: DataValue };
|
||||
}
|
||||
|
||||
const TLC_SH = resolve(import.meta.dir, "../../lib/run-tlc.sh");
|
||||
const SENTINEL = "NotDone";
|
||||
const TLC_INT_MIN = -(2 ** 31);
|
||||
const TLC_INT_MAX = 2 ** 31 - 1;
|
||||
const TLA_IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
||||
|
||||
function sha256(bytes: string | Buffer): string {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
class Inconclusive extends Error {}
|
||||
|
||||
function parseTrace(text: string): TraceLine[] {
|
||||
if (text.length === 0) throw new Inconclusive("trace is empty");
|
||||
if (!text.endsWith("\n")) throw new Inconclusive("last record is incomplete (no trailing newline)");
|
||||
const lines = text.slice(0, -1).split("\n");
|
||||
return lines.map((line, index) => {
|
||||
const at = `line ${index + 1}`;
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(line);
|
||||
} catch {
|
||||
throw new Inconclusive(`${at} is not valid JSON`);
|
||||
}
|
||||
const r = value as TraceLine;
|
||||
if (typeof r !== "object" || r === null) throw new Inconclusive(`${at} is not an object`);
|
||||
if (r.v !== 1) throw new Inconclusive(`${at} has unsupported format version ${String(r.v)}`);
|
||||
if (r.seq !== index + 1) throw new Inconclusive(`${at} has seq ${String(r.seq)}, expected ${index + 1}`);
|
||||
for (const key of ["machine", "inst", "event", "from", "to"] as const) {
|
||||
if (typeof r[key] !== "string") throw new Inconclusive(`${at} field ${key} is not a string`);
|
||||
}
|
||||
if (!Array.isArray(r.effects) || !r.effects.every((e) => typeof e === "string")) {
|
||||
throw new Inconclusive(`${at} field effects is not a list of strings`);
|
||||
}
|
||||
if (typeof r.data !== "object" || r.data === null || Array.isArray(r.data)) {
|
||||
throw new Inconclusive(`${at} field data is not an object`);
|
||||
}
|
||||
for (const [name, v] of Object.entries(r.data)) {
|
||||
if (!TLA_IDENT.test(name)) throw new Inconclusive(`${at} data field ${JSON.stringify(name)} is not a TLA+ identifier`);
|
||||
if (typeof v === "number" && (!Number.isInteger(v) || v < TLC_INT_MIN || v > TLC_INT_MAX)) {
|
||||
throw new Inconclusive(`${at} data field ${name} is not a TLC integer`);
|
||||
}
|
||||
if (!["number", "boolean", "string"].includes(typeof v)) {
|
||||
throw new Inconclusive(`${at} data field ${name} has unsupported type`);
|
||||
}
|
||||
}
|
||||
return r;
|
||||
});
|
||||
}
|
||||
|
||||
function tlaString(s: string): string {
|
||||
const escaped = s
|
||||
.replace(/\\/g, "\\\\")
|
||||
.replace(/"/g, '\\"')
|
||||
.replace(/\n/g, "\\n")
|
||||
.replace(/\r/g, "\\r")
|
||||
.replace(/\t/g, "\\t");
|
||||
if (/[\x00-\x1f]/.test(escaped)) throw new Inconclusive("string contains an unsupported control character");
|
||||
return `"${escaped}"`;
|
||||
}
|
||||
|
||||
function tlaValue(v: DataValue): string {
|
||||
if (typeof v === "boolean") return v ? "TRUE" : "FALSE";
|
||||
if (typeof v === "number") return String(v);
|
||||
return tlaString(v);
|
||||
}
|
||||
|
||||
function observationModule(steps: TraceLine[], source: string, constants: { [name: string]: number }): string {
|
||||
const definitions = Object.entries(constants).map(([name, value]) => {
|
||||
if (!TLA_IDENT.test(name) || !Number.isInteger(value) || value < TLC_INT_MIN || value > TLC_INT_MAX) {
|
||||
throw new Inconclusive(`constant ${name} = ${value} is not a TLA+ identifier and TLC integer`);
|
||||
}
|
||||
return `${name} == ${value}`;
|
||||
});
|
||||
const rows = steps.map((r) => {
|
||||
const fields = Object.entries(r.data);
|
||||
const data = fields.length === 0 ? "<<>>" : `[${fields.map(([k, v]) => `${k} |-> ${tlaValue(v)}`).join(", ")}]`;
|
||||
const effects = `<<${r.effects.map(tlaString).join(", ")}>>`;
|
||||
return ` [event |-> ${tlaString(r.event)}, from |-> ${tlaString(r.from)}, to |-> ${tlaString(r.to)}, effects |-> ${effects}, data |-> ${data}]`;
|
||||
});
|
||||
return [
|
||||
"---- MODULE TraceObs ----",
|
||||
`\\* Generated by tests/tla/drivers/mcp/bridge.ts from ${basename(source)} (${steps.length} steps). Do not edit.`,
|
||||
// Integers for negative values.
|
||||
"EXTENDS Integers",
|
||||
...definitions,
|
||||
"ObsTrace == <<",
|
||||
rows.join(",\n"),
|
||||
">>",
|
||||
"====",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function finish(request: TraceRequest, result: Omit<TraceResult, "reportDir">, extra: object = {}): TraceResult {
|
||||
const full: TraceResult = { ...result, reportDir: request.reportDir };
|
||||
writeFileSync(join(request.reportDir, "verdict.json"), JSON.stringify({ ...full, ...extra }, null, 2) + "\n");
|
||||
return full;
|
||||
}
|
||||
|
||||
export function checkTrace(request: TraceRequest): TraceResult {
|
||||
mkdirSync(request.reportDir, { recursive: true });
|
||||
writeFileSync(join(request.reportDir, "request.json"), JSON.stringify(request, null, 2) + "\n");
|
||||
|
||||
if (!existsSync(request.trace)) {
|
||||
return finish(request, { verdict: "INCONCLUSIVE", reason: "trace file is missing", steps: 0, matched: null });
|
||||
}
|
||||
const raw = readFileSync(request.trace);
|
||||
writeFileSync(join(request.reportDir, "trace.jsonl"), raw);
|
||||
|
||||
let steps: TraceLine[];
|
||||
try {
|
||||
steps = parseTrace(raw.toString("utf8")).filter(
|
||||
(r) => r.machine === request.machine && (request.instance === undefined || r.inst === request.instance),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof Inconclusive) {
|
||||
return finish(request, { verdict: "INCONCLUSIVE", reason: error.message, steps: 0, matched: null });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (steps.length === 0) {
|
||||
return finish(request, {
|
||||
verdict: "INCONCLUSIVE",
|
||||
reason: `no steps for machine ${request.machine}${request.instance ? ` instance ${request.instance}` : ""}`,
|
||||
steps: 0,
|
||||
matched: null,
|
||||
});
|
||||
}
|
||||
|
||||
const wrapperTla = join(request.specDir, `${request.wrapper}.tla`);
|
||||
const wrapperCfg = join(request.specDir, `${request.wrapper}.cfg`);
|
||||
if (!existsSync(wrapperTla) || !existsSync(wrapperCfg)) {
|
||||
return finish(request, {
|
||||
verdict: "TOOL_ERROR",
|
||||
reason: `trace wrapper ${request.wrapper}.tla or .cfg not found in ${request.specDir}`,
|
||||
steps: steps.length,
|
||||
matched: null,
|
||||
});
|
||||
}
|
||||
|
||||
let observation: string;
|
||||
try {
|
||||
observation = observationModule(steps, request.trace, request.constants ?? {});
|
||||
} catch (error) {
|
||||
if (error instanceof Inconclusive) {
|
||||
return finish(request, { verdict: "INCONCLUSIVE", reason: error.message, steps: steps.length, matched: null });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const modelDir = join(request.reportDir, "model");
|
||||
mkdirSync(modelDir, { recursive: true });
|
||||
const hashes: { [file: string]: string } = {};
|
||||
for (const name of readdirSync(request.specDir)) {
|
||||
if (name.endsWith(".tla") || name.endsWith(".cfg")) {
|
||||
copyFileSync(join(request.specDir, name), join(modelDir, name));
|
||||
hashes[name] = sha256(readFileSync(join(modelDir, name)));
|
||||
}
|
||||
}
|
||||
writeFileSync(join(modelDir, "TraceObs.tla"), observation);
|
||||
hashes["TraceObs.tla"] = sha256(observation);
|
||||
hashes["trace.jsonl"] = sha256(raw);
|
||||
|
||||
const tlc = Bun.spawnSync([TLC_SH, join(modelDir, `${request.wrapper}.tla`), join(modelDir, `${request.wrapper}.cfg`), "-deadlock"], {
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
const output = tlc.stdout.toString() + tlc.stderr.toString();
|
||||
writeFileSync(join(request.reportDir, "tlc.log"), output);
|
||||
const jar = /jar_sha256=([0-9a-f]{64})/.exec(output)?.[1] ?? null;
|
||||
const extra = { tlcExit: tlc.exitCode, jarSha256: jar, hashes };
|
||||
|
||||
if (tlc.exitCode === 12 && output.includes(`Invariant ${SENTINEL} is violated`)) {
|
||||
const values = [...output.matchAll(/\/\\ i = (\d+)/g)].map((m) => Number(m[1]));
|
||||
const matched = values.length > 0 ? values[values.length - 1] : null;
|
||||
if (matched === steps.length) {
|
||||
return finish(request, { verdict: "ACCEPTED", reason: `all ${steps.length} steps are a behavior of the model`, steps: steps.length, matched }, extra);
|
||||
}
|
||||
return finish(request, { verdict: "TOOL_ERROR", reason: `sentinel reached at i=${matched}, expected ${steps.length}`, steps: steps.length, matched }, extra);
|
||||
}
|
||||
if (tlc.exitCode === 0 && output.includes("Model checking completed. No error has been found.")) {
|
||||
// Search depth also counts a wrapper's hidden environment steps, so find
|
||||
// the first step the model rejects by checking prefixes of the trace.
|
||||
const accepts = (count: number): boolean => {
|
||||
writeFileSync(join(modelDir, "TraceObs.tla"), observationModule(steps.slice(0, count), request.trace, request.constants ?? {}));
|
||||
const run = Bun.spawnSync([TLC_SH, join(modelDir, `${request.wrapper}.tla`), join(modelDir, `${request.wrapper}.cfg`), "-deadlock"], { stdout: "pipe", stderr: "pipe" });
|
||||
return run.exitCode === 12 && (run.stdout.toString() + run.stderr.toString()).includes(`Invariant ${SENTINEL} is violated`);
|
||||
};
|
||||
let matched = 0;
|
||||
let rejected = steps.length;
|
||||
while (rejected - matched > 1) {
|
||||
const mid = Math.floor((matched + rejected) / 2);
|
||||
if (accepts(mid)) matched = mid;
|
||||
else rejected = mid;
|
||||
}
|
||||
writeFileSync(join(modelDir, "TraceObs.tla"), observation);
|
||||
const next = steps[matched];
|
||||
const detail = `; step ${matched + 1} (${next.event}: ${next.from} -> ${next.to}) is not allowed`;
|
||||
return finish(request, { verdict: "REJECTED", reason: `the model matches only ${matched} of ${steps.length} steps${detail}`, steps: steps.length, matched }, extra);
|
||||
}
|
||||
const firstError = output.split("\n").find((line) => /error/i.test(line)) ?? `TLC exit ${tlc.exitCode}`;
|
||||
return finish(request, { verdict: "TOOL_ERROR", reason: firstError.trim(), steps: steps.length, matched: null }, extra);
|
||||
}
|
||||
|
||||
export function requireAccepted(result: TraceResult): void {
|
||||
if (result.verdict !== "ACCEPTED") {
|
||||
throw new Error(`trace ${result.verdict}: ${result.reason} (evidence: ${result.reportDir})`);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const args = process.argv.slice(2);
|
||||
const get = (flag: string) => {
|
||||
const index = args.indexOf(flag);
|
||||
return index >= 0 ? args[index + 1] : undefined;
|
||||
};
|
||||
const trace = get("--trace");
|
||||
const machine = get("--machine");
|
||||
const specDir = get("--spec-dir");
|
||||
const wrapper = get("--wrapper");
|
||||
const reportDir = get("--report-dir");
|
||||
if (!trace || !machine || !specDir || !wrapper || !reportDir) {
|
||||
console.error("usage: bun tests/tla/drivers/mcp/bridge.ts --trace <file> --machine <name> [--instance <id>] --spec-dir <dir> --wrapper <Module> --report-dir <dir> [--constant Name=Value ...]");
|
||||
process.exit(verdictExit.TOOL_ERROR);
|
||||
}
|
||||
const constants: { [name: string]: number } = {};
|
||||
args.forEach((arg, index) => {
|
||||
if (arg !== "--constant") return;
|
||||
const [name, value] = (args[index + 1] ?? "").split("=");
|
||||
constants[name] = Number(value);
|
||||
});
|
||||
const result = checkTrace({ trace: resolve(trace), machine, instance: get("--instance"), specDir: resolve(specDir), wrapper, reportDir: resolve(reportDir), constants });
|
||||
console.log(JSON.stringify(result));
|
||||
process.exit(verdictExit[result.verdict]);
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Bun driver for the MCP-v2 engine models in tests/tla/mcp/.
|
||||
//
|
||||
// Builds the binaries the selected scenarios need, runs each scenario with an
|
||||
// isolated HOME and MCPV2_TRACE, checks every expected trace through the
|
||||
// bridge, and keeps all evidence under <out>/<unit>/<case>/, where <out> is a
|
||||
// fresh folder under FX_TLA_OUT or the system temp folder.
|
||||
//
|
||||
// Usage: bun tests/tla/drivers/mcp/driver.ts [--unit S05] [--case name] [--repeat N] [--next] [--list]
|
||||
//
|
||||
// The binaries are labs built from labs/ against this checkout's src/mcp/.
|
||||
// Exit: 0 every case passed, 1 a case failed, 2 usage or tooling error.
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { checkTrace } from "./bridge";
|
||||
import type { Scenario, ScenarioContext } from "./scenario";
|
||||
import { REPO_ROOT, outDir } from "../lib/tlc";
|
||||
|
||||
const ROOT = REPO_ROOT;
|
||||
const SCENARIO_DIR = join(import.meta.dir, "scenarios");
|
||||
|
||||
/** Binaries the driver knows how to build, by name. */
|
||||
const BUILDABLE: { [name: string]: string } = {
|
||||
trace_demo: "tests/tla/drivers/mcp/labs/trace_demo.zig",
|
||||
request_lab: "tests/tla/drivers/mcp/labs/request_lab.zig",
|
||||
wire_lab: "tests/tla/drivers/mcp/labs/wire_lab.zig",
|
||||
stdio_lab: "tests/tla/drivers/mcp/labs/stdio_lab.zig",
|
||||
engine_lab: "tests/tla/drivers/mcp/labs/engine_lab.zig",
|
||||
};
|
||||
|
||||
function sha256File(path: string): string {
|
||||
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
}
|
||||
|
||||
function runId(): string {
|
||||
const stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z");
|
||||
return `${stamp}-${crypto.randomUUID().slice(0, 8)}`;
|
||||
}
|
||||
|
||||
async function loadScenarios(): Promise<Scenario[]> {
|
||||
const all: Scenario[] = [];
|
||||
for (const file of readdirSync(SCENARIO_DIR).filter((f) => f.endsWith(".ts")).sort()) {
|
||||
const mod = await import(join(SCENARIO_DIR, file));
|
||||
all.push(...(mod.scenarios as Scenario[]));
|
||||
}
|
||||
const names = new Set<string>();
|
||||
for (const s of all) {
|
||||
if (names.has(s.name)) throw new Error(`duplicate scenario name ${s.name}`);
|
||||
names.add(s.name);
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
function build(names: string[], binDir: string): { [name: string]: { path: string; sha256: string } } {
|
||||
mkdirSync(binDir, { recursive: true });
|
||||
const built: { [name: string]: { path: string; sha256: string } } = {};
|
||||
for (const name of names) {
|
||||
const rootFile = BUILDABLE[name];
|
||||
if (!rootFile) throw new Error(`no build recipe for binary ${name}`);
|
||||
if (!existsSync(join(ROOT, rootFile))) throw new Error(`binary ${name} needs ${rootFile}, which does not exist yet`);
|
||||
const out = join(binDir, name);
|
||||
const result = Bun.spawnSync(
|
||||
["zig", "build-exe", "-O", "ReleaseSafe", "-lc", "--dep", "mcp", `-Mroot=${rootFile}`, "-Mmcp=src/mcp/mcp.zig", "--name", name, `-femit-bin=${out}`],
|
||||
{ cwd: ROOT, stdout: "pipe", stderr: "pipe" },
|
||||
);
|
||||
if (result.exitCode !== 0) throw new Error(`building ${name} failed:\n${result.stderr.toString()}`);
|
||||
built[name] = { path: out, sha256: sha256File(out) };
|
||||
}
|
||||
return built;
|
||||
}
|
||||
|
||||
/** SHA-256 over the sorted `<sha256> <path>` lines of every Zig file in src/mcp/. */
|
||||
function treeHash(): string {
|
||||
const files = (readdirSync(join(ROOT, "src", "mcp"), { recursive: true }) as string[])
|
||||
.filter((name) => name.endsWith(".zig"))
|
||||
.map((name) => `src/mcp/${name}`)
|
||||
.sort();
|
||||
const lines = files.map((file) => `${sha256File(join(ROOT, file))} ${file}\n`).join("");
|
||||
return createHash("sha256").update(lines).digest("hex");
|
||||
}
|
||||
|
||||
interface CaseResult {
|
||||
name: string;
|
||||
unit: string;
|
||||
attempt: number;
|
||||
passed: boolean;
|
||||
problems: string[];
|
||||
exit: number | null;
|
||||
signal: string | null;
|
||||
seconds: number;
|
||||
traces: { machine: string; expected: string; verdict: string; reason: string; reportDir: string }[];
|
||||
evidence: string;
|
||||
}
|
||||
|
||||
function runCase(scenario: Scenario, attempt: number, bins: { [name: string]: string }, evidenceRun: string): CaseResult {
|
||||
const caseDir = join(evidenceRun, attempt > 1 ? `${scenario.name}-${attempt}` : scenario.name);
|
||||
mkdirSync(caseDir, { recursive: true });
|
||||
const scratch = mkdtempSync(join(tmpdir(), "mcpv2-case-"));
|
||||
const ctx: ScenarioContext = {
|
||||
root: ROOT,
|
||||
bin: bins,
|
||||
home: join(scratch, "home"),
|
||||
workdir: join(scratch, "work"),
|
||||
tracePath: join(caseDir, "trace.jsonl"),
|
||||
caseDir,
|
||||
};
|
||||
mkdirSync(ctx.home, { recursive: true });
|
||||
mkdirSync(ctx.workdir, { recursive: true });
|
||||
scenario.setup?.(ctx);
|
||||
|
||||
const command = scenario.command(ctx);
|
||||
const env = {
|
||||
PATH: process.env.PATH ?? "/usr/bin:/bin",
|
||||
HOME: ctx.home,
|
||||
TMPDIR: scratch,
|
||||
MCPV2_HOME: join(ctx.home, ".mcpv2"),
|
||||
MCPV2_TRACE: ctx.tracePath,
|
||||
};
|
||||
writeFileSync(join(caseDir, "scenario.json"), JSON.stringify({ ...scenario, command, env }, null, 2) + "\n");
|
||||
|
||||
const started = performance.now();
|
||||
const proc = Bun.spawnSync(command, { cwd: ctx.workdir, env, stdout: "pipe", stderr: "pipe", timeout: scenario.timeoutMs ?? 30_000 });
|
||||
const seconds = Math.round(performance.now() - started) / 1000;
|
||||
writeFileSync(join(caseDir, "stdout.txt"), proc.stdout);
|
||||
writeFileSync(join(caseDir, "stderr.txt"), proc.stderr);
|
||||
const signal = proc.signalCode ?? null;
|
||||
const exit = proc.exitCode ?? null;
|
||||
writeFileSync(join(caseDir, "process.json"), JSON.stringify({ exit, signal, seconds }, null, 2) + "\n");
|
||||
|
||||
const problems: string[] = [];
|
||||
if (signal !== null) problems.push(`process ended by signal ${signal}`);
|
||||
if (exit !== scenario.expect.exit) problems.push(`exit ${exit}, expected ${scenario.expect.exit}`);
|
||||
if (scenario.expect.stdout) {
|
||||
const problem = scenario.expect.stdout(proc.stdout.toString());
|
||||
if (problem !== null) problems.push(`stdout: ${problem}`);
|
||||
}
|
||||
|
||||
const traces = scenario.expect.traces.map((t, index) => {
|
||||
const result = checkTrace({
|
||||
trace: ctx.tracePath,
|
||||
machine: t.machine,
|
||||
instance: t.instance,
|
||||
specDir: resolve(ROOT, t.specDir),
|
||||
wrapper: t.wrapper,
|
||||
reportDir: join(caseDir, `bridge-${index + 1}-${t.machine}`),
|
||||
constants: t.constants,
|
||||
});
|
||||
if (result.verdict !== t.verdict) problems.push(`trace ${t.machine}: ${result.verdict} (${result.reason}), expected ${t.verdict}`);
|
||||
return { machine: t.machine, expected: t.verdict, verdict: result.verdict, reason: result.reason, reportDir: result.reportDir };
|
||||
});
|
||||
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
const result: CaseResult = { name: scenario.name, unit: scenario.unit, attempt, passed: problems.length === 0, problems, exit, signal, seconds, traces, evidence: caseDir };
|
||||
writeFileSync(join(caseDir, "result.json"), JSON.stringify(result, null, 2) + "\n");
|
||||
return result;
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const args = process.argv.slice(2);
|
||||
const values = (flag: string) => args.flatMap((a, i) => (a === flag && args[i + 1] ? [args[i + 1]] : []));
|
||||
const repeat = Number(values("--repeat")[0] ?? "1");
|
||||
if (!Number.isInteger(repeat) || repeat < 1) {
|
||||
console.error("driver: --repeat must be a positive integer");
|
||||
return 2;
|
||||
}
|
||||
|
||||
let scenarios = await loadScenarios();
|
||||
const units = values("--unit");
|
||||
const cases = values("--case");
|
||||
if (units.length > 0) scenarios = scenarios.filter((s) => units.includes(s.unit));
|
||||
if (cases.length > 0) scenarios = scenarios.filter((s) => cases.includes(s.name));
|
||||
scenarios = scenarios.filter((s) => !s.real);
|
||||
if (args.includes("--list")) {
|
||||
for (const s of scenarios) console.log(`${s.unit} ${s.name.padEnd(36)} ${s.description}`);
|
||||
return 0;
|
||||
}
|
||||
// engine_lab is written for the next MCP-v2 version and does not build
|
||||
// against this checkout's src/mcp/ yet. Its scenarios run only with --next.
|
||||
const nextOnly = (s: Scenario) => s.needs.includes("engine_lab");
|
||||
if (!args.includes("--next")) {
|
||||
const later = scenarios.filter(nextOnly);
|
||||
if (later.length > 0) console.log(`driver: skipping ${later.length} scenario(s) that need engine_lab (next MCP-v2 version); pass --next to run them`);
|
||||
scenarios = scenarios.filter((s) => !nextOnly(s));
|
||||
}
|
||||
if (scenarios.length === 0) {
|
||||
console.error("driver: no scenarios selected");
|
||||
return 2;
|
||||
}
|
||||
|
||||
const id = runId();
|
||||
const out = outDir("mcp");
|
||||
const needs = [...new Set(scenarios.flatMap((s) => s.needs))].sort();
|
||||
const binDir = join(tmpdir(), `mcpv2-bin-${id}`);
|
||||
const built = build(needs, binDir);
|
||||
const environment = { runId: id, treeSha256: treeHash(), binaries: built, bun: Bun.version, args };
|
||||
const bins = Object.fromEntries(Object.entries(built).map(([name, b]) => [name, b.path]));
|
||||
|
||||
const results: CaseResult[] = [];
|
||||
for (const unit of [...new Set(scenarios.map((s) => s.unit))].sort()) {
|
||||
const evidenceRun = join(out, unit);
|
||||
mkdirSync(evidenceRun, { recursive: true });
|
||||
writeFileSync(join(evidenceRun, "environment.json"), JSON.stringify(environment, null, 2) + "\n");
|
||||
const mine: CaseResult[] = [];
|
||||
for (const scenario of scenarios.filter((s) => s.unit === unit)) {
|
||||
for (let attempt = 1; attempt <= repeat; attempt++) {
|
||||
const r = runCase(scenario, attempt, bins, evidenceRun);
|
||||
mine.push(r);
|
||||
const label = repeat > 1 ? `${r.name} #${attempt}` : r.name;
|
||||
console.log(`${r.passed ? "PASS" : "FAIL"} ${unit} ${label.padEnd(36)} exit=${r.exit} ${r.traces.map((t) => `${t.machine}=${t.verdict}`).join(" ")}`);
|
||||
for (const p of r.problems) console.log(` ${p}`);
|
||||
}
|
||||
}
|
||||
writeFileSync(join(evidenceRun, "summary.json"), JSON.stringify(mine, null, 2) + "\n");
|
||||
console.log(`Summary: ${join(evidenceRun, "summary.json")}`);
|
||||
results.push(...mine);
|
||||
}
|
||||
rmSync(binDir, { recursive: true, force: true });
|
||||
return results.every((r) => r.passed) ? 0 : 1;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main()
|
||||
.then((code) => process.exit(code))
|
||||
.catch((error) => {
|
||||
console.error(`driver: tooling error: ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exit(2);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// A stand-in browser for `mcpv2 auth --json` (U12). Runs the command and
|
||||
// passes its output through. When it prints {"authorize": url}, it requests
|
||||
// url without following the redirect, then goes to the redirect's address
|
||||
// itself, as a browser would: mcpv2's loopback listener. With --deny it goes
|
||||
// back with access_denied instead of the code, as a user who refuses.
|
||||
// Exits with the command's exit code.
|
||||
//
|
||||
// Usage: bun tests/tla/drivers/mcp/fakes/browser.ts [--deny] -- <argv...>
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const deny = args[0] === "--deny";
|
||||
const child = Bun.spawn(args.slice(args.indexOf("--") + 1), { stdin: "ignore", stdout: "pipe", stderr: "inherit" });
|
||||
const decoder = new TextDecoder();
|
||||
let pending = "";
|
||||
for await (const chunk of child.stdout) {
|
||||
const text = decoder.decode(chunk, { stream: true });
|
||||
process.stdout.write(text);
|
||||
pending += text;
|
||||
for (let nl = pending.indexOf("\n"); nl >= 0; nl = pending.indexOf("\n")) {
|
||||
const line = pending.slice(0, nl);
|
||||
pending = pending.slice(nl + 1);
|
||||
let url: unknown;
|
||||
try {
|
||||
url = JSON.parse(line).authorize;
|
||||
} catch {}
|
||||
if (typeof url !== "string") continue;
|
||||
const back = new URL((await fetch(url, { redirect: "manual" })).headers.get("location") ?? "");
|
||||
if (deny) {
|
||||
back.searchParams.delete("code");
|
||||
back.searchParams.set("error", "access_denied");
|
||||
}
|
||||
await fetch(back);
|
||||
}
|
||||
}
|
||||
process.exit(await child.exited);
|
||||
@@ -0,0 +1,292 @@
|
||||
// Scripted fake Streamable HTTP MCP server (U06, U07), like fakes/stdio.ts.
|
||||
//
|
||||
// Listens on 127.0.0.1 at a free port and prints {"listening": <port>} first.
|
||||
// Every POST request is matched against the rules in order: the first rule
|
||||
// whose method (and, if given, version, cursor, or tool name) matches and
|
||||
// still has uses left answers it. An unmatched request gets 404 with -32601
|
||||
// (the 2026 answer for an unknown method); notifications and replies get 202.
|
||||
// Every request is logged with its headers to --log as JSONL, and so is every
|
||||
// stream the client closes early.
|
||||
//
|
||||
// 2025 sessions (U07), with --session: initialize's answer assigns
|
||||
// `Mcp-Session-Id: s<n>`. Later requests without it get 400, and with an
|
||||
// ended or unknown one 404 with a JSON-RPC error and no id, as the TypeScript
|
||||
// SDK answers. A rule's `endSession` ends the session after answering. SSE
|
||||
// answers may carry event ids (`ids`), a `retry`, and `poll` closes the stream
|
||||
// after its first events, keeping the answer: a GET with Last-Event-ID gets
|
||||
// it. A GET without one is the listening stream (--listen 405|open|close).
|
||||
// DELETE ends the session (--delete-status, 200 by default).
|
||||
//
|
||||
// OAuth (U09), with --auth <config.json>: see fakes/oauth.ts. Its endpoints are
|
||||
// answered first, and the MCP endpoint's gate runs before any rule.
|
||||
//
|
||||
// Usage: bun tests/tla/drivers/mcp/fakes/http.ts --script <rules.json> [--log <file>]
|
||||
// [--session | --no-session-id] [--listen 405|open|close] [--delete-status <n>]
|
||||
// [--initialized-delay <ms>] [--auth <config.json>]
|
||||
|
||||
import { appendFileSync, readFileSync } from "node:fs";
|
||||
import { createAuth, type AuthConfig } from "./oauth";
|
||||
|
||||
export interface HttpRule {
|
||||
method: string;
|
||||
times?: number;
|
||||
/** Match the 2026 `_meta` protocol version; null matches none sent. */
|
||||
version?: string | null;
|
||||
cursor?: string | null;
|
||||
/** Match `params.name` (tools/call). */
|
||||
name?: string;
|
||||
/** HTTP status, 200 by default. */
|
||||
status?: number;
|
||||
/** Answer as an SSE stream instead of a JSON body. */
|
||||
sse?: boolean;
|
||||
/** Messages sent before the answer (SSE only), such as progress notifications. */
|
||||
before?: unknown[];
|
||||
result?: unknown;
|
||||
error?: { code: number; message: string; data?: unknown };
|
||||
/** A raw body instead of a JSON-RPC answer, with this content type. */
|
||||
raw?: string;
|
||||
contentType?: string;
|
||||
/** Wait before answering, in milliseconds. */
|
||||
delayMs?: number;
|
||||
/** SSE: send the `before` messages, then break the stream without an answer. */
|
||||
drop?: boolean;
|
||||
/** SSE: send the `before` messages, then keep the stream open without an answer. */
|
||||
hang?: boolean;
|
||||
/** Extra response headers. */
|
||||
headers?: Record<string, string>;
|
||||
/** SSE: give every event an id (`<stream>-<n>`), starting with a priming event. */
|
||||
ids?: boolean;
|
||||
/** SSE: the `retry` field sent with the first event. */
|
||||
retry?: number;
|
||||
/** SSE: close the stream after the first events without the answer; a GET with Last-Event-ID resumes it. */
|
||||
poll?: boolean;
|
||||
/** 2025: end the session after answering. */
|
||||
endSession?: boolean;
|
||||
/** SSE messages sent `laterMs` (default 300) after `before`, on a stream left open (U10). */
|
||||
later?: unknown[];
|
||||
laterMs?: number;
|
||||
/** SSE (U11): after `before`, wait for the client's replies to the requests in it (at most `awaitMs`, default 5000). */
|
||||
awaitReply?: boolean;
|
||||
awaitMs?: number;
|
||||
/** Messages sent on the open listening GET stream `laterMs` after the rule fired (U11). */
|
||||
listenLater?: unknown[];
|
||||
/** SSE: send the answer, then leave the stream open, as some servers do. */
|
||||
keepOpen?: boolean;
|
||||
}
|
||||
|
||||
interface Message {
|
||||
jsonrpc?: string;
|
||||
id?: string | number | null;
|
||||
method?: string;
|
||||
params?: { _meta?: Record<string, unknown>; cursor?: string; name?: string };
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const get = (flag: string) => {
|
||||
const index = args.indexOf(flag);
|
||||
return index >= 0 ? args[index + 1] : undefined;
|
||||
};
|
||||
const rules = JSON.parse(readFileSync(get("--script") ?? "/dev/null", "utf8") || "[]") as HttpRule[];
|
||||
const used = rules.map(() => 0);
|
||||
const logPath = get("--log");
|
||||
const sessions = args.includes("--session");
|
||||
// Without --session, answers carry a fixed session id (2026 clients must ignore it) unless --no-session-id.
|
||||
const noSessionId = args.includes("--no-session-id");
|
||||
const listenMode = get("--listen") ?? "405";
|
||||
const deleteStatus = Number(get("--delete-status") ?? "200");
|
||||
// Holds the 202 to notifications/initialized this long, logging when it was answered.
|
||||
const initializedDelay = Number(get("--initialized-delay") ?? "0");
|
||||
// Holds the listening GET's head this long (U11).
|
||||
const listenDelay = Number(get("--listen-delay") ?? "0");
|
||||
const log = (entry: object) => {
|
||||
if (logPath) appendFileSync(logPath, JSON.stringify({ t: Date.now(), ...entry }) + "\n");
|
||||
};
|
||||
|
||||
const authPath = get("--auth");
|
||||
const auth = authPath ? createAuth(JSON.parse(readFileSync(authPath, "utf8")) as AuthConfig, () => `http://127.0.0.1:${server.port}`, log) : null;
|
||||
|
||||
// A rule's result and SSE messages may say "$id" for the request's id, as a
|
||||
// listen's subscriptionId must (U10).
|
||||
const withId = <T>(value: T, id: unknown): T => JSON.parse(JSON.stringify(value).replaceAll('"$id"', JSON.stringify(id ?? null)));
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const sseEvent = (message: unknown, id?: string, retry?: number) =>
|
||||
encoder.encode(`${retry !== undefined ? `retry: ${retry}\n` : ""}${id !== undefined ? `id: ${id}\n` : ""}event: message\ndata: ${message === undefined ? "" : JSON.stringify(message)}\n\n`);
|
||||
|
||||
// 2025 sessions.
|
||||
let sessionCount = 0;
|
||||
let currentSession: string | null = null;
|
||||
const ended = new Set<string>();
|
||||
// Streams kept for a GET with Last-Event-ID: stream name -> the answer still to send.
|
||||
const kept = new Map<string, { answer: unknown; next: number }>();
|
||||
let streamCount = 0;
|
||||
// The open listening GET stream, for `listenLater` (U11).
|
||||
let listenController: ReadableStreamDefaultController | null = null;
|
||||
// Replies to the server's requests that a rule waits for: JSON of the id -> resolve.
|
||||
const replies = new Map<string, () => void>();
|
||||
|
||||
function sessionProblem(req: Request, initialize: boolean): Response | null {
|
||||
if (!sessions || initialize) return null;
|
||||
const id = req.headers.get("mcp-session-id");
|
||||
if (!id) return new Response("Bad Request: No valid session ID provided", { status: 400 });
|
||||
if (id !== currentSession || ended.has(id))
|
||||
return Response.json({ jsonrpc: "2.0", error: { code: -32001, message: "Session not found" }, id: null }, { status: 404 });
|
||||
return null;
|
||||
}
|
||||
|
||||
const server = Bun.serve({
|
||||
hostname: "127.0.0.1",
|
||||
port: 0,
|
||||
idleTimeout: 0,
|
||||
async fetch(req) {
|
||||
const headers = Object.fromEntries(req.headers.entries());
|
||||
const routed = auth ? await auth.route(req) : null;
|
||||
if (routed) return routed;
|
||||
if (req.method === "DELETE") {
|
||||
log({ dir: "in", httpMethod: "DELETE", headers });
|
||||
const denied = auth?.check(req, "DELETE");
|
||||
if (denied) return denied;
|
||||
const problem = sessionProblem(req, false);
|
||||
if (problem) return problem;
|
||||
if (currentSession) ended.add(currentSession);
|
||||
return new Response(null, { status: deleteStatus });
|
||||
}
|
||||
if (req.method === "GET") {
|
||||
log({ dir: "in", httpMethod: "GET", headers });
|
||||
const deniedGet = auth?.check(req, "GET");
|
||||
if (deniedGet) return deniedGet;
|
||||
const problem = sessionProblem(req, false);
|
||||
if (problem) return problem;
|
||||
const last = req.headers.get("last-event-id");
|
||||
if (last) {
|
||||
const name = last.slice(0, last.lastIndexOf("-"));
|
||||
const stream = kept.get(name);
|
||||
if (!stream) return new Response("Unknown stream", { status: 400 });
|
||||
kept.delete(name);
|
||||
return new Response(sseEvent(stream.answer, `${name}-${stream.next}`), { headers: { "content-type": "text/event-stream" } });
|
||||
}
|
||||
if (listenMode === "405") return new Response("Method Not Allowed", { status: 405 });
|
||||
const name = `listen${++streamCount}`;
|
||||
const body = new ReadableStream({
|
||||
start(controller) {
|
||||
listenController = controller;
|
||||
controller.enqueue(sseEvent(undefined, `${name}-0`));
|
||||
if (listenMode === "close") controller.close();
|
||||
},
|
||||
cancel() {
|
||||
listenController = null;
|
||||
log({ dir: "closed", stream: name });
|
||||
},
|
||||
});
|
||||
if (listenDelay > 0) await Bun.sleep(listenDelay);
|
||||
log({ dir: "listen_open", stream: name });
|
||||
return new Response(body, { headers: { "content-type": "text/event-stream" } });
|
||||
}
|
||||
if (req.method !== "POST") {
|
||||
log({ dir: "in", httpMethod: req.method, headers });
|
||||
return new Response("Method Not Allowed", { status: 405 });
|
||||
}
|
||||
const text = await req.text();
|
||||
log({ dir: "in", httpMethod: "POST", headers, body: text });
|
||||
let message: Message;
|
||||
try {
|
||||
message = JSON.parse(text) as Message;
|
||||
} catch {
|
||||
return Response.json({ jsonrpc: "2.0", id: null, error: { code: -32700, message: "Parse error" } }, { status: 400 });
|
||||
}
|
||||
const denied = auth?.check(req, message.method);
|
||||
if (denied) return denied;
|
||||
const problem = sessionProblem(req, message.method === "initialize");
|
||||
if (problem) return problem;
|
||||
// Notifications and replies to server requests.
|
||||
if (message.id === undefined || message.method === undefined) {
|
||||
if (message.method === undefined && message.id !== undefined) replies.get(JSON.stringify(message.id))?.();
|
||||
if (message.method === "notifications/initialized" && initializedDelay > 0) {
|
||||
await Bun.sleep(initializedDelay);
|
||||
log({ dir: "answered", method: message.method });
|
||||
}
|
||||
return new Response(null, { status: 202 });
|
||||
}
|
||||
const version = message.params?._meta?.["io.modelcontextprotocol/protocolVersion"] ?? null;
|
||||
const index = rules.findIndex(
|
||||
(r, i) =>
|
||||
r.method === message.method &&
|
||||
(r.version === undefined || r.version === version) &&
|
||||
(r.cursor === undefined || r.cursor === (message.params?.cursor ?? null)) &&
|
||||
(r.name === undefined || r.name === message.params?.name) &&
|
||||
(r.times === undefined || used[i] < r.times),
|
||||
);
|
||||
if (index < 0) return Response.json({ jsonrpc: "2.0", id: message.id ?? null, error: { code: -32601, message: "Method not found" } }, { status: 404 });
|
||||
used[index] += 1;
|
||||
const rule = rules[index];
|
||||
if (rule.listenLater) {
|
||||
const messages = rule.listenLater;
|
||||
setTimeout(() => {
|
||||
for (const m of messages) listenController?.enqueue(sseEvent(m));
|
||||
}, rule.laterMs ?? 300);
|
||||
}
|
||||
if (rule.delayMs) await Bun.sleep(rule.delayMs);
|
||||
const status = rule.status ?? 200;
|
||||
const extra: Record<string, string> = { ...(rule.headers ?? {}) };
|
||||
if (sessions && message.method === "initialize" && status === 200 && !rule.error) {
|
||||
currentSession = `s${++sessionCount}`;
|
||||
extra["mcp-session-id"] = currentSession;
|
||||
} else if (!sessions && !noSessionId && !("mcp-session-id" in extra)) extra["mcp-session-id"] = "fake-session";
|
||||
if (rule.endSession && currentSession) ended.add(currentSession);
|
||||
if (rule.raw !== undefined) return new Response(rule.raw, { status, headers: { "content-type": rule.contentType ?? "text/plain", ...extra } });
|
||||
const answer = rule.error ? { jsonrpc: "2.0", id: message.id ?? null, error: rule.error } : { jsonrpc: "2.0", id: message.id ?? null, result: withId(rule.result ?? {}, message.id) };
|
||||
if (!rule.sse) return Response.json(answer, { status, headers: extra });
|
||||
const id = message.id;
|
||||
const name = `stream${++streamCount}`;
|
||||
let n = 0;
|
||||
const nextId = () => (rule.ids ? `${name}-${n++}` : undefined);
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
controller.enqueue(encoder.encode(": keep-alive\n\n"));
|
||||
if (rule.ids) controller.enqueue(sseEvent(undefined, nextId(), rule.retry));
|
||||
else if (rule.retry !== undefined) controller.enqueue(encoder.encode(`retry: ${rule.retry}\n\n`));
|
||||
const asked: Promise<void>[] = [];
|
||||
for (const m of rule.before ?? []) {
|
||||
const msg = withId(m, id) as Message;
|
||||
if (rule.awaitReply && msg.method !== undefined && msg.id !== undefined) {
|
||||
asked.push(new Promise<void>((resolve) => replies.set(JSON.stringify(msg.id), resolve)));
|
||||
}
|
||||
controller.enqueue(sseEvent(msg, nextId()));
|
||||
}
|
||||
if (asked.length > 0) {
|
||||
await Promise.race([Promise.all(asked), Bun.sleep(rule.awaitMs ?? 5000)]);
|
||||
log({ dir: "replied", id });
|
||||
}
|
||||
if (rule.later) {
|
||||
await Bun.sleep(rule.laterMs ?? 300);
|
||||
for (const m of rule.later) controller.enqueue(sseEvent(withId(m, id), nextId()));
|
||||
}
|
||||
if (rule.drop) {
|
||||
await Bun.sleep(20);
|
||||
controller.error(new Error("dropped by the fake server"));
|
||||
return;
|
||||
}
|
||||
if (rule.hang) return;
|
||||
if (rule.poll) {
|
||||
kept.set(name, { answer, next: n });
|
||||
await Bun.sleep(20);
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(sseEvent(answer, nextId()));
|
||||
if (!rule.keepOpen) controller.close();
|
||||
},
|
||||
cancel() {
|
||||
log({ dir: "closed", id });
|
||||
},
|
||||
});
|
||||
req.signal.addEventListener("abort", () => log({ dir: "aborted", id }));
|
||||
return new Response(stream, { status, headers: { "content-type": "text/event-stream", "cache-control": "no-cache", ...extra } });
|
||||
},
|
||||
});
|
||||
|
||||
console.log(JSON.stringify({ listening: server.port }));
|
||||
process.on("SIGTERM", () => {
|
||||
server.stop(true);
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,236 @@
|
||||
// Scripted fake OAuth for tests/tla/drivers/mcp/fakes/http.ts (U09): protected resource
|
||||
// metadata for the MCP endpoint, and authorization servers at /as1, /as2, ...,
|
||||
// each with metadata, /authorize, /token, and /register. The MCP endpoint's
|
||||
// gate answers 401 with a challenge without a valid token, and 403
|
||||
// insufficient_scope when the token lacks a method's scopes.
|
||||
//
|
||||
// /authorize approves at once and redirects to the client's redirect_uri with
|
||||
// a code, the state, and iss as configured; nothing is ever sent to that URI.
|
||||
// The token endpoint checks the code, the PKCE verifier, the redirect URI, and
|
||||
// resource; refresh tokens rotate unless told not to. Tokens can be limited to
|
||||
// a number of uses, and after a number of tokens the resource metadata can
|
||||
// name the next server, whose server then rejects the old ones (migration).
|
||||
// Every auth request is logged as {dir: "auth", path, query, form, json,
|
||||
// authorization}.
|
||||
|
||||
export interface AuthServer {
|
||||
/** Where its metadata is: RFC 8414's well-known URI, or OpenID's appended to its path. */
|
||||
metadata?: "oauth" | "oidc";
|
||||
/** The metadata's issuer, if not its own URL. */
|
||||
issuer?: string;
|
||||
/** List S256 (default true). */
|
||||
pkce?: boolean;
|
||||
/** authorization_response_iss_parameter_supported (default true). */
|
||||
issSupported?: boolean;
|
||||
/** What the redirect's iss is (default correct). */
|
||||
iss?: "correct" | "omit" | "wrong";
|
||||
/** A registration endpoint (default true). */
|
||||
dcr?: boolean;
|
||||
scopesSupported?: string[];
|
||||
expiresIn?: number;
|
||||
/** Issue refresh tokens (default true), rotated (default true). */
|
||||
refresh?: boolean;
|
||||
rotate?: boolean;
|
||||
/** Answer every refresh with 400 and this error. */
|
||||
refreshError?: string;
|
||||
/** The token endpoint is plain HTTP to another host. */
|
||||
insecureEndpoint?: boolean;
|
||||
/** The token endpoint is on an internal address: 0.0.0.0, which reaches this machine, on port 1. */
|
||||
internalEndpoint?: boolean;
|
||||
/** The only scopes it grants, whatever is asked. */
|
||||
grantOnly?: string[];
|
||||
/** Deny every authorization request with error=access_denied and this description. */
|
||||
deny?: string;
|
||||
/** A revocation endpoint (RFC 7009); a revoked token is forgotten. */
|
||||
revoke?: boolean;
|
||||
/** Refuse every authorization code at the token endpoint with this error. */
|
||||
codeError?: { error: string; description?: string };
|
||||
/** The token_type its tokens carry, instead of Bearer. */
|
||||
tokenType?: string;
|
||||
/** Appended to every access token it issues, such as "\r\n". */
|
||||
tokenSuffix?: string;
|
||||
}
|
||||
|
||||
export interface AuthConfig {
|
||||
/** Where the resource metadata is: "path" (default), "root", or a path. */
|
||||
prm?: string;
|
||||
/** resource_metadata in the challenge (default true), or this URL instead. */
|
||||
inChallenge?: boolean;
|
||||
/** "foreign": the same metadata URL on another origin (localhost for 127.0.0.1). */
|
||||
challengeUrl?: string;
|
||||
/** The resource metadata's resource, if not the endpoint's URL. */
|
||||
resource?: string;
|
||||
/** The resource metadata names this authorization server instead of its own. */
|
||||
authorizationServer?: string;
|
||||
scopesSupported?: string[];
|
||||
/** The 401 challenge's scope. */
|
||||
challengeScope?: string;
|
||||
/** Scopes a token needs per method. */
|
||||
requiredScopes?: Record<string, string[]>;
|
||||
servers: AuthServer[];
|
||||
/** After this many tokens, the resource metadata names the next server. */
|
||||
migrateAfterTokens?: number;
|
||||
/** Each access token is accepted this many times. */
|
||||
tokenUses?: number;
|
||||
}
|
||||
|
||||
type Log = (entry: object) => void;
|
||||
|
||||
interface Token {
|
||||
server: number;
|
||||
scopes: string[];
|
||||
uses: number;
|
||||
}
|
||||
|
||||
export function createAuth(config: AuthConfig, base: () => string, log: Log) {
|
||||
let active = 0;
|
||||
let tokens = 0;
|
||||
let clients = 0;
|
||||
const access = new Map<string, Token>();
|
||||
const refresh = new Map<string, { server: number; scopes: string[] }>();
|
||||
const codes = new Map<string, { server: number; challenge: string; redirect: string; scopes: string[]; resource: string }>();
|
||||
const prmPath = config.prm === undefined || config.prm === "path" ? "/.well-known/oauth-protected-resource/mcp" : config.prm === "root" ? "/.well-known/oauth-protected-resource" : config.prm;
|
||||
const prmUrl = () => base() + prmPath;
|
||||
const serverUrl = (i: number) => `${base()}/as${i + 1}`;
|
||||
const issuerOf = (i: number) => config.servers[i].issuer ?? serverUrl(i);
|
||||
|
||||
function challenge(error?: string, scope?: string): string {
|
||||
const parts = ["Bearer"];
|
||||
const params: string[] = [];
|
||||
if (error) params.push(`error="${error}"`);
|
||||
if (scope) params.push(`scope="${scope}"`);
|
||||
const url = config.challengeUrl === "foreign" ? prmUrl().replace("127.0.0.1", "localhost") : (config.challengeUrl ?? prmUrl());
|
||||
if (config.inChallenge !== false) params.push(`resource_metadata="${url}"`);
|
||||
return parts.concat(params.join(", ")).join(" ");
|
||||
}
|
||||
|
||||
function metadata(i: number) {
|
||||
const s = config.servers[i];
|
||||
const url = serverUrl(i);
|
||||
return {
|
||||
issuer: issuerOf(i),
|
||||
authorization_endpoint: `${url}/authorize`,
|
||||
token_endpoint: s.insecureEndpoint ? "http://auth.example.com/token" : s.internalEndpoint ? "https://0.0.0.0:1/token" : `${url}/token`,
|
||||
...(s.dcr === false ? {} : { registration_endpoint: `${url}/register` }),
|
||||
...(s.revoke ? { revocation_endpoint: `${url}/revoke` } : {}),
|
||||
response_types_supported: ["code"],
|
||||
grant_types_supported: ["authorization_code", "refresh_token"],
|
||||
...(s.pkce === false ? {} : { code_challenge_methods_supported: ["S256"] }),
|
||||
...(s.issSupported === false ? {} : { authorization_response_iss_parameter_supported: true }),
|
||||
token_endpoint_auth_methods_supported: ["none"],
|
||||
...(s.scopesSupported ? { scopes_supported: s.scopesSupported } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function issue(server: number, scopes: string[]) {
|
||||
const s = config.servers[server];
|
||||
tokens += 1;
|
||||
const at = `at${tokens}${s.tokenSuffix ?? ""}`;
|
||||
access.set(at, { server, scopes, uses: 0 });
|
||||
const body: Record<string, unknown> = { access_token: at, token_type: s.tokenType ?? "Bearer", expires_in: s.expiresIn ?? 3600, scope: scopes.join(" ") };
|
||||
if (s.refresh !== false) {
|
||||
const rt = `rt${tokens}`;
|
||||
refresh.set(rt, { server, scopes });
|
||||
body.refresh_token = rt;
|
||||
}
|
||||
if (config.migrateAfterTokens !== undefined && tokens >= config.migrateAfterTokens && active + 1 < config.servers.length) active += 1;
|
||||
return body;
|
||||
}
|
||||
|
||||
async function sha256url(text: string): Promise<string> {
|
||||
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)));
|
||||
return Buffer.from(digest).toString("base64url");
|
||||
}
|
||||
|
||||
return {
|
||||
/** An auth endpoint's answer, or null for the MCP endpoint. */
|
||||
async route(req: Request): Promise<Response | null> {
|
||||
const url = new URL(req.url);
|
||||
const path = url.pathname;
|
||||
if (path === prmPath && req.method === "GET") {
|
||||
log({ dir: "auth", path, host: req.headers.get("host"), encoding: req.headers.get("accept-encoding") });
|
||||
return Response.json({ resource: config.resource ?? `${base()}/mcp`, authorization_servers: [config.authorizationServer ?? issuerOf(active)], ...(config.scopesSupported ? { scopes_supported: config.scopesSupported } : {}) });
|
||||
}
|
||||
const m = path.match(/^(?:\/\.well-known\/(oauth-authorization-server|openid-configuration))?\/as(\d+)(\/.*)?$/);
|
||||
if (!m) return null;
|
||||
const i = Number(m[2]) - 1;
|
||||
const s = config.servers[i];
|
||||
if (!s) return null;
|
||||
const rest = m[3] ?? "";
|
||||
const form = req.method === "POST" && (req.headers.get("content-type") ?? "").includes("form") ? Object.fromEntries(new URLSearchParams(await req.text())) : undefined;
|
||||
const json = req.method === "POST" && (req.headers.get("content-type") ?? "").includes("json") ? await req.json() : undefined;
|
||||
log({ dir: "auth", path, host: req.headers.get("host"), encoding: req.headers.get("accept-encoding"), query: Object.fromEntries(url.searchParams), form, json, authorization: req.headers.get("authorization") });
|
||||
const wellKnown = m[1] ?? (rest === "/.well-known/openid-configuration" ? "openid-configuration-appended" : undefined);
|
||||
if (wellKnown) {
|
||||
const want = s.metadata === "oidc" ? "openid-configuration-appended" : "oauth-authorization-server";
|
||||
return wellKnown === want ? Response.json(metadata(i)) : new Response("Not Found", { status: 404 });
|
||||
}
|
||||
if (rest === "/authorize") {
|
||||
const q = url.searchParams;
|
||||
const code = `code${codes.size + 1}`;
|
||||
codes.set(code, { server: i, challenge: q.get("code_challenge") ?? "", redirect: q.get("redirect_uri") ?? "", scopes: (q.get("scope") ?? "").split(" ").filter(Boolean), resource: q.get("resource") ?? "" });
|
||||
const back = new URL(q.get("redirect_uri") ?? "http://invalid/");
|
||||
if (s.deny) {
|
||||
back.searchParams.set("error", "access_denied");
|
||||
back.searchParams.set("error_description", s.deny);
|
||||
} else back.searchParams.set("code", code);
|
||||
if (q.get("state")) back.searchParams.set("state", q.get("state")!);
|
||||
const iss = s.iss ?? "correct";
|
||||
if (iss === "correct") back.searchParams.set("iss", issuerOf(i));
|
||||
if (iss === "wrong") back.searchParams.set("iss", "https://evil.example.com");
|
||||
return Response.redirect(back.toString(), 302);
|
||||
}
|
||||
if (rest === "/register" && req.method === "POST") {
|
||||
clients += 1;
|
||||
return Response.json({ client_id: `client${clients}-as${i + 1}`, token_endpoint_auth_method: "none", redirect_uris: json?.redirect_uris ?? [] }, { status: 201 });
|
||||
}
|
||||
if (rest === "/revoke" && req.method === "POST" && form && s.revoke) {
|
||||
refresh.delete(form.token);
|
||||
access.delete(form.token);
|
||||
return new Response(null, { status: 200 });
|
||||
}
|
||||
if (rest === "/token" && req.method === "POST" && form) {
|
||||
if (form.grant_type === "authorization_code") {
|
||||
const c = codes.get(form.code);
|
||||
codes.delete(form.code);
|
||||
if (s.codeError) {
|
||||
const { error, description } = s.codeError;
|
||||
return Response.json(description === undefined ? { error } : { error, error_description: description }, { status: 400 });
|
||||
}
|
||||
if (!c || c.server !== i || c.redirect !== form.redirect_uri || !form.resource || c.challenge !== (await sha256url(form.code_verifier ?? "")))
|
||||
return Response.json({ error: "invalid_grant" }, { status: 400 });
|
||||
return Response.json(issue(i, s.grantOnly ? c.scopes.filter((x) => s.grantOnly!.includes(x)) : c.scopes));
|
||||
}
|
||||
if (form.grant_type === "refresh_token") {
|
||||
if (s.refreshError) return Response.json({ error: s.refreshError }, { status: 400 });
|
||||
const r = refresh.get(form.refresh_token);
|
||||
if (!r || r.server !== i || !form.resource) return Response.json({ error: "invalid_grant" }, { status: 400 });
|
||||
const body = issue(i, r.scopes);
|
||||
if (s.rotate === false) {
|
||||
refresh.delete(body.refresh_token as string);
|
||||
delete body.refresh_token;
|
||||
} else refresh.delete(form.refresh_token);
|
||||
return Response.json(body);
|
||||
}
|
||||
return Response.json({ error: "unsupported_grant_type" }, { status: 400 });
|
||||
}
|
||||
return new Response("Not Found", { status: 404 });
|
||||
},
|
||||
|
||||
/** The gate in front of the MCP endpoint: an answer for a request it refuses. */
|
||||
check(req: Request, method: string | undefined): Response | null {
|
||||
const header = req.headers.get("authorization") ?? "";
|
||||
const t = header.startsWith("Bearer ") ? access.get(header.slice(7)) : undefined;
|
||||
const limit = config.tokenUses ?? Infinity;
|
||||
if (!t || t.server !== active || t.uses >= limit) {
|
||||
return Response.json({ error: "invalid_token" }, { status: 401, headers: { "www-authenticate": challenge(undefined, config.challengeScope) } });
|
||||
}
|
||||
const needed = (method && config.requiredScopes?.[method]) ?? [];
|
||||
if (!needed.every((n) => t.scopes.includes(n))) {
|
||||
return Response.json({ error: "insufficient_scope" }, { status: 403, headers: { "www-authenticate": challenge("insufficient_scope", needed.join(" ")) } });
|
||||
}
|
||||
t.uses += 1;
|
||||
return null;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Tests for the scripted fake stdio server.
|
||||
//
|
||||
// bun test ./tests/tla/drivers/mcp/fakes/stdio.test.ts
|
||||
|
||||
import { afterEach, expect, test } from "bun:test";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Rule } from "./stdio";
|
||||
|
||||
const SERVER = join(import.meta.dir, "stdio.ts");
|
||||
let dirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
||||
dirs = [];
|
||||
});
|
||||
|
||||
async function exchange(rules: Rule[], lines: string[], flags: string[] = []) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mcpv2-fake-stdio-"));
|
||||
dirs.push(dir);
|
||||
const script = join(dir, "rules.json");
|
||||
const log = join(dir, "log.jsonl");
|
||||
writeFileSync(script, JSON.stringify(rules));
|
||||
const proc = Bun.spawn([process.execPath, SERVER, "--script", script, "--log", log, ...flags], { stdin: "pipe", stdout: "pipe", stderr: "pipe" });
|
||||
for (const line of lines) proc.stdin.write(line + "\n");
|
||||
proc.stdin.end();
|
||||
const exit = await proc.exited;
|
||||
const parse = (l: string) => {
|
||||
try {
|
||||
return JSON.parse(l);
|
||||
} catch {
|
||||
return l; // deliberately malformed output from a `raw` rule
|
||||
}
|
||||
};
|
||||
const out = (await new Response(proc.stdout).text()).split("\n").filter(Boolean).map(parse);
|
||||
const logged = readFileSync(log, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
||||
return { exit, out, logged };
|
||||
}
|
||||
|
||||
const request = (id: number, method: string) => JSON.stringify({ jsonrpc: "2.0", id, method, params: {} });
|
||||
|
||||
test("answers a matching request with the scripted result and copies the id", async () => {
|
||||
const { exit, out } = await exchange([{ method: "server/discover", result: { supportedVersions: ["2026-07-28"] } }], [request(7, "server/discover")]);
|
||||
expect(exit).toBe(0);
|
||||
expect(out).toEqual([{ jsonrpc: "2.0", id: 7, result: { supportedVersions: ["2026-07-28"] } }]);
|
||||
});
|
||||
|
||||
test("answers an unmatched request with -32601 and ignores notifications", async () => {
|
||||
const { out } = await exchange([], [request(1, "tools/list"), JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" })]);
|
||||
expect(out).toEqual([{ jsonrpc: "2.0", id: 1, error: { code: -32601, message: "Method not found" } }]);
|
||||
});
|
||||
|
||||
test("rules with a use limit fall through to the next matching rule", async () => {
|
||||
const rules: Rule[] = [
|
||||
{ method: "tools/list", times: 1, error: { code: -32022, message: "Unsupported protocol version", data: { supported: ["2025-11-25"] } } },
|
||||
{ method: "tools/list", result: { tools: [] } },
|
||||
];
|
||||
const { out } = await exchange(rules, [request(1, "tools/list"), request(2, "tools/list")]);
|
||||
expect(out[0].error.code).toBe(-32022);
|
||||
expect(out[1]).toEqual({ jsonrpc: "2.0", id: 2, result: { tools: [] } });
|
||||
});
|
||||
|
||||
test("can send notifications before the reply, raw malformed lines, and crash", async () => {
|
||||
const rules: Rule[] = [
|
||||
{ method: "tools/call", times: 1, before: [{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: 1, progress: 1 } }], result: { content: [] } },
|
||||
{ method: "tools/call", raw: "{not json", then: "crash" },
|
||||
];
|
||||
const proc = await exchange(rules, [request(1, "tools/call"), request(2, "tools/call"), request(3, "tools/call")]);
|
||||
expect(proc.exit).toBe(1);
|
||||
expect(proc.out[proc.out.length - 1]).toBe("{not json");
|
||||
const lines = proc.logged.filter((l) => l.dir === "out").map((l) => l.line);
|
||||
expect(lines).toEqual([
|
||||
JSON.stringify({ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: 1, progress: 1 } }),
|
||||
JSON.stringify({ jsonrpc: "2.0", id: 1, result: { content: [] } }),
|
||||
"{not json",
|
||||
]);
|
||||
expect(proc.logged.filter((l) => l.dir === "in").length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
test("rules can match the request's 2026 protocol version, or its absence", async () => {
|
||||
const rules: Rule[] = [
|
||||
{ method: "tools/list", version: "2026-07-28", result: { era: "modern" } },
|
||||
{ method: "tools/list", version: null, result: { era: "legacy" } },
|
||||
];
|
||||
const modern = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: { _meta: { "io.modelcontextprotocol/protocolVersion": "2026-07-28" } } });
|
||||
const other = JSON.stringify({ jsonrpc: "2.0", id: 2, method: "tools/list", params: { _meta: { "io.modelcontextprotocol/protocolVersion": "2027-01-01" } } });
|
||||
const { out } = await exchange(rules, [modern, request(3, "tools/list"), other]);
|
||||
expect(out.map((m: { id: number; result?: { era: string }; error?: { code: number } }) => [m.id, m.result?.era ?? m.error?.code])).toEqual([
|
||||
[1, "modern"],
|
||||
[3, "legacy"],
|
||||
[2, -32601],
|
||||
]);
|
||||
});
|
||||
|
||||
test("--start-delay holds input until the delay passes", async () => {
|
||||
const started = Date.now();
|
||||
const { out } = await exchange([{ method: "ping", result: {} }], [request(1, "ping")], ["--start-delay", "300"]);
|
||||
expect(out).toEqual([{ jsonrpc: "2.0", id: 1, result: {} }]);
|
||||
expect(Date.now() - started).toBeGreaterThanOrEqual(300);
|
||||
});
|
||||
|
||||
test("rules can match the request's cursor, or its absence", async () => {
|
||||
const rules: Rule[] = [
|
||||
{ method: "tools/list", cursor: null, result: { page: 1 } },
|
||||
{ method: "tools/list", cursor: "", result: { page: 2 } },
|
||||
];
|
||||
const list = (id: number, cursor?: string) => JSON.stringify({ jsonrpc: "2.0", id, method: "tools/list", params: cursor === undefined ? {} : { cursor } });
|
||||
const { out } = await exchange(rules, [list(1), list(2, ""), list(3, "x")]);
|
||||
expect(out.map((m: { id: number; result?: { page: number }; error?: { code: number } }) => [m.id, m.result?.page ?? m.error?.code])).toEqual([
|
||||
[1, 1],
|
||||
[2, 2],
|
||||
[3, -32601],
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
// Scripted fake MCP server over stdio.
|
||||
//
|
||||
// Reads newline-delimited JSON-RPC from stdin and answers from a script. The
|
||||
// first rule whose method matches and still has uses left is applied. A
|
||||
// request with no matching rule gets -32601. Notifications get no reply unless
|
||||
// a rule says so. Every line in and out is appended to --log as JSONL.
|
||||
//
|
||||
// Usage: bun tests/tla/drivers/mcp/fakes/stdio.ts --script <rules.json> [--log <file>]
|
||||
// [--banner <text>] [--exit-at-start <code>] [--start-delay <ms>] [--linger] [--ignore-term]
|
||||
// [--once <file>] [--exit-after <ms>]
|
||||
//
|
||||
// --banner write <text> to stdout first, as a server printing a non-MCP line would
|
||||
// --exit-at-start exit with <code> before reading anything (a crash loop)
|
||||
// --once a rule's "exit" or "crash" happens in the first process only, which leaves <file>
|
||||
// --exit-after exit with code 1 this long after starting (a server that dies later)
|
||||
// --start-delay hold every input line until <ms> after start (a slow server)
|
||||
// --linger keep running after stdin closes (forces the client's SIGTERM)
|
||||
// --ignore-term ignore SIGTERM (forces the client's SIGKILL)
|
||||
|
||||
import { appendFileSync, existsSync, readFileSync, writeFileSync, writeSync } from "node:fs";
|
||||
|
||||
export interface Rule {
|
||||
/** JSON-RPC method to match. "*" matches any method. */
|
||||
method: string;
|
||||
/** Match only requests whose 2026 `_meta` protocol version is this; null matches none sent. */
|
||||
version?: string | null;
|
||||
/** Match only requests with this `cursor` param; null matches none sent. */
|
||||
cursor?: string | null;
|
||||
/** Match only tools/call requests for this tool (U12). */
|
||||
tool?: string;
|
||||
/** How many times the rule applies. Defaults to unlimited. */
|
||||
times?: number;
|
||||
/** Lines to write before the reply, for example notifications. */
|
||||
before?: unknown[];
|
||||
/** Reply: a result, an error, a raw line (for malformed output), or nothing. */
|
||||
result?: unknown;
|
||||
error?: { code: number; message: string; data?: unknown };
|
||||
raw?: string;
|
||||
noReply?: boolean;
|
||||
/** Wait before replying, in milliseconds. */
|
||||
delayMs?: number;
|
||||
/** Lines to write to stderr before the reply, `stderrRepeat` times (default 1). */
|
||||
stderr?: string[];
|
||||
stderrRepeat?: number;
|
||||
/** Write one stdout line of this many "x" bytes before the reply (an oversized line). */
|
||||
rawBytes?: number;
|
||||
/** After replying: "exit" closes cleanly with code 0, "crash" exits with code 1. */
|
||||
then?: "exit" | "crash";
|
||||
}
|
||||
|
||||
// A rule's result and `before` lines may say "$id" for the request's id, as a
|
||||
// listen's subscriptionId must (U10).
|
||||
const withId = <T>(value: T, id: unknown): T => JSON.parse(JSON.stringify(value).replaceAll('"$id"', JSON.stringify(id ?? null)));
|
||||
|
||||
interface Message {
|
||||
jsonrpc?: string;
|
||||
id?: string | number | null;
|
||||
method?: string;
|
||||
params?: unknown;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const args = process.argv.slice(2);
|
||||
const get = (flag: string) => {
|
||||
const index = args.indexOf(flag);
|
||||
return index >= 0 ? args[index + 1] : undefined;
|
||||
};
|
||||
const scriptPath = get("--script");
|
||||
if (!scriptPath) {
|
||||
process.stderr.write("usage: fakes/stdio.ts --script <rules.json> [--log <file>]\n");
|
||||
process.exit(2);
|
||||
}
|
||||
const exitAtStart = get("--exit-at-start");
|
||||
if (exitAtStart !== undefined) process.exit(Number(exitAtStart));
|
||||
const exitAfter = get("--exit-after");
|
||||
if (exitAfter !== undefined) setTimeout(() => process.exit(1), Number(exitAfter));
|
||||
const once = get("--once");
|
||||
// A later process finds the file and skips the rule's ending.
|
||||
const ends = once === undefined || !existsSync(once);
|
||||
const linger = args.includes("--linger");
|
||||
if (args.includes("--ignore-term")) process.on("SIGTERM", () => {});
|
||||
const rules = JSON.parse(readFileSync(scriptPath, "utf8")) as Rule[];
|
||||
const used = rules.map(() => 0);
|
||||
const logPath = get("--log");
|
||||
const log = (dir: "in" | "out", line: string) => {
|
||||
if (logPath) appendFileSync(logPath, JSON.stringify({ dir, t: Date.now(), line }) + "\n");
|
||||
};
|
||||
const send = (line: string) => {
|
||||
log("out", line);
|
||||
// Synchronous so nothing is lost when a rule exits or crashes right after.
|
||||
writeSync(1, line + "\n");
|
||||
};
|
||||
|
||||
const banner = get("--banner");
|
||||
if (banner !== undefined) send(banner);
|
||||
|
||||
let pending = "";
|
||||
const startDelay = Number(get("--start-delay") ?? 0);
|
||||
let queue: Promise<void> = startDelay > 0 ? Bun.sleep(startDelay) : Promise.resolve();
|
||||
|
||||
const handle = async (line: string) => {
|
||||
log("in", line);
|
||||
let message: Message;
|
||||
try {
|
||||
message = JSON.parse(line) as Message;
|
||||
} catch {
|
||||
send(JSON.stringify({ jsonrpc: "2.0", id: null, error: { code: -32700, message: "Parse error" } }));
|
||||
return;
|
||||
}
|
||||
const isRequest = message.id !== undefined && message.id !== null && typeof message.method === "string";
|
||||
const params = message.params as { _meta?: Record<string, unknown>; cursor?: string; name?: string } | undefined;
|
||||
const version = params?._meta?.["io.modelcontextprotocol/protocolVersion"] ?? null;
|
||||
const cursor = params?.cursor ?? null;
|
||||
const index = rules.findIndex(
|
||||
(r, i) =>
|
||||
(r.method === "*" || r.method === message.method) &&
|
||||
(r.version === undefined || r.version === version) &&
|
||||
(r.cursor === undefined || r.cursor === cursor) &&
|
||||
(r.tool === undefined || r.tool === params?.name) &&
|
||||
(r.times === undefined || used[i] < r.times),
|
||||
);
|
||||
if (index < 0) {
|
||||
if (isRequest) send(JSON.stringify({ jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "Method not found" } }));
|
||||
return;
|
||||
}
|
||||
used[index] += 1;
|
||||
const rule = rules[index];
|
||||
if (rule.delayMs) await Bun.sleep(rule.delayMs);
|
||||
for (let n = 0; n < (rule.stderrRepeat ?? 1); n++) {
|
||||
for (const line of rule.stderr ?? []) writeSync(2, line + "\n");
|
||||
}
|
||||
if (rule.rawBytes) send("x".repeat(rule.rawBytes));
|
||||
for (const extra of rule.before ?? []) send(JSON.stringify(withId(extra, message.id)));
|
||||
if (rule.raw !== undefined) send(rule.raw);
|
||||
else if (rule.error) send(JSON.stringify({ jsonrpc: "2.0", id: message.id ?? null, error: rule.error }));
|
||||
else if (!rule.noReply && (isRequest || rule.result !== undefined)) {
|
||||
send(JSON.stringify({ jsonrpc: "2.0", id: message.id ?? null, result: withId(rule.result ?? {}, message.id) }));
|
||||
}
|
||||
if (rule.then !== undefined && ends && once !== undefined) writeFileSync(once, "");
|
||||
if (rule.then === "exit" && ends) process.exit(0);
|
||||
if (rule.then === "crash" && ends) process.exit(1);
|
||||
};
|
||||
|
||||
process.stdin.setEncoding("utf8");
|
||||
process.stdin.on("data", (chunk: string) => {
|
||||
pending += chunk;
|
||||
let newline: number;
|
||||
while ((newline = pending.indexOf("\n")) >= 0) {
|
||||
const line = pending.slice(0, newline).replace(/\r$/, "");
|
||||
pending = pending.slice(newline + 1);
|
||||
if (line.length > 0) queue = queue.then(() => handle(line));
|
||||
}
|
||||
});
|
||||
process.stdin.on("end", () => {
|
||||
if (linger) setInterval(() => {}, 1000);
|
||||
else queue.then(() => process.exit(0));
|
||||
});
|
||||
}
|
||||
|
||||
if (import.meta.main) main();
|
||||
@@ -0,0 +1,146 @@
|
||||
[
|
||||
{
|
||||
"source": "2025-11-25_basic_lifecycle block 1",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"initialize\", \"params\": { \"protocolVersion\": \"2025-11-25\", \"capabilities\": { \"roots\": { \"listChanged\": true }, \"sampling\": {}, \"elicitation\": { \"form\": {}, \"url\": {} }, \"tasks\": { \"requests\": { \"elicitation\": { \"create\": {} }, \"sampling\": { \"createMessage\": {} } } } }, \"clientInfo\": { \"name\": \"ExampleClient\", \"title\": \"Example Client Display Name\", \"version\": \"1.0.0\", \"description\": \"An example MCP client application\", \"icons\": [ { \"src\": \"https://example.com/icon.png\", \"mimeType\": \"image/png\", \"sizes\": [\"48x48\"] } ], \"websiteUrl\": \"https://example.com\" } } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_basic_lifecycle block 2",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"result\": { \"protocolVersion\": \"2025-11-25\", \"capabilities\": { \"logging\": {}, \"prompts\": { \"listChanged\": true }, \"resources\": { \"subscribe\": true, \"listChanged\": true }, \"tools\": { \"listChanged\": true }, \"tasks\": { \"list\": {}, \"cancel\": {}, \"requests\": { \"tools\": { \"call\": {} } } } }, \"serverInfo\": { \"name\": \"ExampleServer\", \"title\": \"Example Server Display Name\", \"version\": \"1.0.0\", \"description\": \"An example MCP server providing tools and resources\", \"icons\": [ { \"src\": \"https://example.com/server-icon.svg\", \"mimeType\": \"image/svg+xml\", \"sizes\": [\"any\"] } ], \"websiteUrl\": \"https://example.com/server\" }, \"instructions\": \"Optional instructions for the client\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_basic_lifecycle block 3",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/initialized\" }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_basic_lifecycle block 4",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"error\": { \"code\": -32602, \"message\": \"Unsupported protocol version\", \"data\": { \"supported\": [\"2024-11-05\"], \"requested\": \"1.0.0\" } } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_basic_utilities_cancellation block 1",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/cancelled\", \"params\": { \"requestId\": \"123\", \"reason\": \"User requested cancellation\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_client_elicitation block 9",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"elicitation/create\", \"params\": { \"mode\": \"form\", \"message\": \"Please provide your GitHub username\", \"requestedSchema\": { \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\" } }, \"required\": [\"name\"] } } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_client_elicitation block 10",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"result\": { \"action\": \"accept\", \"content\": { \"name\": \"octocat\" } } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_client_elicitation block 11",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"elicitation/create\", \"params\": { \"mode\": \"form\", \"message\": \"Please provide your contact information\", \"requestedSchema\": { \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\", \"description\": \"Your full name\" }, \"email\": { \"type\": \"string\", \"format\": \"email\", \"description\": \"Your email address\" }, \"age\": { \"type\": \"number\", \"minimum\": 18, \"description\": \"Your age\" } }, \"required\": [\"name\", \"email\"] } } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_client_elicitation block 12",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 2, \"result\": { \"action\": \"accept\", \"content\": { \"name\": \"Monalisa Octocat\", \"email\": \"octocat@github.com\", \"age\": 30 } } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_client_elicitation block 13",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 3, \"method\": \"elicitation/create\", \"params\": { \"mode\": \"url\", \"elicitationId\": \"550e8400-e29b-41d4-a716-446655440000\", \"url\": \"https://mcp.example.com/ui/set_api_key\", \"message\": \"Please provide your API key to continue.\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_client_elicitation block 14",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 3, \"result\": { \"action\": \"accept\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2025-11-25_client_elicitation block 15",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/elicitation/complete\", \"params\": { \"elicitationId\": \"550e8400-e29b-41d4-a716-446655440000\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_index block 3",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/call\", \"params\": { \"name\": \"get_weather\", \"arguments\": { \"location\": \"New York\" }, \"_meta\": { \"traceparent\": \"00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01\" } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_patterns_cancellation block 1",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/cancelled\", \"params\": { \"requestId\": \"123\", \"reason\": \"User requested cancellation\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_patterns_progress block 1",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"some_method\", \"params\": { \"_meta\": { \"progressToken\": \"abc123\" } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_patterns_progress block 2",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/progress\", \"params\": { \"progressToken\": \"abc123\", \"progress\": 50, \"total\": 100, \"message\": \"Reticulating splines...\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_patterns_subscriptions block 1",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"subscriptions/listen\", \"params\": { \"_meta\": { \"io.modelcontextprotocol/protocolVersion\": \"2026-07-28\", \"io.modelcontextprotocol/clientInfo\": { \"name\": \"ExampleClient\", \"version\": \"1.0.0\" }, \"io.modelcontextprotocol/clientCapabilities\": {} }, \"notifications\": { \"toolsListChanged\": true, \"resourceSubscriptions\": [\"file:///project/config.json\"] } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_patterns_subscriptions block 2",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/subscriptions/acknowledged\", \"params\": { \"_meta\": { \"io.modelcontextprotocol/subscriptionId\": 1 }, \"notifications\": { \"toolsListChanged\": true, \"resourceSubscriptions\": [\"file:///project/config.json\"] } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_patterns_subscriptions block 3",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/resources/updated\", \"params\": { \"_meta\": { \"io.modelcontextprotocol/subscriptionId\": 1 }, \"uri\": \"file:///project/config.json\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_patterns_subscriptions block 4",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"result\": { \"resultType\": \"complete\", \"_meta\": { \"io.modelcontextprotocol/subscriptionId\": 1 } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_transports_streamable-http block 2",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"error\": { \"code\": -32020, \"message\": \"Header mismatch: Mcp-Name header value 'foo' does not match body value 'bar'\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_basic_versioning block 1",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"error\": { \"code\": -32022, \"message\": \"Unsupported protocol version\", \"data\": { \"supported\": [\"2026-07-28\", \"2025-11-25\"], \"requested\": \"1900-01-01\" } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_discover block 1",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": \"discover-1\", \"method\": \"server/discover\", \"params\": { \"_meta\": { \"io.modelcontextprotocol/protocolVersion\": \"2026-07-28\", \"io.modelcontextprotocol/clientInfo\": { \"name\": \"ExampleClient\", \"version\": \"1.0.0\" }, \"io.modelcontextprotocol/clientCapabilities\": {} } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_discover block 2",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": \"discover-1\", \"result\": { \"resultType\": \"complete\", \"supportedVersions\": [\"2026-07-28\"], \"capabilities\": { \"tools\": {}, \"resources\": {} }, \"_meta\": { \"io.modelcontextprotocol/serverInfo\": { \"name\": \"ExampleServer\", \"version\": \"1.0.0\" } }, \"instructions\": \"This server provides weather and resource utilities.\", \"ttlMs\": 3600000, \"cacheScope\": \"public\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 2",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"tools/list\", \"params\": { \"cursor\": \"optional-cursor-value\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 3",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 1, \"result\": { \"resultType\": \"complete\", \"tools\": [ { \"name\": \"get_weather\", \"title\": \"Weather Information Provider\", \"description\": \"Get current weather information for a location\", \"inputSchema\": { \"type\": \"object\", \"properties\": { \"location\": { \"type\": \"string\", \"description\": \"City name or zip code\" } }, \"required\": [\"location\"] }, \"icons\": [ { \"src\": \"https://example.com/weather-icon.png\", \"mimeType\": \"image/png\", \"sizes\": [\"48x48\"] } ] } ], \"nextCursor\": \"next-page-cursor\", \"ttlMs\": 300000, \"cacheScope\": \"public\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 4",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/call\", \"params\": { \"name\": \"get_weather\", \"arguments\": { \"location\": \"New York\" } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 5",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 2, \"result\": { \"resultType\": \"complete\", \"content\": [ { \"type\": \"text\", \"text\": \"Current weather in New York:\\nTemperature: 72°F\\nConditions: Partly cloudy\" } ], \"isError\": false } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 6",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 2, \"result\": { \"resultType\": \"input_required\", \"inputRequests\": { \"github_login\": { \"method\": \"elicitation/create\", \"params\": { \"mode\": \"form\", \"message\": \"Please provide your GitHub username\", \"requestedSchema\": { \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\" } }, \"required\": [\"name\"] } } } }, \"requestState\": \"eyJsb2NhdGlvbiI6Ik5ldyBZb3JrIn0...\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 7",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 3, \"method\": \"tools/call\", \"params\": { \"name\": \"get_weather\", \"arguments\": { \"location\": \"New York\" }, \"inputResponses\": { \"github_login\": { \"action\": \"accept\", \"content\": { \"name\": \"octocat\" } } }, \"requestState\": \"eyJsb2NhdGlvbiI6Ik5ldyBZb3JrIn0...\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 8",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"method\": \"notifications/tools/list_changed\" }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 16",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 5, \"result\": { \"resultType\": \"complete\", \"content\": [ { \"type\": \"text\", \"text\": \"{\\\"temperature\\\": 22.5, \\\"conditions\\\": \\\"Partly cloudy\\\", \\\"humidity\\\": 65}\" } ], \"structuredContent\": { \"temperature\": 22.5, \"conditions\": \"Partly cloudy\", \"humidity\": 65 } } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 18",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 6, \"result\": { \"resultType\": \"complete\", \"content\": [ { \"type\": \"text\", \"text\": \"Found 2 users: Alice (alice@example.com) and Bob (bob@example.com).\" } ], \"structuredContent\": [ { \"id\": \"1\", \"name\": \"Alice\", \"email\": \"alice@example.com\" }, { \"id\": \"2\", \"name\": \"Bob\", \"email\": \"bob@example.com\" } ] } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 22",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 3, \"error\": { \"code\": -32602, \"message\": \"Unknown tool: invalid_tool_name\" } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_tools block 23",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": 4, \"result\": { \"resultType\": \"complete\", \"content\": [ { \"type\": \"text\", \"text\": \"Invalid departure date: must be in the future. Current date is 08/08/2025.\" } ], \"isError\": true } }"
|
||||
},
|
||||
{
|
||||
"source": "2026-07-28_server_utilities_pagination block 2",
|
||||
"input": "{ \"jsonrpc\": \"2.0\", \"id\": \"124\", \"method\": \"resources/list\", \"params\": { \"cursor\": \"eyJwYWdlIjogMn0=\" } }"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,133 @@
|
||||
// Shared builder for scenarios that run tests/tla/drivers/mcp/labs/stdio_lab.zig against the
|
||||
// scripted fake stdio server (U02 transport-level and U03 to U05 scenarios).
|
||||
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { Rule } from "./fakes/stdio";
|
||||
import type { Scenario, ScenarioContext, TraceExpectation } from "./scenario";
|
||||
|
||||
const FAKE = join(import.meta.dir, "fakes/stdio.ts");
|
||||
export const PING = '{"jsonrpc":"2.0","id":1,"method":"ping"}';
|
||||
export const PING2 = '{"jsonrpc":"2.0","id":2,"method":"ping"}';
|
||||
|
||||
export interface Seen {
|
||||
did?: { action?: string; state?: string; error?: string; expect?: string; met?: boolean; era_error?: string };
|
||||
era?: { ready?: string; failed?: string };
|
||||
tools?: { event: string; fresh?: boolean; count?: number; names?: string[]; invalid?: number; duplicate?: number };
|
||||
call?: { id: number; kind: string; is_error?: boolean; content?: string; structured?: string | null; code?: number };
|
||||
seen?: { message?: string; stderr?: string; dropped?: string; state?: string; lost?: boolean };
|
||||
sub?: { sub: number; deliver?: string; acknowledged?: number; ended?: string };
|
||||
outcome?: { id: number; outcome: string };
|
||||
final?: {
|
||||
state: string;
|
||||
generation: number;
|
||||
attempts: number;
|
||||
term_sent: boolean;
|
||||
kill_sent: boolean;
|
||||
errored: boolean;
|
||||
stderr_dropped: number;
|
||||
spawn_error: string | null;
|
||||
in_flight?: number;
|
||||
era?: string;
|
||||
era_version?: string;
|
||||
era_failure?: string | null;
|
||||
catalog?: string;
|
||||
tools?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface LabCase {
|
||||
name: string;
|
||||
unit: string;
|
||||
description: string;
|
||||
rules?: Rule[];
|
||||
/** Extra fake-server flags, or a full replacement command when `server` is set. */
|
||||
fakeFlags?: string[];
|
||||
server?: string[];
|
||||
labFlags?: string[];
|
||||
script: string[];
|
||||
/** The model's failure limit; the lab's default is 8. */
|
||||
maxAttempts?: number;
|
||||
verdict?: "ACCEPTED" | "REJECTED";
|
||||
/** Also check the request machine's trace (U02 over stdio). */
|
||||
requestTrace?: boolean;
|
||||
/** Also check the era machine's trace (U04): how many preferred versions the lab has. */
|
||||
era?: { preferred: number; verdict?: "ACCEPTED" | "REJECTED" };
|
||||
/** Also check the catalog machine's trace (U05): the lab's page limit. */
|
||||
catalog?: { maxPages: number; verdict?: "ACCEPTED" | "REJECTED" };
|
||||
/** Also check the subscription machine's trace (U10). */
|
||||
subscription?: { maxTries?: number; verdict?: "ACCEPTED" | "REJECTED" };
|
||||
/** Checks the lab's output and the fake server's log of every line it saw. */
|
||||
check?: (lines: Seen[], final: NonNullable<Seen["final"]>, serverLog: { dir: string; line: string }[]) => string | null;
|
||||
timeoutMs?: number;
|
||||
real?: boolean;
|
||||
}
|
||||
|
||||
export function labScenario(c: LabCase): Scenario {
|
||||
let caseDir = "";
|
||||
const traces: TraceExpectation[] = [
|
||||
{ machine: "stdio", instance: "lab", specDir: "tests/tla/mcp", wrapper: "StdioConnTrace", verdict: c.verdict ?? "ACCEPTED", constants: { ObsMaxAttempts: c.maxAttempts ?? 8 } },
|
||||
];
|
||||
if (c.requestTrace) traces.push({ machine: "request", instance: "lab", specDir: "tests/tla/mcp", wrapper: "RequestTrace", verdict: "ACCEPTED" });
|
||||
if (c.era) {
|
||||
traces.push({ machine: "era", instance: "lab", specDir: "tests/tla/mcp", wrapper: "EraTrace", verdict: c.era.verdict ?? "ACCEPTED", constants: { ObsPreferredCount: c.era.preferred } });
|
||||
}
|
||||
if (c.catalog) {
|
||||
traces.push({ machine: "catalog", instance: "lab", specDir: "tests/tla/mcp", wrapper: "CatalogTrace", verdict: c.catalog.verdict ?? "ACCEPTED", constants: { ObsMaxPages: c.catalog.maxPages } });
|
||||
}
|
||||
if (c.subscription) {
|
||||
traces.push({ machine: "subscription", instance: "lab", specDir: "tests/tla/mcp", wrapper: "SubscriptionTrace", verdict: c.subscription.verdict ?? "ACCEPTED", constants: { ObsMaxTries: c.subscription.maxTries ?? 8 } });
|
||||
}
|
||||
return {
|
||||
name: c.name,
|
||||
unit: c.unit,
|
||||
description: c.description,
|
||||
needs: ["stdio_lab"],
|
||||
setup: (ctx: ScenarioContext) => {
|
||||
caseDir = ctx.caseDir;
|
||||
writeFileSync(join(ctx.caseDir, "rules.json"), JSON.stringify(c.rules ?? []));
|
||||
writeFileSync(join(ctx.caseDir, "script.txt"), c.script.join("\n") + "\n");
|
||||
},
|
||||
command: (ctx: ScenarioContext) => [
|
||||
ctx.bin.stdio_lab,
|
||||
ctx.tracePath,
|
||||
join(ctx.caseDir, "script.txt"),
|
||||
...(c.labFlags ?? []),
|
||||
"--",
|
||||
...(c.server ?? [process.execPath, FAKE, "--script", join(ctx.caseDir, "rules.json"), "--log", join(ctx.caseDir, "server.jsonl"), ...(c.fakeFlags ?? [])]),
|
||||
],
|
||||
expect: {
|
||||
exit: 0,
|
||||
traces,
|
||||
stdout: (text) => {
|
||||
const lines = text.split("\n").filter(Boolean).map((l) => JSON.parse(l) as Seen);
|
||||
const unmet = lines.find((l) => l.did?.met === false || l.did?.error !== undefined);
|
||||
if (unmet) return `script step failed: ${JSON.stringify(unmet.did)}`;
|
||||
const final = lines.at(-1)?.final;
|
||||
if (!final) return "no final line";
|
||||
if (!c.check) return null;
|
||||
let serverLog: { dir: string; line: string }[] = [];
|
||||
try {
|
||||
serverLog = readFileSync(join(caseDir, "server.jsonl"), "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
||||
} catch {
|
||||
// No log: the server never started.
|
||||
}
|
||||
return c.check(lines, final, serverLog);
|
||||
},
|
||||
},
|
||||
timeoutMs: c.timeoutMs,
|
||||
real: c.real,
|
||||
};
|
||||
}
|
||||
|
||||
export const messages = (lines: Seen[]) => lines.filter((l) => l.seen?.message !== undefined).map((l) => l.seen!.message!);
|
||||
export const expectFinal = (final: NonNullable<Seen["final"]>, want: Partial<NonNullable<Seen["final"]>>): string | null => {
|
||||
for (const [key, value] of Object.entries(want)) {
|
||||
const got = (final as unknown as Record<string, unknown>)[key];
|
||||
if (got !== value) return `final ${key} is ${JSON.stringify(got)}, expected ${JSON.stringify(value)}`;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
export const outcomes = (lines: Seen[]) =>
|
||||
Object.fromEntries(lines.filter((l) => l.outcome).map((l) => [l.outcome!.id, l.outcome!.outcome]));
|
||||
@@ -0,0 +1,379 @@
|
||||
//! Written for the next MCP-v2 version: it does not build against this
|
||||
//! checkout's src/mcp/ until that version lands, so the driver runs its
|
||||
//! scenarios only with --next.
|
||||
//!
|
||||
//! U12 lab: runs a script of host actions against `mcp.engine`, playing the
|
||||
//! host: its permission answers, its stderr sink, and what it does with each
|
||||
//! event. One run traces every server's machines into one file, the engine
|
||||
//! gate's under the server's name and each server's under "<name>.". Dev
|
||||
//! only; never moves into fx.
|
||||
//!
|
||||
//! Usage: engine_lab <trace-path> <script> [--deny TOOL]... [--capabilities JSON]
|
||||
//! [--max-attempts N] [--no-listen] [--preferred V1,V2] [--probe-ms N]
|
||||
//! [--define NAME=VALUE]... (each {NAME} in the script becomes VALUE)
|
||||
//!
|
||||
//! Script, one action per line (blank lines and lines starting with # are skipped):
|
||||
//! server <name> stdio <argv-json> declare a server; before the first action
|
||||
//! server <name> http <url> they are the engine's, after it `reload` uses them
|
||||
//! tools <name> | disconnect <name> | reconnect <name>
|
||||
//! call <name> <tool> [arguments-json] calls are numbered from 1 in this order
|
||||
//! cancel <k> | respond <k> <values-json-array>
|
||||
//! elicit <name> <slot> accept|decline|cancel [content-json]
|
||||
//! reload <name>... the declared servers to keep or add, in order
|
||||
//! status
|
||||
//! expect <tag-prefix> [timeout_ms] an event whose tag starts so, seen and not yet
|
||||
//! expected, or the next one (default 10000)
|
||||
//! drain <ms> handle whatever arrives for that long; what was
|
||||
//! seen is no longer expected
|
||||
//!
|
||||
//! Event tags: <server>.phase.<phase>, <server>.called.<k>.<outcome>,
|
||||
//! <server>.input.<k>, <server>.<server event>[.ok|.fail], for example
|
||||
//! "a.tools.ok" or "b.elicit". Output lines: {"did":...}, {"allow":...},
|
||||
//! {"event":...}, {"status":[...]}, and one {"final":...} with the stderr
|
||||
//! lines each server's sink took. Exit: 0 the script ran, 1 an expectation
|
||||
//! wasn't met, 2 usage.
|
||||
|
||||
const std = @import("std");
|
||||
const mcp = @import("mcp");
|
||||
const engine = mcp.engine;
|
||||
|
||||
/// Trace code is compiled in.
|
||||
pub const mcpv2_trace = true;
|
||||
|
||||
const max_servers = 8;
|
||||
|
||||
const Declared = struct {
|
||||
name: []const u8,
|
||||
transport: engine.Transport,
|
||||
/// Lines the stderr sink took, from reader tasks.
|
||||
logged: std.atomic.Value(u32) = .init(0),
|
||||
};
|
||||
|
||||
const Lab = struct {
|
||||
gpa: std.mem.Allocator,
|
||||
arena: std.mem.Allocator,
|
||||
out: *std.Io.Writer,
|
||||
deny: []const []const u8,
|
||||
declared: [max_servers]Declared = undefined,
|
||||
count: usize = 0,
|
||||
e: ?*engine.Engine = null,
|
||||
/// Engine handles by call number.
|
||||
calls: std.ArrayList(engine.Handle) = .empty,
|
||||
/// The 2025 elicitations shown, in order; scripts name them from 1.
|
||||
elicits: std.ArrayList(mcp.core.elicitation.Id) = .empty,
|
||||
/// Tags of events no `expect` has claimed yet, so a script needn't
|
||||
/// know in which order two servers answer.
|
||||
unclaimed: std.ArrayList([]const u8) = .empty,
|
||||
|
||||
fn print(lab: *Lab, comptime kind: []const u8, value: anytype) !void {
|
||||
var json: std.json.Stringify = .{ .writer = lab.out };
|
||||
try json.beginObject();
|
||||
try json.objectField(kind);
|
||||
try json.write(value);
|
||||
try json.endObject();
|
||||
try lab.out.writeByte('\n');
|
||||
try lab.out.flush();
|
||||
}
|
||||
|
||||
fn allow(context: *anyopaque, server: []const u8, tool: []const u8, arguments: ?[]const u8) bool {
|
||||
const lab: *Lab = @ptrCast(@alignCast(context));
|
||||
_ = arguments;
|
||||
var ok = true;
|
||||
for (lab.deny) |d| ok = ok and !std.mem.eql(u8, d, tool);
|
||||
lab.print("allow", .{ .server = server, .tool = tool, .ok = ok }) catch {};
|
||||
return ok;
|
||||
}
|
||||
|
||||
fn log(context: *anyopaque, server: []const u8, line: []const u8) void {
|
||||
const lab: *Lab = @ptrCast(@alignCast(context));
|
||||
_ = line;
|
||||
for (lab.declared[0..lab.count]) |*d| if (std.mem.eql(u8, d.name, server)) {
|
||||
_ = d.logged.fetchAdd(1, .monotonic);
|
||||
};
|
||||
}
|
||||
|
||||
fn configs(lab: *Lab, names: []const []const u8) ![]engine.Config {
|
||||
const list = try lab.arena.alloc(engine.Config, names.len);
|
||||
for (names, list) |name, *c| {
|
||||
const d = lab.find(name) orelse return error.UnknownServer;
|
||||
c.* = .{ .name = d.name, .transport = d.transport };
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
fn find(lab: *Lab, name: []const u8) ?*Declared {
|
||||
var i = lab.count;
|
||||
while (i > 0) {
|
||||
i -= 1;
|
||||
if (std.mem.eql(u8, lab.declared[i].name, name)) return &lab.declared[i];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn number(lab: *const Lab, handle: engine.Handle) usize {
|
||||
for (lab.calls.items, 1..) |h, k| if (h == handle) return k;
|
||||
return 0;
|
||||
}
|
||||
|
||||
fn tag(lab: *Lab, event: engine.Event) ![]const u8 {
|
||||
const s = event.server;
|
||||
return switch (event.kind) {
|
||||
.phase => |p| lab.arena.print("{s}.phase.{s}", .{ s, @tagName(p) }),
|
||||
.called => |c| lab.arena.print("{s}.called.{d}.{s}", .{ s, lab.number(c.call), @tagName(c.outcome) }),
|
||||
.input => |q| lab.arena.print("{s}.input.{d}", .{ s, lab.number(q.call) }),
|
||||
.server => |v| switch (v) {
|
||||
.tools => |t| lab.arena.print("{s}.tools.{s}", .{ s, if (t.ok) "ok" else "fail" }),
|
||||
else => lab.arena.print("{s}.{s}", .{ s, @tagName(v) }),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/// Prints one event, keeps its tag, and returns it.
|
||||
fn show(lab: *Lab, event: engine.Event) ![]const u8 {
|
||||
const t = try lab.tag(event);
|
||||
try lab.unclaimed.append(lab.arena, t);
|
||||
const e = lab.e.?;
|
||||
switch (event.kind) {
|
||||
.called => |c| {
|
||||
const answer: ?[]const u8 = if (c.answer) |a| switch (a) {
|
||||
.result => |r| r.content,
|
||||
.failure => |f| try lab.arena.print("error {d}", .{f.code}),
|
||||
else => @tagName(a),
|
||||
} else null;
|
||||
try lab.print("event", .{ .tag = t, .maybe_ran = c.maybe_ran, .timed_out = c.timed_out, .answer = answer });
|
||||
},
|
||||
.input => |q| try lab.print("event", .{ .tag = t, .requests = q.requests.len }),
|
||||
.server => |v| switch (v) {
|
||||
.tools => {
|
||||
const list = try e.toolList(event.server);
|
||||
const names = try lab.arena.alloc([]const u8, list.len());
|
||||
for (names, 0..) |*n, i| n.* = list.get(i).name;
|
||||
try lab.print("event", .{ .tag = t, .names = names });
|
||||
},
|
||||
.elicit => |x| {
|
||||
try lab.elicits.append(lab.gpa, x.id);
|
||||
try lab.print("event", .{ .tag = t, .k = lab.elicits.items.len, .required = x.required });
|
||||
},
|
||||
.failed => |f| try lab.print("event", .{ .tag = t, .why = switch (f) {
|
||||
.era => |why| @tagName(why),
|
||||
else => @tagName(f),
|
||||
} }),
|
||||
else => try lab.print("event", .{ .tag = t }),
|
||||
},
|
||||
.phase => try lab.print("event", .{ .tag = t }),
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
fn claim(lab: *Lab, want: []const u8) bool {
|
||||
for (lab.unclaimed.items, 0..) |t, i| if (std.mem.startsWith(u8, t, want)) {
|
||||
_ = lab.unclaimed.orderedRemove(i);
|
||||
return true;
|
||||
};
|
||||
return false;
|
||||
}
|
||||
|
||||
fn status(lab: *Lab) !void {
|
||||
const e = lab.e.?;
|
||||
const list = try lab.arena.alloc(engine.Status, e.len());
|
||||
for (list, 0..) |*s, i| s.* = e.status(i);
|
||||
try lab.print("status", list);
|
||||
}
|
||||
|
||||
fn run(lab: *Lab, line: []const u8) !bool {
|
||||
var words = std.mem.tokenizeScalar(u8, line, ' ');
|
||||
const action = words.next() orelse return true;
|
||||
if (std.mem.eql(u8, action, "server")) {
|
||||
const name = try lab.arena.dupe(u8, words.next() orelse return error.Usage);
|
||||
const kind = words.next() orelse return error.Usage;
|
||||
const rest = std.mem.trim(u8, words.rest(), " ");
|
||||
const transport: engine.Transport = if (std.mem.eql(u8, kind, "http"))
|
||||
.{ .http = .{ .url = try lab.arena.dupe(u8, rest) } }
|
||||
else
|
||||
.{ .stdio = .{ .argv = try std.json.parseFromSliceLeaky([]const []const u8, lab.arena, rest, .{}) } };
|
||||
if (lab.count == max_servers) return error.Usage;
|
||||
lab.declared[lab.count] = .{ .name = name, .transport = transport };
|
||||
lab.count += 1;
|
||||
try lab.print("did", .{ .action = "server", .name = name });
|
||||
return true;
|
||||
}
|
||||
const e = lab.e.?;
|
||||
if (std.mem.eql(u8, action, "tools")) {
|
||||
try e.needTools(words.next() orelse return error.Usage);
|
||||
} else if (std.mem.eql(u8, action, "disconnect")) {
|
||||
try e.disconnect(words.next() orelse return error.Usage);
|
||||
} else if (std.mem.eql(u8, action, "reconnect")) {
|
||||
try e.reconnect(words.next() orelse return error.Usage);
|
||||
} else if (std.mem.eql(u8, action, "call")) {
|
||||
const server = words.next() orelse return error.Usage;
|
||||
const tool = words.next() orelse return error.Usage;
|
||||
const args = std.mem.trim(u8, words.rest(), " ");
|
||||
const handle = try e.call(server, tool, if (args.len > 0) args else null);
|
||||
try lab.calls.append(lab.gpa, handle);
|
||||
try lab.print("did", .{ .action = "call", .k = lab.calls.items.len });
|
||||
return true;
|
||||
} else if (std.mem.eql(u8, action, "cancel")) {
|
||||
const k = try std.fmt.parseInt(usize, words.next() orelse return error.Usage, 10);
|
||||
try e.cancel(lab.calls.items[k - 1]);
|
||||
} else if (std.mem.eql(u8, action, "respond")) {
|
||||
const k = try std.fmt.parseInt(usize, words.next() orelse return error.Usage, 10);
|
||||
const parsed = try std.json.parseFromSliceLeaky(std.json.Value, lab.arena, words.rest(), .{});
|
||||
const values = try lab.arena.alloc([]const u8, parsed.array.items.len);
|
||||
for (parsed.array.items, values) |v, *out| out.* = try std.json.Stringify.valueAlloc(lab.arena, v, .{});
|
||||
try e.respond(lab.calls.items[k - 1], values);
|
||||
} else if (std.mem.eql(u8, action, "elicit")) {
|
||||
const server = words.next() orelse return error.Usage;
|
||||
const k = try std.fmt.parseInt(usize, words.next() orelse return error.Usage, 10);
|
||||
if (k == 0 or k > lab.elicits.items.len) return error.Usage;
|
||||
const verb = words.next() orelse return error.Usage;
|
||||
const content = std.mem.trim(u8, words.rest(), " ");
|
||||
const act = std.meta.stringToEnum(mcp.core.elicitation.Action, verb) orelse return error.Usage;
|
||||
e.answerElicitation(server, lab.elicits.items[k - 1], .{ .action = act, .content = if (content.len > 0) content else null }) catch |err| {
|
||||
try lab.print("did", .{ .action = "elicit", .@"error" = @errorName(err) });
|
||||
return true;
|
||||
};
|
||||
} else if (std.mem.eql(u8, action, "reload")) {
|
||||
var names: std.ArrayList([]const u8) = .empty;
|
||||
while (words.next()) |name| try names.append(lab.arena, name);
|
||||
try e.reload(try lab.configs(names.items));
|
||||
} else if (std.mem.eql(u8, action, "status")) {
|
||||
try lab.status();
|
||||
return true;
|
||||
} else if (std.mem.eql(u8, action, "expect")) {
|
||||
const want = words.next() orelse return error.Usage;
|
||||
const ms = if (words.next()) |t| try std.fmt.parseInt(u32, t, 10) else 10_000;
|
||||
const deadline = now(e) + ms;
|
||||
var claimed = lab.claim(want);
|
||||
while (!claimed and now(e) < deadline) {
|
||||
const event = try e.next(@intCast(@max(0, deadline - now(e)))) orelse break;
|
||||
_ = try lab.show(event);
|
||||
claimed = lab.claim(want);
|
||||
}
|
||||
if (claimed) {
|
||||
try lab.print("did", .{ .action = "expect", .expect = want, .met = true });
|
||||
return true;
|
||||
}
|
||||
try lab.print("did", .{ .action = "expect", .expect = want, .met = false });
|
||||
return false;
|
||||
} else if (std.mem.eql(u8, action, "drain")) {
|
||||
const ms = try std.fmt.parseInt(u32, words.next() orelse return error.Usage, 10);
|
||||
const deadline = now(e) + ms;
|
||||
while (now(e) < deadline) {
|
||||
const event = try e.next(@intCast(@max(0, deadline - now(e)))) orelse break;
|
||||
_ = try lab.show(event);
|
||||
}
|
||||
lab.unclaimed.clearRetainingCapacity();
|
||||
} else return error.Usage;
|
||||
try lab.print("did", .{ .action = action });
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
fn now(e: *const engine.Engine) i64 {
|
||||
return std.Io.Clock.awake.now(e.io).toMilliseconds();
|
||||
}
|
||||
|
||||
pub fn main(init: std.process.Init) !void {
|
||||
const io = init.io;
|
||||
const gpa = init.gpa;
|
||||
var arena_state: std.heap.ArenaAllocator = .init(gpa);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
const args = try init.minimal.args.toSlice(arena);
|
||||
|
||||
var stdout_buffer: [64 * 1024]u8 = undefined;
|
||||
var stdout = std.Io.File.stdout().writer(io, &stdout_buffer);
|
||||
const out = &stdout.interface;
|
||||
|
||||
if (args.len < 3) {
|
||||
std.debug.print("usage: engine_lab <trace-path> <script> [options]\n", .{});
|
||||
std.process.exit(2);
|
||||
}
|
||||
var deny: std.ArrayList([]const u8) = .empty;
|
||||
var options: engine.Options = .{ .client = .{ .info = .{ .name = "mcpv2-engine-lab", .version = "0.0.0" } } };
|
||||
var preferred: ?[]const []const u8 = null;
|
||||
var defines: std.ArrayList([2][]const u8) = .empty;
|
||||
var i: usize = 3;
|
||||
while (i < args.len) : (i += 1) {
|
||||
const flag = args[i];
|
||||
const value = if (i + 1 < args.len) args[i + 1] else "";
|
||||
if (std.mem.eql(u8, flag, "--no-listen")) {
|
||||
options.listen_tools = false;
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
if (std.mem.eql(u8, flag, "--deny")) {
|
||||
try deny.append(arena, value);
|
||||
} else if (std.mem.eql(u8, flag, "--capabilities")) {
|
||||
options.client.capabilities = value;
|
||||
} else if (std.mem.eql(u8, flag, "--max-attempts")) {
|
||||
options.gate.max_attempts = try std.fmt.parseInt(u8, value, 10);
|
||||
} else if (std.mem.eql(u8, flag, "--stable-ms")) {
|
||||
options.gate.stable_ms = try std.fmt.parseInt(u32, value, 10);
|
||||
} else if (std.mem.eql(u8, flag, "--define")) {
|
||||
const eq = std.mem.findScalar(u8, value, '=') orelse {
|
||||
std.debug.print("engine_lab: --define needs NAME=VALUE\n", .{});
|
||||
std.process.exit(2);
|
||||
};
|
||||
try defines.append(arena, .{ try arena.print("{{{s}}}", .{value[0..eq]}), value[eq + 1 ..] });
|
||||
} else if (std.mem.eql(u8, flag, "--probe-ms")) {
|
||||
options.era.probe_timeout_ms = try std.fmt.parseInt(u32, value, 10);
|
||||
} else if (std.mem.eql(u8, flag, "--timeout-ms")) {
|
||||
options.timeouts.idle_ms = try std.fmt.parseInt(u32, value, 10);
|
||||
} else if (std.mem.eql(u8, flag, "--preferred")) {
|
||||
var list: std.ArrayList([]const u8) = .empty;
|
||||
var it = std.mem.splitScalar(u8, value, ',');
|
||||
while (it.next()) |v| try list.append(arena, v);
|
||||
preferred = list.items;
|
||||
} else {
|
||||
std.debug.print("engine_lab: unknown option {s}\n", .{flag});
|
||||
std.process.exit(2);
|
||||
}
|
||||
}
|
||||
if (preferred) |p| options.era.preferred = p;
|
||||
|
||||
const trace_file = try std.Io.Dir.cwd().createFile(io, args[1], .{});
|
||||
defer trace_file.close(io);
|
||||
var trace_buffer: [16 * 1024]u8 = undefined;
|
||||
var trace_writer = trace_file.writer(io, &trace_buffer);
|
||||
var tracer: mcp.trace.Writer = .init(&trace_writer.interface);
|
||||
options.trace = &tracer;
|
||||
|
||||
var script = try std.Io.Dir.cwd().readFileAlloc(io, args[2], arena, .limited(1 << 20));
|
||||
for (defines.items) |d| script = try std.mem.replaceOwned(u8, arena, script, d[0], d[1]);
|
||||
var lab: Lab = .{ .gpa = gpa, .arena = arena, .out = out, .deny = deny.items };
|
||||
var e: engine.Engine = undefined;
|
||||
var ok = true;
|
||||
var lines = std.mem.splitScalar(u8, script, '\n');
|
||||
while (lines.next()) |raw| {
|
||||
const line = std.mem.trim(u8, raw, " \r\t");
|
||||
if (line.len == 0 or line[0] == '#') continue;
|
||||
if (lab.e == null and !std.mem.startsWith(u8, line, "server ")) {
|
||||
var names: std.ArrayList([]const u8) = .empty;
|
||||
for (lab.declared[0..lab.count]) |d| try names.append(arena, d.name);
|
||||
try e.init(io, gpa, .{ .context = &lab, .allow = Lab.allow, .log = Lab.log }, options, try lab.configs(names.items));
|
||||
lab.e = &e;
|
||||
}
|
||||
const met = lab.run(line) catch |err| blk: {
|
||||
try lab.print("did", .{ .line = line, .@"error" = @errorName(err) });
|
||||
break :blk false;
|
||||
};
|
||||
trace_writer.interface.flush() catch {};
|
||||
if (!met) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lab.e != null) {
|
||||
try lab.status();
|
||||
e.deinit();
|
||||
}
|
||||
trace_writer.interface.flush() catch {};
|
||||
const Logged = struct { name: []const u8, lines: u32 };
|
||||
const logged = try arena.alloc(Logged, lab.count);
|
||||
for (lab.declared[0..lab.count], logged) |*d, *l| l.* = .{ .name = d.name, .lines = d.logged.load(.monotonic) };
|
||||
try lab.print("final", .{ .ok = ok, .logged = logged });
|
||||
lab.calls.deinit(gpa);
|
||||
lab.elicits.deinit(gpa);
|
||||
std.process.exit(if (ok) 0 else 1);
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//! U02 lab: drives the real request core with a seeded random event stream and
|
||||
//! writes its trace, so the Bun driver can have TLC check the core against
|
||||
//! spec/Request.tla before any transport exists. Dev only; never moves into fx.
|
||||
//!
|
||||
//! Usage: request_lab <trace-path> [--seed N] [--steps N] [--slots N]
|
||||
//! [--plant-bug late-delivery]
|
||||
//!
|
||||
//! `--plant-bug late-delivery` is a known-bad control: when a late response
|
||||
//! arrives, the lab reports the core as having delivered it a second time.
|
||||
|
||||
const std = @import("std");
|
||||
const mcp = @import("mcp");
|
||||
|
||||
/// Trace code is compiled in.
|
||||
pub const mcpv2_trace = true;
|
||||
const request = mcp.core.request;
|
||||
|
||||
const Options = struct {
|
||||
trace_path: []const u8,
|
||||
seed: u64 = 1,
|
||||
steps: u32 = 200,
|
||||
slots: usize = 4,
|
||||
plant_late_delivery: bool = false,
|
||||
};
|
||||
|
||||
fn parseOptions(args: []const [:0]const u8) !Options {
|
||||
if (args.len < 2) return error.InvalidArguments;
|
||||
var options: Options = .{ .trace_path = args[1] };
|
||||
var index: usize = 2;
|
||||
while (index < args.len) : (index += 1) {
|
||||
const arg = args[index];
|
||||
const value = if (index + 1 < args.len) args[index + 1] else "";
|
||||
if (std.mem.eql(u8, arg, "--seed")) {
|
||||
options.seed = try std.fmt.parseInt(u64, value, 10);
|
||||
} else if (std.mem.eql(u8, arg, "--steps")) {
|
||||
options.steps = try std.fmt.parseInt(u32, value, 10);
|
||||
} else if (std.mem.eql(u8, arg, "--slots")) {
|
||||
options.slots = try std.fmt.parseInt(usize, value, 10);
|
||||
} else if (std.mem.eql(u8, arg, "--plant-bug") and std.mem.eql(u8, value, "late-delivery")) {
|
||||
options.plant_late_delivery = true;
|
||||
} else return error.InvalidArguments;
|
||||
index += 1;
|
||||
}
|
||||
if (options.slots == 0 or options.slots > 64) return error.InvalidArguments;
|
||||
return options;
|
||||
}
|
||||
|
||||
/// Picks the next event. Ids range over every issued id plus one unknown, so
|
||||
/// late, duplicate, and unknown-id messages all occur.
|
||||
fn nextEvent(random: std.Random, table: *const request.Table, sent_initialize: bool) request.Event {
|
||||
const id = random.intRangeAtMost(request.Id, 1, table.next_id);
|
||||
return switch (random.uintLessThan(u8, 10)) {
|
||||
0, 1 => .{ .send = if (!sent_initialize and random.uintLessThan(u8, 3) == 0) .initialize else .normal },
|
||||
2, 3 => .{ .response_received = .{ .id = id, .response = if (random.boolean()) .result else .peer_error } },
|
||||
4 => .{ .cancel_requested = id },
|
||||
5 => .{ .timer_fired = .{ .id = id, .kind = .hard, .generation = 0 } },
|
||||
6 => .{ .timer_fired = .{ .id = id, .kind = .idle, .generation = random.intRangeAtMost(u32, 1, 3) } },
|
||||
7 => .{ .transport_lost = id },
|
||||
8 => .{ .progress_received = .{ .token = id, .value = @floatFromInt(random.uintLessThan(u8, 5)) } },
|
||||
else => .{ .server_cancel_received = id },
|
||||
};
|
||||
}
|
||||
|
||||
pub fn main(init: std.process.Init) !void {
|
||||
const io = init.io;
|
||||
const arena = init.arena.allocator();
|
||||
const args = try init.minimal.args.toSlice(arena);
|
||||
const options = parseOptions(args) catch {
|
||||
std.debug.print("usage: request_lab <trace-path> [--seed N] [--steps N] [--slots N] [--plant-bug late-delivery]\n", .{});
|
||||
std.process.exit(2);
|
||||
};
|
||||
|
||||
var file = try std.Io.Dir.cwd().createFile(io, options.trace_path, .{});
|
||||
defer file.close(io);
|
||||
var buffer: [8192]u8 = undefined;
|
||||
var file_writer = file.writer(io, &buffer);
|
||||
var trace: mcp.trace.Writer = .init(&file_writer.interface);
|
||||
|
||||
const slots = try arena.alloc(request.Slot, options.slots);
|
||||
var table: request.Table = .init(slots, .{});
|
||||
var prng: std.Random.DefaultPrng = .init(options.seed);
|
||||
const random = prng.random();
|
||||
var sent_initialize = false;
|
||||
|
||||
var delivered: u32 = 0;
|
||||
for (0..options.steps) |_| {
|
||||
const event = nextEvent(random, &table, sent_initialize);
|
||||
var out: request.Output = .{};
|
||||
table.step(event, &out) catch |err| switch (err) {
|
||||
// Refused steps change nothing and have no model action.
|
||||
error.TooManyInFlight, error.InitializeNotCancellable => continue,
|
||||
error.IdsExhausted => return err,
|
||||
};
|
||||
if (event == .send and event.send == .initialize) sent_initialize = true;
|
||||
for (out.effects()) |effect| delivered += @intFromBool(effect == .deliver);
|
||||
|
||||
if (options.plant_late_delivery and event == .response_received and out.transition.?.from == .done) {
|
||||
out.transition.?.state = .{ .outcome = .result, .delivered = 2, .cancel_sent = false, .progress = 0 };
|
||||
}
|
||||
try request.writeTrace(&trace, "lab", &out);
|
||||
}
|
||||
try file_writer.interface.flush();
|
||||
std.debug.print("request_lab: seed {d}, {d} steps, {d} ids, {d} outcomes delivered\n", .{
|
||||
options.seed, trace.next_seq - 1, table.next_id - 1, delivered,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,799 @@
|
||||
//! U03 to U05 and U10 lab: runs a script of host actions against a real
|
||||
//! server process through `mcp.stdio.Process`. Requests go through the U02
|
||||
//! request table and the U01 encoder, `detect` runs U04 version detection,
|
||||
//! `tools` and `call` run the U05 catalog, and `listen` runs U10
|
||||
//! subscriptions, so one run traces up to five machines ("stdio",
|
||||
//! "request", "era", "catalog", "subscription") for TLC.
|
||||
//! Everything the host sees is printed as JSON lines. Dev only; never moves
|
||||
//! into fx.
|
||||
//!
|
||||
//! Usage: stdio_lab <trace-path> <script> [options] -- <server argv...>
|
||||
//! Options: --max-attempts N, --grace-ms N, --term-ms N, --backoff-ms N,
|
||||
//! --max-line N, --idle-ms N, --hard-ms N, --protocol-version V
|
||||
//! (V adds the 2026 per-request _meta to every request, without detection),
|
||||
//! --preferred V1,V2 (modern versions, most preferred first), --legacy V,
|
||||
//! --probe-ms N, --max-pages N (the catalog's page limit)
|
||||
//!
|
||||
//! Script, one action per line (blank lines and lines starting with # are skipped):
|
||||
//! start | stop | reconnect
|
||||
//! detect detect the era now and on every new process;
|
||||
//! later requests carry the chosen version
|
||||
//! era modern|legacy tell the stdio core the era by hand
|
||||
//! tools the host needs the tool list (U05)
|
||||
//! call <name> [arguments-json] call a tool
|
||||
//! invalidate mark the tool list stale
|
||||
//! send request|notification|response <json> write a raw message
|
||||
//! request <method> [params-json] send through the request table
|
||||
//! cancel <id>
|
||||
//! listen <sub> <filter> open subscription <sub> (0..3) with a filter: bits
|
||||
//! 1 tools, 2 prompts, 4 resources (U10)
|
||||
//! unlisten <sub> cancel it: notifications/cancelled
|
||||
//! expect message|stderr|dropped|<state>|era-<era state>|catalog-idle|catalog-fresh|catalog-stale
|
||||
//! [timeout_ms] (default 10000)
|
||||
//! outcome <id> <outcome> [timeout_ms] wait for a request's outcome
|
||||
//! drain <ms> handle whatever arrives for that long
|
||||
//!
|
||||
//! Output lines: {"did":...} after each action, {"seen":...} for each thing
|
||||
//! `next` returns, {"outcome":...} when a request ends, {"era":...} and
|
||||
//! {"tools":...} for detection and catalog results, {"call":...} for each
|
||||
//! tool call answer, {"sub":...} for what a subscription delivers, its
|
||||
//! acknowledgment and its end, and one {"final":...}.
|
||||
//! Exit codes: 0 the script ran, 1 an expectation wasn't met, 2 usage.
|
||||
|
||||
const std = @import("std");
|
||||
const mcp = @import("mcp");
|
||||
|
||||
/// Trace code is compiled in.
|
||||
pub const mcpv2_trace = true;
|
||||
const core = mcp.core.stdio_conn;
|
||||
const request = mcp.core.request;
|
||||
const era = mcp.core.era;
|
||||
const catalog = mcp.core.catalog;
|
||||
const subscription = mcp.core.subscription;
|
||||
const subscriptions = mcp.subscriptions;
|
||||
const tools = mcp.tools;
|
||||
const wire = mcp.wire;
|
||||
|
||||
const Options = struct {
|
||||
trace_path: []const u8,
|
||||
script_path: []const u8,
|
||||
argv: []const []const u8,
|
||||
config: core.Config = .{},
|
||||
timeouts: request.Timeouts = .{},
|
||||
max_line: usize = 16 * 1024 * 1024,
|
||||
protocol_version: ?[]const u8 = null,
|
||||
era: era.Config = .{},
|
||||
catalog: catalog.Config = .{},
|
||||
};
|
||||
|
||||
fn parseOptions(arena: std.mem.Allocator, args: []const []const u8) !Options {
|
||||
if (args.len < 5) return error.Usage;
|
||||
var options: Options = .{ .trace_path = args[1], .script_path = args[2], .argv = &.{} };
|
||||
var i: usize = 3;
|
||||
while (i < args.len) : (i += 2) {
|
||||
if (std.mem.eql(u8, args[i], "--")) {
|
||||
options.argv = args[i + 1 ..];
|
||||
break;
|
||||
}
|
||||
if (i + 1 >= args.len) return error.Usage;
|
||||
const flag = args[i];
|
||||
if (std.mem.eql(u8, flag, "--protocol-version")) {
|
||||
options.protocol_version = args[i + 1];
|
||||
continue;
|
||||
}
|
||||
if (std.mem.eql(u8, flag, "--legacy")) {
|
||||
options.era.legacy = try versionList(arena, args[i + 1]);
|
||||
continue;
|
||||
}
|
||||
if (std.mem.eql(u8, flag, "--preferred")) {
|
||||
options.era.preferred = try versionList(arena, args[i + 1]);
|
||||
continue;
|
||||
}
|
||||
const value = try std.fmt.parseInt(u32, args[i + 1], 10);
|
||||
if (std.mem.eql(u8, flag, "--max-attempts")) {
|
||||
options.config.max_attempts = @intCast(value);
|
||||
} else if (std.mem.eql(u8, flag, "--grace-ms")) {
|
||||
options.config.grace_ms = value;
|
||||
} else if (std.mem.eql(u8, flag, "--term-ms")) {
|
||||
options.config.term_ms = value;
|
||||
} else if (std.mem.eql(u8, flag, "--backoff-ms")) {
|
||||
options.config.backoff_ms = value;
|
||||
options.config.backoff_cap_ms = value * 4;
|
||||
} else if (std.mem.eql(u8, flag, "--max-line")) {
|
||||
options.max_line = value;
|
||||
} else if (std.mem.eql(u8, flag, "--idle-ms")) {
|
||||
options.timeouts.idle_ms = value;
|
||||
} else if (std.mem.eql(u8, flag, "--hard-ms")) {
|
||||
options.timeouts.hard_ms = value;
|
||||
} else if (std.mem.eql(u8, flag, "--probe-ms")) {
|
||||
options.era.probe_timeout_ms = value;
|
||||
} else if (std.mem.eql(u8, flag, "--max-pages")) {
|
||||
if (value == 0) return error.Usage;
|
||||
options.catalog.max_pages = value;
|
||||
} else return error.Usage;
|
||||
}
|
||||
if (options.argv.len == 0) return error.Usage;
|
||||
return options;
|
||||
}
|
||||
|
||||
const max_ids = 64;
|
||||
|
||||
/// The steps call each other (a timed-out initialize steps the detector), so
|
||||
/// their error set is spelled out.
|
||||
const StepError = std.Io.Writer.Error || wire.EncodeError;
|
||||
|
||||
/// Copies each top-level field of a JSON object into the message being written.
|
||||
fn writeFields(w: *wire.Writer, object: []const u8) !void {
|
||||
var stack: [512]u8 = undefined;
|
||||
var fba: std.heap.FixedBufferAllocator = .init(&stack);
|
||||
var scanner: std.json.Scanner = .initCompleteInput(fba.allocator(), object);
|
||||
defer scanner.deinit();
|
||||
if ((try scanner.next()) != .object_begin) return error.Usage;
|
||||
while (true) switch (try scanner.next()) {
|
||||
.object_end => return,
|
||||
.string => |key| {
|
||||
_ = try scanner.peekNextTokenType();
|
||||
const start = scanner.cursor;
|
||||
try scanner.skipValue();
|
||||
try w.field(key);
|
||||
try w.raw(object[start..scanner.cursor]);
|
||||
},
|
||||
// Keys written with escapes aren't needed by any script.
|
||||
else => return error.Usage,
|
||||
};
|
||||
}
|
||||
|
||||
const Lab = struct {
|
||||
io: std.Io,
|
||||
gpa: std.mem.Allocator,
|
||||
process: *mcp.stdio.Process,
|
||||
out: *std.Io.Writer,
|
||||
trace: *mcp.trace.Writer,
|
||||
table: request.Table,
|
||||
timers: [64]struct { timer: request.Timer, due_ms: i64 } = undefined,
|
||||
timer_count: usize = 0,
|
||||
outcomes: [max_ids]?request.Outcome = @splat(null),
|
||||
/// What the request being sent is: a detection message, or an operation
|
||||
/// with the version it carries: a scripted request (`pending_method` and
|
||||
/// `pending_params`), a tool list page, or a tool call (the tool's name
|
||||
/// and arguments in `pending_method` and `pending_params`).
|
||||
pending: union(enum) { discover: u8, initialize, script: era.Version, list: era.Version, call: era.Version, listen: struct { version: era.Version, filter: subscription.Filter } } = .{ .script = .none },
|
||||
pending_method: []const u8 = "",
|
||||
pending_params: ?[]const u8 = null,
|
||||
protocol_version: ?[]const u8 = null,
|
||||
detector: era.Detector,
|
||||
client: era.Client = .{ .info = .{ .name = "mcpv2-lab", .version = "0.0.0" } },
|
||||
/// Set by `detect`: every new process detects before anything else.
|
||||
detecting: bool = false,
|
||||
/// The process generation detection last started for.
|
||||
era_generation: core.Generation = 0,
|
||||
/// The detection request in flight, if any.
|
||||
detection: ?struct { id: request.Id, initialize: bool } = null,
|
||||
/// Probe timers (U04) and expiry timers (U05).
|
||||
machine_timers: [16]struct { kind: enum { probe, expiry }, value: u64, due_ms: i64 } = undefined,
|
||||
machine_timer_count: usize = 0,
|
||||
catalog: catalog.Catalog = .init(.{}),
|
||||
store: tools.Store = .{},
|
||||
/// The process generation the catalog last started for.
|
||||
catalog_generation: core.Generation = 0,
|
||||
/// The tool list page request in flight, if any.
|
||||
list_id: ?request.Id = null,
|
||||
call_ids: std.bit_set.Static(max_ids) = .empty,
|
||||
/// U10: one server's subscriptions; the listens' ids.
|
||||
subs: subscription.Table = .init(.{ .stdio = true }),
|
||||
listen_ids: std.bit_set.Static(max_ids) = .empty,
|
||||
|
||||
fn print(lab: *Lab, comptime kind: []const u8, value: anytype) !void {
|
||||
var json: std.json.Stringify = .{ .writer = lab.out };
|
||||
try json.beginObject();
|
||||
try json.objectField(kind);
|
||||
try json.write(value);
|
||||
try json.endObject();
|
||||
try lab.out.writeByte('\n');
|
||||
try lab.out.flush();
|
||||
}
|
||||
|
||||
/// Steps the detector, traces it, and carries out its effects. Returns
|
||||
/// the output, or null when the detector refused the event.
|
||||
fn stepEra(lab: *Lab, event: era.Event) StepError!?era.Output {
|
||||
var out: era.Output = .{};
|
||||
lab.detector.step(event, &out) catch |err| {
|
||||
try lab.print("did", .{ .era_error = @errorName(err) });
|
||||
return null;
|
||||
};
|
||||
try era.writeTrace(lab.trace, "lab", &out);
|
||||
for (out.effects()) |effect| switch (effect) {
|
||||
.send_discover => |index| try lab.sendDetection(.{ .discover = index }),
|
||||
.send_initialize => try lab.sendDetection(.initialize),
|
||||
.send_initialized => {
|
||||
var buffer: std.Io.Writer.Allocating = .init(lab.gpa);
|
||||
defer buffer.deinit();
|
||||
try era.writeInitialized(&buffer.writer);
|
||||
lab.process.send(.notification, buffer.written()) catch {};
|
||||
},
|
||||
.arm_probe_timer => |a| lab.addTimer(.probe, a.probe, a.after_ms),
|
||||
.ready => |version| {
|
||||
lab.process.setEra(if (version == .legacy) .legacy else .modern) catch {};
|
||||
try lab.print("era", .{ .ready = lab.versionName(version) });
|
||||
// The catalog starts over for each new process, once its era is known.
|
||||
const generation = lab.process.conn.generation;
|
||||
if (lab.catalog_generation != generation) {
|
||||
lab.catalog_generation = generation;
|
||||
try lab.stepCatalog(.{ .process_started = if (version == .legacy) .legacy else .modern });
|
||||
}
|
||||
// SUBS-05: the listens a process exit ended go out again
|
||||
// once the new process's era is known.
|
||||
if (!lab.subs.up) try lab.stepSubs(.transport_up);
|
||||
},
|
||||
.failed => |failure| try lab.print("era", .{ .failed = @tagName(failure) }),
|
||||
};
|
||||
return out;
|
||||
}
|
||||
|
||||
fn addTimer(lab: *Lab, kind: @FieldType(@TypeOf(lab.machine_timers[0]), "kind"), value: u64, after_ms: u32) void {
|
||||
lab.machine_timers[lab.machine_timer_count] = .{ .kind = kind, .value = value, .due_ms = lab.now() + after_ms };
|
||||
lab.machine_timer_count += 1;
|
||||
}
|
||||
|
||||
/// Steps the catalog, traces it, and carries out its effects on the store.
|
||||
fn stepCatalog(lab: *Lab, event: catalog.Event) StepError!void {
|
||||
var out: catalog.Output = .{};
|
||||
lab.catalog.step(event, &out) catch |err| return lab.print("did", .{ .catalog_error = @errorName(err) });
|
||||
try catalog.writeTrace(lab.trace, "lab", &out);
|
||||
for (out.effects()) |effect| switch (effect) {
|
||||
.send_page => {
|
||||
lab.list_id = try lab.sendOperation(.list);
|
||||
// Never written (the table refused it): the page failed.
|
||||
if (lab.list_id == null and lab.catalog.pending != .none) try lab.stepCatalog(.page_failed);
|
||||
},
|
||||
.discard_pages => lab.store.discard(),
|
||||
.publish => |fresh| {
|
||||
lab.store.publish();
|
||||
try lab.printTools("publish", fresh);
|
||||
},
|
||||
.fail => {
|
||||
lab.store.discard();
|
||||
try lab.print("tools", .{ .event = "fail" });
|
||||
},
|
||||
.serve => try lab.printTools("serve", true),
|
||||
.arm_expiry => |a| lab.addTimer(.expiry, a.generation, a.after_ms),
|
||||
};
|
||||
}
|
||||
|
||||
fn printTools(lab: *Lab, event: []const u8, fresh: bool) StepError!void {
|
||||
var names: [max_ids][]const u8 = undefined;
|
||||
const count = @min(lab.store.current.len(), max_ids);
|
||||
for (names[0..count], 0..) |*name, i| name.* = lab.store.current.get(i).name;
|
||||
try lab.print("tools", .{ .event = event, .fresh = fresh, .count = lab.store.current.len(), .names = names[0..count] });
|
||||
}
|
||||
|
||||
/// Reads a tool list page into the store and says what it means for the
|
||||
/// catalog. The engine takes this over in U12.
|
||||
fn pageEvent(lab: *Lab, body: @FieldType(wire.Response, "body")) StepError!catalog.Event {
|
||||
switch (body) {
|
||||
.failure => |f| return if (f.code == wire.code.invalid_params and lab.catalog.pending == .next) .invalid_cursor else .page_failed,
|
||||
.result => |r| {
|
||||
if (r.kind != .complete) return .page_failed;
|
||||
const page = tools.parsePage(r.raw) catch return .page_failed;
|
||||
const added = lab.store.addPage(lab.gpa, page) catch return .page_failed;
|
||||
if (added.invalid + added.duplicate > 0) {
|
||||
try lab.print("tools", .{ .event = "dropped", .invalid = added.invalid, .duplicate = added.duplicate });
|
||||
}
|
||||
return .{ .page_received = .{ .more = page.next_cursor != null, .ttl_ms = page.ttl_ms } };
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies the answer to request `id` for the catalog: a list page, or
|
||||
/// a tool call answer.
|
||||
fn catalogAnswer(lab: *Lab, id: request.Id, body: @FieldType(wire.Response, "body")) StepError!void {
|
||||
if (lab.list_id == id) {
|
||||
lab.list_id = null;
|
||||
return lab.stepCatalog(try lab.pageEvent(body));
|
||||
}
|
||||
if (id >= max_ids or !lab.call_ids.isSet(id)) return;
|
||||
lab.call_ids.unset(id);
|
||||
const call = tools.classifyCall(body);
|
||||
switch (call) {
|
||||
.result => |r| try lab.print("call", .{ .id = id, .kind = "result", .is_error = r.is_error, .content = r.content, .structured = r.structured }),
|
||||
.failure => |f| try lab.print("call", .{ .id = id, .kind = "failure", .code = f.code }),
|
||||
else => try lab.print("call", .{ .id = id, .kind = @tagName(call) }),
|
||||
}
|
||||
if (tools.suggestsChangedList(call)) try lab.stepCatalog(.invalidated);
|
||||
}
|
||||
|
||||
/// The `_meta` for an operation request: what detection chose, or else the
|
||||
/// `--protocol-version` option.
|
||||
fn metaFor(lab: *const Lab, version: era.Version, id: request.Id) wire.Meta {
|
||||
if (lab.detecting) return era.meta(lab.detector.config, lab.client, version, id);
|
||||
return .{
|
||||
.progress_token = id,
|
||||
.modern = if (lab.protocol_version) |v| .{
|
||||
.protocol_version = v,
|
||||
.client_capabilities = lab.client.capabilities,
|
||||
.client_info = lab.client.info,
|
||||
} else null,
|
||||
};
|
||||
}
|
||||
|
||||
/// Sends an operation request (anything but detection) through the request
|
||||
/// table. Once `detect` was used, the detector must allow it (VER-08) and
|
||||
/// picks its version. Returns its id, or null when it wasn't sent.
|
||||
fn sendOperation(lab: *Lab, kind: enum { script, list, call }) StepError!?request.Id {
|
||||
var version: era.Version = .none;
|
||||
if (lab.detecting) {
|
||||
const sent = try lab.stepEra(.operation_sent) orelse return null;
|
||||
version = sent.meta.?;
|
||||
}
|
||||
lab.pending = switch (kind) {
|
||||
.script => .{ .script = version },
|
||||
.list => .{ .list = version },
|
||||
.call => .{ .call = version },
|
||||
};
|
||||
const id = lab.table.next_id;
|
||||
const initialize = kind == .script and std.mem.eql(u8, lab.pending_method, "initialize");
|
||||
try lab.stepRequest(.{ .send = if (initialize) .initialize else .normal });
|
||||
return if (lab.table.next_id != id) id else null;
|
||||
}
|
||||
|
||||
/// Sends a listen for subscription `sub` (U10) through the request table,
|
||||
/// then tells the subscription core its id.
|
||||
fn sendListen(lab: *Lab, sub: u8, filter: subscription.Filter, resend: bool) StepError!void {
|
||||
var version: era.Version = .none;
|
||||
if (lab.detecting) {
|
||||
const sent = try lab.stepEra(.operation_sent) orelse return;
|
||||
version = sent.meta.?;
|
||||
}
|
||||
lab.pending = .{ .listen = .{ .version = version, .filter = filter } };
|
||||
const id = lab.table.next_id;
|
||||
try lab.stepRequest(.{ .send = .listen });
|
||||
if (lab.table.next_id == id or id >= max_ids) return;
|
||||
lab.listen_ids.set(id);
|
||||
try lab.stepSubs(if (resend) .{ .resent = .{ .sub = sub, .id = id } } else .{ .opened = .{ .sub = sub, .id = id, .filter = filter } });
|
||||
}
|
||||
|
||||
fn stepSubs(lab: *Lab, event: subscription.Event) StepError!void {
|
||||
var out: subscription.Output = .{};
|
||||
lab.subs.step(event, &out);
|
||||
try subscription.writeTrace(lab.trace, "lab", &lab.subs, event, &out);
|
||||
for (out.effects()) |effect| switch (effect) {
|
||||
.deliver => |d| {
|
||||
try lab.print("sub", .{ .sub = d.sub, .deliver = @tagName(d.kind) });
|
||||
if (d.kind == .tools) try lab.stepCatalog(.invalidated);
|
||||
},
|
||||
.acknowledged => |a| try lab.print("sub", .{ .sub = a.sub, .acknowledged = a.filter }),
|
||||
.ended => |e| try lab.print("sub", .{ .sub = e.sub, .ended = @tagName(e.end) }),
|
||||
// SUBS-04: on stdio a cancel is notifications/cancelled.
|
||||
.cancel_listen => |id| try lab.stepRequest(.{ .cancel_requested = id }),
|
||||
// On stdio the listens go out when the process is back instead.
|
||||
.arm_retry => {},
|
||||
.relisten => |sub| try lab.sendListen(sub, lab.subs.subs[sub].asked, true),
|
||||
};
|
||||
}
|
||||
|
||||
/// Sends a detection message through the request table. An abandoned
|
||||
/// probe is never cancelled: a legacy server must see nothing before
|
||||
/// initialize.
|
||||
fn sendDetection(lab: *Lab, what: @FieldType(Lab, "pending")) StepError!void {
|
||||
const id = lab.table.next_id;
|
||||
lab.pending = what;
|
||||
try lab.stepRequest(.{ .send = if (what == .initialize) .initialize else .normal });
|
||||
if (lab.table.next_id != id) lab.detection = .{ .id = id, .initialize = what == .initialize };
|
||||
}
|
||||
|
||||
/// Classifies the answer to request `id` for the detector: the detection
|
||||
/// request's answer, or -32022 to an operation request.
|
||||
fn eraAnswer(lab: *Lab, id: request.Id, body: @FieldType(wire.Response, "body")) !void {
|
||||
const config = lab.detector.config;
|
||||
if (lab.detection) |pending| if (pending.id == id) {
|
||||
lab.detection = null;
|
||||
_ = try lab.stepEra(if (pending.initialize) era.classifyInitialize(config, body) else era.classifyDiscover(config, body));
|
||||
return;
|
||||
};
|
||||
if (body == .failure) if (era.classifyOperationRejection(config, body.failure)) |event| {
|
||||
_ = try lab.stepEra(event);
|
||||
};
|
||||
}
|
||||
|
||||
/// Starts detection for each new process once `detect` was used.
|
||||
fn syncEra(lab: *Lab) !void {
|
||||
const c = lab.process.conn;
|
||||
if (!lab.detecting or c.state != .running or c.generation == lab.era_generation) return;
|
||||
lab.era_generation = c.generation;
|
||||
lab.detection = null;
|
||||
_ = try lab.stepEra(.process_started);
|
||||
_ = try lab.stepEra(.detect_requested);
|
||||
}
|
||||
|
||||
fn versionName(lab: *const Lab, version: era.Version) []const u8 {
|
||||
return era.versionName(lab.detector.config, version) orelse "none";
|
||||
}
|
||||
|
||||
fn now(lab: *Lab) i64 {
|
||||
return std.Io.Clock.awake.now(lab.io).toMilliseconds();
|
||||
}
|
||||
|
||||
/// Steps the request table and carries out its effects.
|
||||
fn stepRequest(lab: *Lab, event: request.Event) StepError!void {
|
||||
var out: request.Output = .{};
|
||||
lab.table.step(event, &out) catch |err| return lab.print("did", .{ .request_error = @errorName(err) });
|
||||
try request.writeTrace(lab.trace, "lab", &out);
|
||||
var lost: ?request.Id = null;
|
||||
for (out.effects()) |effect| switch (effect) {
|
||||
.write_request => |w| lab.sendEncoded(.request, w.id) catch {
|
||||
// Not written (the connection isn't running): the request is lost.
|
||||
lost = w.id;
|
||||
},
|
||||
.send_cancel => |id| lab.sendEncoded(.notification, id) catch {},
|
||||
.deliver => |d| {
|
||||
if (d.id < max_ids) lab.outcomes[d.id] = d.outcome;
|
||||
try lab.print("outcome", .{ .id = d.id, .outcome = @tagName(d.outcome) });
|
||||
// Answers are classified where they arrive; here only a
|
||||
// detection request that ended without one.
|
||||
const unanswered = d.outcome != .result and d.outcome != .peer_error;
|
||||
if (lab.detection) |pending| if (pending.id == d.id and unanswered) {
|
||||
lab.detection = null;
|
||||
if (pending.initialize and d.outcome == .timed_out) _ = try lab.stepEra(.initialize_timed_out);
|
||||
};
|
||||
if (unanswered and lab.list_id == d.id) {
|
||||
lab.list_id = null;
|
||||
try lab.stepCatalog(.page_failed);
|
||||
}
|
||||
if (unanswered and d.id < max_ids) lab.call_ids.unset(d.id);
|
||||
// SUBS-03: a server's notifications/cancelled ended a listen.
|
||||
if (d.outcome == .ended) try lab.stepSubs(.{ .final = .{ .id = d.id, .end = .server_cancelled } });
|
||||
},
|
||||
.arm_timer => |a| {
|
||||
lab.timers[lab.timer_count] = .{ .timer = a.timer, .due_ms = lab.now() + a.after_ms };
|
||||
lab.timer_count += 1;
|
||||
},
|
||||
.disarm_timers => |id| {
|
||||
var i: usize = 0;
|
||||
while (i < lab.timer_count) {
|
||||
if (lab.timers[i].timer.id == id) {
|
||||
lab.timer_count -= 1;
|
||||
lab.timers[i] = lab.timers[lab.timer_count];
|
||||
} else i += 1;
|
||||
}
|
||||
},
|
||||
};
|
||||
if (lost) |id| try lab.stepRequest(.{ .transport_lost = id });
|
||||
}
|
||||
|
||||
/// Encodes request `id` (or a cancel for it) and writes it to the server.
|
||||
fn sendEncoded(lab: *Lab, kind: core.MessageKind, id: request.Id) !void {
|
||||
var buffer: std.Io.Writer.Allocating = .init(lab.gpa);
|
||||
defer buffer.deinit();
|
||||
const config = lab.detector.config;
|
||||
if (kind == .request) switch (lab.pending) {
|
||||
.discover => |index| try era.writeDiscover(&buffer.writer, config, lab.client, index, id),
|
||||
.initialize => try era.writeInitialize(&buffer.writer, config, lab.client, id),
|
||||
.script => |version| {
|
||||
var w: wire.Writer = try .request(&buffer.writer, id, lab.pending_method, lab.metaFor(version, id));
|
||||
if (lab.pending_params) |params| try writeFields(&w, params);
|
||||
try w.end();
|
||||
},
|
||||
.list => |version| {
|
||||
const cursor = if (lab.catalog.pending == .next) lab.store.nextCursor() else null;
|
||||
try tools.writeList(&buffer.writer, id, lab.metaFor(version, id), cursor);
|
||||
},
|
||||
.call => |version| try tools.writeCall(&buffer.writer, id, lab.metaFor(version, id), lab.pending_method, lab.pending_params, .{}),
|
||||
.listen => |l| try subscriptions.writeListen(&buffer.writer, id, lab.metaFor(l.version, id), l.filter),
|
||||
} else {
|
||||
var w: wire.Writer = try .notification(&buffer.writer, "notifications/cancelled", .{});
|
||||
try w.field("requestId");
|
||||
try w.int(@intCast(id));
|
||||
try w.end();
|
||||
}
|
||||
try lab.process.send(kind, buffer.written());
|
||||
}
|
||||
|
||||
/// Feeds what the host saw into the request table.
|
||||
fn handle(lab: *Lab, incoming: mcp.stdio.Incoming) !void {
|
||||
switch (incoming) {
|
||||
.message => |m| {
|
||||
try lab.print("seen", .{ .message = m.bytes });
|
||||
switch (m.message) {
|
||||
.response => |r| if (r.id) |id| {
|
||||
try lab.stepRequest(.{ .response_received = .{
|
||||
.id = id,
|
||||
.response = if (r.body == .result) .result else .peer_error,
|
||||
} });
|
||||
try lab.eraAnswer(id, r.body);
|
||||
try lab.catalogAnswer(id, r.body);
|
||||
// A result ends a listen gracefully, an error refuses it.
|
||||
if (id < max_ids and lab.listen_ids.isSet(id)) try lab.stepSubs(.{ .final = .{ .id = id, .end = if (r.body == .result) .result else .refused } });
|
||||
} else if (r.body == .failure and lab.detector.pending == .discover) {
|
||||
// An error without an id during the probe means a legacy server.
|
||||
_ = try lab.stepEra(.legacy_answer);
|
||||
},
|
||||
.notification => |n| if (std.mem.eql(u8, n.method, "notifications/cancelled")) {
|
||||
if (subscriptions.cancelledId(n.params)) |id| try lab.stepRequest(.{ .server_cancel_received = id });
|
||||
} else if (subscriptions.subscriptionId(n.params)) |sid| switch (subscriptions.notice(n.method, n.params)) {
|
||||
// SUBS-02: on stdio the subscriptionId is the only way to tell them apart.
|
||||
.acknowledged => |f| try lab.stepSubs(.{ .acknowledged = .{ .id = sid, .filter = f } }),
|
||||
.changed => |kind| try lab.stepSubs(.{ .notified = .{ .id = sid, .kind = kind } }),
|
||||
.other => {},
|
||||
} else if (std.mem.eql(u8, n.method, "notifications/progress")) {
|
||||
var found: [2]?[]const u8 = undefined;
|
||||
wire.objectFields(n.params orelse "{}", &.{ "progressToken", "progress" }, &found) catch return;
|
||||
const token = std.fmt.parseInt(request.Id, found[0] orelse return, 10) catch return;
|
||||
const value = std.fmt.parseFloat(f64, found[1] orelse return) catch return;
|
||||
try lab.stepRequest(.{ .progress_received = .{ .token = token, .value = value } });
|
||||
} else if (std.mem.eql(u8, n.method, "notifications/tools/list_changed")) {
|
||||
// Invalidates everything cached for the server.
|
||||
try lab.stepCatalog(.invalidated);
|
||||
},
|
||||
.request => {},
|
||||
}
|
||||
},
|
||||
.stderr => |text| try lab.print("seen", .{ .stderr = text }),
|
||||
.dropped => |err| try lab.print("seen", .{ .dropped = @errorName(err) }),
|
||||
.changed => |c| {
|
||||
try lab.print("seen", .{ .state = @tagName(c.state), .lost = c.lost });
|
||||
if (c.lost) {
|
||||
for (lab.table.slots) |slot| if (slot.used) try lab.stepRequest(.{ .transport_lost = slot.id });
|
||||
// SUBS-05: the exit ends every listen.
|
||||
if (lab.subs.up) try lab.stepSubs(.transport_down);
|
||||
}
|
||||
// A subscription waiting to listen again is demand.
|
||||
if (c.state == .idle and lab.subs.waiting()) lab.process.start() catch {};
|
||||
},
|
||||
.timeout => {},
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles one event or due timer, waiting at most until `until_ms`.
|
||||
/// Returns what arrived, or `timeout`.
|
||||
fn pump(lab: *Lab, until_ms: i64) !mcp.stdio.Incoming {
|
||||
var wait = until_ms;
|
||||
for (lab.timers[0..lab.timer_count]) |t| wait = @min(wait, t.due_ms);
|
||||
for (lab.machine_timers[0..lab.machine_timer_count]) |t| wait = @min(wait, t.due_ms);
|
||||
const incoming = try lab.process.next(@intCast(@max(0, wait - lab.now())));
|
||||
if (incoming != .timeout) {
|
||||
try lab.handle(incoming);
|
||||
try lab.syncEra();
|
||||
return incoming;
|
||||
}
|
||||
var p: usize = 0;
|
||||
while (p < lab.machine_timer_count) {
|
||||
if (lab.machine_timers[p].due_ms <= lab.now()) {
|
||||
const timer = lab.machine_timers[p];
|
||||
lab.machine_timer_count -= 1;
|
||||
lab.machine_timers[p] = lab.machine_timers[lab.machine_timer_count];
|
||||
switch (timer.kind) {
|
||||
.probe => _ = try lab.stepEra(.{ .probe_timer_fired = @intCast(timer.value) }),
|
||||
.expiry => try lab.stepCatalog(.{ .expired = timer.value }),
|
||||
}
|
||||
} else p += 1;
|
||||
}
|
||||
// Fire every request timer that is due.
|
||||
var i: usize = 0;
|
||||
while (i < lab.timer_count) {
|
||||
if (lab.timers[i].due_ms <= lab.now()) {
|
||||
const timer = lab.timers[i].timer;
|
||||
lab.timer_count -= 1;
|
||||
lab.timers[i] = lab.timers[lab.timer_count];
|
||||
try lab.stepRequest(.{ .timer_fired = timer });
|
||||
i = 0;
|
||||
} else i += 1;
|
||||
}
|
||||
return .timeout;
|
||||
}
|
||||
|
||||
fn matches(target: []const u8, incoming: mcp.stdio.Incoming) bool {
|
||||
return switch (incoming) {
|
||||
.message => std.mem.eql(u8, target, "message"),
|
||||
.stderr => std.mem.eql(u8, target, "stderr"),
|
||||
.dropped => std.mem.eql(u8, target, "dropped"),
|
||||
.changed => |c| std.mem.eql(u8, target, @tagName(c.state)),
|
||||
.timeout => false,
|
||||
};
|
||||
}
|
||||
|
||||
fn expect(lab: *Lab, target: []const u8, timeout_ms: u32) !bool {
|
||||
const until = lab.now() + timeout_ms;
|
||||
while (true) {
|
||||
if (std.meta.stringToEnum(core.State, target)) |state| {
|
||||
if (lab.process.conn.state == state) return true;
|
||||
}
|
||||
if (std.mem.startsWith(u8, target, "era-")) {
|
||||
if (std.meta.stringToEnum(era.State, target[4..])) |state| if (lab.detector.state == state) return true;
|
||||
}
|
||||
if (std.mem.eql(u8, target, "catalog-idle") and !lab.catalog.listing) return true;
|
||||
if (std.mem.startsWith(u8, target, "catalog-")) {
|
||||
if (std.meta.stringToEnum(catalog.Cache, target[8..])) |cache| if (lab.catalog.cache == cache and !lab.catalog.listing) return true;
|
||||
}
|
||||
if (lab.now() >= until) return false;
|
||||
if (matches(target, try lab.pump(until))) return true;
|
||||
}
|
||||
}
|
||||
|
||||
fn awaitOutcome(lab: *Lab, id: request.Id, want: []const u8, timeout_ms: u32) !bool {
|
||||
const until = lab.now() + timeout_ms;
|
||||
while (true) {
|
||||
if (id < max_ids) if (lab.outcomes[id]) |got| return std.mem.eql(u8, @tagName(got), want);
|
||||
if (lab.now() >= until) return false;
|
||||
_ = try lab.pump(until);
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs one script line. Returns false when an expectation isn't met.
|
||||
fn run(lab: *Lab, line: []const u8) !bool {
|
||||
var words = std.mem.tokenizeScalar(u8, line, ' ');
|
||||
const verb = words.next() orelse return true;
|
||||
const p = lab.process;
|
||||
if (std.mem.eql(u8, verb, "expect") or std.mem.eql(u8, verb, "outcome")) {
|
||||
const first = words.next() orelse return error.Usage;
|
||||
const met = if (std.mem.eql(u8, verb, "expect")) blk: {
|
||||
const timeout = if (words.next()) |t| try std.fmt.parseInt(u32, t, 10) else 10_000;
|
||||
break :blk try lab.expect(first, timeout);
|
||||
} else blk: {
|
||||
const want = words.next() orelse return error.Usage;
|
||||
const timeout = if (words.next()) |t| try std.fmt.parseInt(u32, t, 10) else 10_000;
|
||||
break :blk try lab.awaitOutcome(try std.fmt.parseInt(request.Id, first, 10), want, timeout);
|
||||
};
|
||||
try lab.print("did", .{ .expect = line, .met = met });
|
||||
return met;
|
||||
}
|
||||
if (std.mem.eql(u8, verb, "drain")) {
|
||||
const until = lab.now() + try std.fmt.parseInt(u32, words.next() orelse return error.Usage, 10);
|
||||
while (lab.now() < until) _ = try lab.pump(until);
|
||||
return true;
|
||||
}
|
||||
if (std.mem.eql(u8, verb, "detect")) {
|
||||
lab.detecting = true;
|
||||
try lab.syncEra();
|
||||
try lab.print("did", .{ .action = verb, .era = @tagName(lab.detector.state) });
|
||||
return true;
|
||||
}
|
||||
if (std.mem.eql(u8, verb, "request")) {
|
||||
lab.pending_method = words.next() orelse return error.Usage;
|
||||
const rest = std.mem.trim(u8, words.rest(), " ");
|
||||
lab.pending_params = if (rest.len > 0) rest else null;
|
||||
_ = try lab.sendOperation(.script);
|
||||
try lab.print("did", .{ .action = verb, .next_id = lab.table.next_id });
|
||||
return true;
|
||||
}
|
||||
if (std.mem.eql(u8, verb, "tools") or std.mem.eql(u8, verb, "invalidate")) {
|
||||
try lab.stepCatalog(if (verb[0] == 't') .need else .invalidated);
|
||||
try lab.print("did", .{ .action = verb, .cache = @tagName(lab.catalog.cache), .listing = lab.catalog.listing });
|
||||
return true;
|
||||
}
|
||||
if (std.mem.eql(u8, verb, "call")) {
|
||||
lab.pending_method = words.next() orelse return error.Usage;
|
||||
const rest = std.mem.trim(u8, words.rest(), " ");
|
||||
lab.pending_params = if (rest.len > 0) rest else null;
|
||||
const id = try lab.sendOperation(.call);
|
||||
if (id) |n| if (n < max_ids) lab.call_ids.set(n);
|
||||
try lab.print("did", .{ .action = verb, .id = id });
|
||||
return true;
|
||||
}
|
||||
if (std.mem.eql(u8, verb, "listen") or std.mem.eql(u8, verb, "unlisten")) {
|
||||
const sub = std.fmt.parseInt(u8, words.next() orelse return error.Usage, 10) catch return error.Usage;
|
||||
if (verb[0] == 'u') {
|
||||
try lab.stepSubs(.{ .cancel = sub });
|
||||
} else {
|
||||
const filter = std.fmt.parseInt(subscription.Filter, words.next() orelse return error.Usage, 10) catch return error.Usage;
|
||||
if (!lab.subs.canOpen(sub)) return error.Usage;
|
||||
try lab.sendListen(sub, filter, false);
|
||||
}
|
||||
try lab.print("did", .{ .action = verb, .sub = sub });
|
||||
return true;
|
||||
}
|
||||
if (std.mem.eql(u8, verb, "cancel")) {
|
||||
try lab.stepRequest(.{ .cancel_requested = try std.fmt.parseInt(request.Id, words.next() orelse return error.Usage, 10) });
|
||||
try lab.print("did", .{ .action = verb });
|
||||
return true;
|
||||
}
|
||||
const result: mcp.stdio.Error!void = if (std.mem.eql(u8, verb, "start"))
|
||||
p.start()
|
||||
else if (std.mem.eql(u8, verb, "stop"))
|
||||
p.stop()
|
||||
else if (std.mem.eql(u8, verb, "reconnect"))
|
||||
p.reconnect()
|
||||
else if (std.mem.eql(u8, verb, "era"))
|
||||
p.setEra(if (std.mem.eql(u8, words.next() orelse "", "legacy")) .legacy else .modern)
|
||||
else if (std.mem.eql(u8, verb, "send")) blk: {
|
||||
const kind = std.meta.stringToEnum(core.MessageKind, words.next() orelse "") orelse return error.Usage;
|
||||
break :blk p.send(kind, std.mem.trim(u8, words.rest(), " "));
|
||||
} else return error.Usage;
|
||||
if (result) |_| {
|
||||
try lab.print("did", .{ .action = verb, .state = @tagName(p.conn.state) });
|
||||
} else |err| {
|
||||
try lab.print("did", .{ .action = verb, .@"error" = @errorName(err) });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/// A comma-separated version list, for --preferred and --legacy.
|
||||
fn versionList(arena: std.mem.Allocator, text: []const u8) ![]const []const u8 {
|
||||
var list: std.ArrayList([]const u8) = .empty;
|
||||
var versions = std.mem.tokenizeScalar(u8, text, ',');
|
||||
while (versions.next()) |v| try list.append(arena, v);
|
||||
if (list.items.len == 0 or list.items.len > era.max_preferred) return error.Usage;
|
||||
return list.items;
|
||||
}
|
||||
|
||||
pub fn main(init: std.process.Init) !void {
|
||||
const io = init.io;
|
||||
const arena = init.arena.allocator();
|
||||
const args = try init.minimal.args.toSlice(arena);
|
||||
const options = parseOptions(arena, args) catch {
|
||||
std.debug.print("usage: stdio_lab <trace-path> <script> [--max-attempts N] [--grace-ms N] [--term-ms N] " ++
|
||||
"[--backoff-ms N] [--max-line N] [--idle-ms N] [--hard-ms N] [--protocol-version V] [--preferred V1,V2] " ++
|
||||
"[--legacy V] [--probe-ms N] [--max-pages N] -- <server argv...>\n", .{});
|
||||
std.process.exit(2);
|
||||
};
|
||||
const script = try std.Io.Dir.cwd().readFileAlloc(io, options.script_path, arena, .limited(1 << 20));
|
||||
|
||||
var trace_file = try std.Io.Dir.cwd().createFile(io, options.trace_path, .{});
|
||||
defer trace_file.close(io);
|
||||
var trace_buffer: [8192]u8 = undefined;
|
||||
var trace_writer = trace_file.writer(io, &trace_buffer);
|
||||
var trace: mcp.trace.Writer = .init(&trace_writer.interface);
|
||||
|
||||
var out_buffer: [8192]u8 = undefined;
|
||||
var stdout = std.Io.File.stdout().writer(io, &out_buffer);
|
||||
|
||||
var process: mcp.stdio.Process = undefined;
|
||||
process.init(io, init.gpa, .{
|
||||
.argv = options.argv,
|
||||
.environ_map = init.environ_map,
|
||||
.config = options.config,
|
||||
.max_line = options.max_line,
|
||||
.trace = &trace,
|
||||
.trace_instance = "lab",
|
||||
});
|
||||
var slots: [16]request.Slot = undefined;
|
||||
var lab: Lab = .{
|
||||
.io = io,
|
||||
.gpa = init.gpa,
|
||||
.process = &process,
|
||||
.out = &stdout.interface,
|
||||
.trace = &trace,
|
||||
.table = .init(&slots, options.timeouts),
|
||||
.protocol_version = options.protocol_version,
|
||||
.detector = .init(options.era),
|
||||
.catalog = .init(options.catalog),
|
||||
};
|
||||
|
||||
var ok = true;
|
||||
var lines = std.mem.splitScalar(u8, script, '\n');
|
||||
while (lines.next()) |raw| {
|
||||
const line = std.mem.trim(u8, raw, " \r\t");
|
||||
if (line.len == 0 or line[0] == '#') continue;
|
||||
const ran = lab.run(line);
|
||||
lab.syncEra() catch {};
|
||||
if (!(ran catch |err| {
|
||||
try lab.print("did", .{ .line = line, .@"error" = @errorName(err) });
|
||||
ok = false;
|
||||
break;
|
||||
})) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const c = process.conn;
|
||||
try lab.print("final", .{
|
||||
.state = @tagName(c.state),
|
||||
.generation = c.generation,
|
||||
.attempts = c.attempts,
|
||||
.term_sent = c.term_sent,
|
||||
.kill_sent = c.kill_sent,
|
||||
.errored = c.errored,
|
||||
.stderr_dropped = process.stderr_dropped.load(.monotonic),
|
||||
.spawn_error = if (process.spawn_error) |err| @errorName(err) else null,
|
||||
.in_flight = lab.table.inFlightCount(),
|
||||
.era = @tagName(lab.detector.state),
|
||||
.era_version = lab.versionName(lab.detector.version),
|
||||
.era_failure = if (lab.detector.failure) |f| @tagName(f) else null,
|
||||
.catalog = @tagName(lab.catalog.cache),
|
||||
.tools = lab.store.current.len(),
|
||||
});
|
||||
lab.store.deinit(init.gpa);
|
||||
process.deinit();
|
||||
try trace_writer.interface.flush();
|
||||
if (!ok) std.process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
//! Bridge self-test program. Not product code.
|
||||
//!
|
||||
//! Runs the Selftest machine as a pure step function over events from the
|
||||
//! command line and writes a real trace through `mcp.trace`. With
|
||||
//! `--allow-late-delivery` it reproduces the planted bug from
|
||||
//! Selftest.tla, so the bridge must reject its trace.
|
||||
//!
|
||||
//! Usage: trace_demo <trace-path> [--allow-late-delivery] <event>...
|
||||
//! Events: response_received, cancel_requested
|
||||
|
||||
const std = @import("std");
|
||||
const mcp = @import("mcp");
|
||||
|
||||
/// Trace code is compiled in.
|
||||
pub const mcpv2_trace = true;
|
||||
|
||||
const Status = enum { pending, done, cancelled };
|
||||
|
||||
const Event = enum { response_received, cancel_requested };
|
||||
|
||||
const Machine = struct {
|
||||
status: Status = .pending,
|
||||
outcomes: i64 = 0,
|
||||
|
||||
/// Pure: returns the next state, or null when the event is not allowed.
|
||||
fn step(self: Machine, event: Event, allow_late_delivery: bool) ?Machine {
|
||||
return switch (event) {
|
||||
.response_received => switch (self.status) {
|
||||
.pending => .{ .status = .done, .outcomes = self.outcomes + 1 },
|
||||
.cancelled => if (allow_late_delivery) .{ .status = .done, .outcomes = self.outcomes + 1 } else null,
|
||||
.done => null,
|
||||
},
|
||||
.cancel_requested => switch (self.status) {
|
||||
.pending => .{ .status = .cancelled, .outcomes = self.outcomes + 1 },
|
||||
.cancelled, .done => null,
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
pub fn main(init: std.process.Init) !void {
|
||||
const io = init.io;
|
||||
const args = try init.minimal.args.toSlice(init.arena.allocator());
|
||||
if (args.len < 2) {
|
||||
std.debug.print("usage: trace_demo <trace-path> [--allow-late-delivery] <event>...\n", .{});
|
||||
return error.InvalidArguments;
|
||||
}
|
||||
|
||||
var allow_late_delivery = false;
|
||||
for (args[2..]) |arg| {
|
||||
if (std.mem.eql(u8, arg, "--allow-late-delivery")) allow_late_delivery = true;
|
||||
}
|
||||
|
||||
var file = try std.Io.Dir.cwd().createFile(io, args[1], .{});
|
||||
defer file.close(io);
|
||||
var buffer: [4096]u8 = undefined;
|
||||
var file_writer = file.writer(io, &buffer);
|
||||
var trace: mcp.trace.Writer = .init(&file_writer.interface);
|
||||
|
||||
var machine: Machine = .{};
|
||||
for (args[2..]) |arg| {
|
||||
if (std.mem.eql(u8, arg, "--allow-late-delivery")) continue;
|
||||
const event = std.meta.stringToEnum(Event, arg) orelse {
|
||||
std.debug.print("trace_demo: unknown event {s}\n", .{arg});
|
||||
return error.InvalidArguments;
|
||||
};
|
||||
const next = machine.step(event, allow_late_delivery) orelse continue;
|
||||
try trace.write(.{
|
||||
.machine = "selftest",
|
||||
.instance = "demo",
|
||||
.event = @tagName(event),
|
||||
.from = @tagName(machine.status),
|
||||
.to = @tagName(next.status),
|
||||
.data = &.{.{ .name = "outcomes", .value = .{ .int = next.outcomes } }},
|
||||
});
|
||||
machine = next;
|
||||
}
|
||||
try file_writer.interface.flush();
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
//! U01 lab: decodes each line of an input file with the real wire decoder and
|
||||
//! prints what it found, one JSON line per input line, for the u01 Bun
|
||||
//! scenarios. Empty lines are skipped. Dev only; never moves into fx.
|
||||
//!
|
||||
//! Output per line, by message kind:
|
||||
//! {"kind":"response","id":7,"result":"complete"} or "code":-32602 for errors
|
||||
//! {"kind":"notification","method":"notifications/progress"}
|
||||
//! {"kind":"request","id":"\"srv-1\"","method":"ping"} (id is the raw JSON text)
|
||||
//! {"error":"InvalidJson"}
|
||||
//!
|
||||
//! Usage: wire_lab <input-file>. Exit codes: 0 done, 2 usage.
|
||||
|
||||
const std = @import("std");
|
||||
const mcp = @import("mcp");
|
||||
|
||||
pub fn main(init: std.process.Init) !void {
|
||||
const io = init.io;
|
||||
const arena = init.arena.allocator();
|
||||
const args = try init.minimal.args.toSlice(arena);
|
||||
if (args.len != 2) {
|
||||
std.debug.print("usage: wire_lab <input-file>\n", .{});
|
||||
std.process.exit(2);
|
||||
}
|
||||
const input = try std.Io.Dir.cwd().readFileAlloc(io, args[1], arena, .unlimited);
|
||||
|
||||
var buffer: [8192]u8 = undefined;
|
||||
var stdout = std.Io.File.stdout().writer(io, &buffer);
|
||||
const out = &stdout.interface;
|
||||
var lines = std.mem.splitScalar(u8, input, '\n');
|
||||
while (lines.next()) |line| {
|
||||
if (line.len == 0) continue;
|
||||
var json: std.json.Stringify = .{ .writer = out };
|
||||
try json.beginObject();
|
||||
if (mcp.wire.decode(line)) |message| switch (message) {
|
||||
.response => |r| {
|
||||
try json.objectField("kind");
|
||||
try json.write("response");
|
||||
try json.objectField("id");
|
||||
try json.write(r.id);
|
||||
switch (r.body) {
|
||||
.result => |result| {
|
||||
try json.objectField("result");
|
||||
try json.write(@tagName(result.kind));
|
||||
},
|
||||
.failure => |failure| {
|
||||
try json.objectField("code");
|
||||
try json.write(failure.code);
|
||||
},
|
||||
}
|
||||
},
|
||||
.notification => |n| {
|
||||
try json.objectField("kind");
|
||||
try json.write("notification");
|
||||
try json.objectField("method");
|
||||
try json.write(n.method);
|
||||
},
|
||||
.request => |q| {
|
||||
try json.objectField("kind");
|
||||
try json.write("request");
|
||||
try json.objectField("id");
|
||||
try json.write(q.id);
|
||||
try json.objectField("method");
|
||||
try json.write(q.method);
|
||||
},
|
||||
} else |err| {
|
||||
try json.objectField("error");
|
||||
try json.write(@errorName(err));
|
||||
}
|
||||
try json.endObject();
|
||||
try out.writeByte('\n');
|
||||
}
|
||||
try out.flush();
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Scenario contract for the MCP-v2 Bun driver.
|
||||
|
||||
import type { Verdict } from "./bridge";
|
||||
|
||||
export interface ScenarioContext {
|
||||
/** Repository root. */
|
||||
root: string;
|
||||
/** Binaries built for this run, by name (for example "stdio_lab", "trace_demo"). */
|
||||
bin: { [name: string]: string };
|
||||
/** Temporary HOME for this case. */
|
||||
home: string;
|
||||
/** Temporary working directory for this case. */
|
||||
workdir: string;
|
||||
/** Where the binary under test writes its trace (also exported as MCPV2_TRACE). */
|
||||
tracePath: string;
|
||||
/** Folder for scenario-specific files such as fake-server scripts and logs. */
|
||||
caseDir: string;
|
||||
}
|
||||
|
||||
export interface TraceExpectation {
|
||||
machine: string;
|
||||
instance?: string;
|
||||
/** Folder holding the model and its trace wrapper. Relative paths resolve from the repository root. */
|
||||
specDir: string;
|
||||
wrapper: string;
|
||||
/** ACCEPTED for normal runs. REJECTED only for known-bad controls. */
|
||||
verdict: Verdict;
|
||||
/** Model constants the trace can't show, defined in the generated TraceObs module. */
|
||||
constants?: { [name: string]: number };
|
||||
}
|
||||
|
||||
export interface Scenario {
|
||||
/** Unique name, used with --case. */
|
||||
name: string;
|
||||
/** Ledger item that owns the scenario, for example "S05" or "U03". */
|
||||
unit: string;
|
||||
description: string;
|
||||
/** Binaries the scenario needs, built by the driver. */
|
||||
needs: string[];
|
||||
/** Command line to run. Called after the case folders exist. */
|
||||
command(ctx: ScenarioContext): string[];
|
||||
/** Optional setup, for example writing a fake-server script into caseDir. */
|
||||
setup?(ctx: ScenarioContext): void;
|
||||
expect: {
|
||||
exit: number;
|
||||
traces: TraceExpectation[];
|
||||
/** Optional stdout check. Returns a problem description, or null when stdout is right. */
|
||||
stdout?: (text: string) => string | null;
|
||||
};
|
||||
/** Talks to a real server; the driver here never runs it. */
|
||||
real?: boolean;
|
||||
/** Hard limit for the command, in milliseconds. Defaults to 30 s. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Driver plumbing scenarios: a real Zig binary writes its trace to
|
||||
// MCPV2_TRACE, and the driver checks it through the bridge. The second case is
|
||||
// a known-bad control that must be REJECTED.
|
||||
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const selftestTrace = { machine: "selftest", specDir: "tests/tla/mcp/selftest", wrapper: "SelftestTrace" } as const;
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
{
|
||||
name: "selftest-follows-model",
|
||||
unit: "S05",
|
||||
description: "Cancel a pending request; the trace is a behavior of Selftest.tla",
|
||||
needs: ["trace_demo"],
|
||||
command: (ctx) => [ctx.bin.trace_demo, ctx.tracePath, "cancel_requested", "response_received"],
|
||||
expect: { exit: 0, traces: [{ ...selftestTrace, verdict: "ACCEPTED" }] },
|
||||
},
|
||||
{
|
||||
name: "selftest-planted-late-delivery",
|
||||
unit: "S05",
|
||||
description: "Known-bad control: the binary delivers a response after a cancel",
|
||||
needs: ["trace_demo"],
|
||||
command: (ctx) => [ctx.bin.trace_demo, ctx.tracePath, "--allow-late-delivery", "cancel_requested", "response_received"],
|
||||
expect: { exit: 0, traces: [{ ...selftestTrace, verdict: "REJECTED" }] },
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,149 @@
|
||||
// U01 wire format, step 6. The real decoder, run by tests/tla/drivers/mcp/labs/wire_lab.zig,
|
||||
// against every JSON-RPC example in the MCP specification pages
|
||||
// (fixtures/spec-examples.json), plus hand-written cases for ids, malformed input, and oversized
|
||||
// messages. U01 has no model, so these cases check exact output instead of
|
||||
// traces.
|
||||
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const EXAMPLES = resolve(import.meta.dir, "../fixtures/spec-examples.json");
|
||||
|
||||
interface Case {
|
||||
/** Where the input came from, for failure messages. */
|
||||
source: string;
|
||||
input: string | Buffer;
|
||||
/** The exact line wire_lab must print. */
|
||||
expected: string;
|
||||
}
|
||||
|
||||
const MAX_U64 = 18446744073709551615n;
|
||||
|
||||
/** The decoder's id for a response, computed independently: an integer, or a
|
||||
* string of plain decimal digits, that fits in 64 bits. */
|
||||
function ownId(id: unknown): number | bigint | null {
|
||||
if (typeof id === "number") return Number.isSafeInteger(id) && id >= 0 ? id : null;
|
||||
if (typeof id === "string" && /^(0|[1-9][0-9]*)$/.test(id)) {
|
||||
const value = BigInt(id);
|
||||
return value <= MAX_U64 ? value : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function line(fields: [string, unknown][]): string {
|
||||
return "{" + fields.map(([k, v]) => `${JSON.stringify(k)}:${typeof v === "bigint" ? v.toString() : JSON.stringify(v)}`).join(",") + "}";
|
||||
}
|
||||
|
||||
/** What wire_lab prints for a valid example message, derived from JSON.parse. */
|
||||
function expectedFor(message: any): string {
|
||||
if ("method" in message) {
|
||||
if (!("id" in message)) return line([["kind", "notification"], ["method", message.method]]);
|
||||
if (message.id === null) return line([["error", "InvalidId"]]);
|
||||
return line([["kind", "request"], ["id", JSON.stringify(message.id)], ["method", message.method]]);
|
||||
}
|
||||
const id = ownId(message.id);
|
||||
if ("error" in message) return line([["kind", "response"], ["id", id], ["code", message.error.code]]);
|
||||
const type = message.result?.resultType;
|
||||
const kind = type === undefined || type === "complete" ? "complete" : type === "input_required" ? "input_required" : "unrecognized";
|
||||
return line([["kind", "response"], ["id", id], ["result", kind]]);
|
||||
}
|
||||
|
||||
/** Every JSON-RPC 2.0 message example in the MCP specification pages, as
|
||||
* extracted into fixtures/spec-examples.json with line breaks folded to spaces. */
|
||||
function specExamples(): Case[] {
|
||||
const examples = JSON.parse(readFileSync(EXAMPLES, "utf8")) as { source: string; input: string }[];
|
||||
return examples.map(({ source, input }) => ({ source, input, expected: expectedFor(JSON.parse(input)) }));
|
||||
}
|
||||
|
||||
const raw = (source: string, input: string | Buffer, expected: string): Case => ({ source, input, expected });
|
||||
const response = (id: string, result = "complete") => `{"kind":"response","id":${id},"result":"${result}"}`;
|
||||
const failure = (name: string) => `{"error":"${name}"}`;
|
||||
|
||||
const idCases: Case[] = [
|
||||
...[
|
||||
["7", "7"], ['"7"', "7"], ['"0"', "0"], ["0", "0"], ['"007"', "null"], ['"abc"', "null"], ['"1_0"', "null"],
|
||||
["-1", "null"], ["1.5", "null"], ["1e2", "null"], ["null", "null"],
|
||||
["9007199254740991", "9007199254740991"], ['"18446744073709551615"', "18446744073709551615"],
|
||||
["18446744073709551615", "18446744073709551615"], ["18446744073709551616", "null"],
|
||||
].map(([id, expected]) => raw(`response id ${id}`, `{"jsonrpc":"2.0","id":${id},"result":{}}`, response(expected))),
|
||||
raw("response without an id", '{"jsonrpc":"2.0","result":{}}', response("null")),
|
||||
raw("response id is an object", '{"jsonrpc":"2.0","id":{},"result":{}}', failure("InvalidId")),
|
||||
raw("response id is true", '{"jsonrpc":"2.0","id":true,"result":{}}', failure("InvalidId")),
|
||||
raw("server request, string id", '{"jsonrpc":"2.0","id":"srv-1","method":"ping"}', '{"kind":"request","id":"\\"srv-1\\"","method":"ping"}'),
|
||||
raw("server request, number id", '{"jsonrpc":"2.0","id":5,"method":"ping"}', '{"kind":"request","id":"5","method":"ping"}'),
|
||||
raw("server request, null id", '{"jsonrpc":"2.0","id":null,"method":"ping"}', failure("InvalidId")),
|
||||
raw("server request, fractional id", '{"jsonrpc":"2.0","id":1.5,"method":"ping"}', failure("InvalidId")),
|
||||
];
|
||||
|
||||
const deep = (levels: number) => `{"jsonrpc":"2.0","id":1,"result":{"a":${"[".repeat(levels)}${"]".repeat(levels)}}}`;
|
||||
const malformedCases: Case[] = [
|
||||
raw("truncated", '{"jsonrpc":"2.0","id":1,"result":{}', failure("InvalidJson")),
|
||||
raw("trailing bytes", '{"jsonrpc":"2.0","id":1,"result":{}} x', failure("InvalidJson")),
|
||||
raw("invalid UTF-8 in a string", Buffer.concat([Buffer.from('{"jsonrpc":"2.0","id":1,"result":{"t":"'), Buffer.from([0xff]), Buffer.from('"}}')]), failure("InvalidJson")),
|
||||
raw("batch", '[{"jsonrpc":"2.0","method":"ping"}]', failure("NotAnObject")),
|
||||
raw("bare null", "null", failure("NotAnObject")),
|
||||
raw("empty object", "{}", failure("NotJsonRpc2")),
|
||||
raw("version 1.0", '{"jsonrpc":"1.0","id":1,"result":{}}', failure("NotJsonRpc2")),
|
||||
raw("version as a number", '{"jsonrpc":2.0,"id":1,"result":{}}', failure("NotJsonRpc2")),
|
||||
raw("result and error", '{"jsonrpc":"2.0","id":1,"result":{},"error":{"code":1,"message":"x"}}', failure("Ambiguous")),
|
||||
raw("neither result nor error", '{"jsonrpc":"2.0","id":1}', failure("Ambiguous")),
|
||||
raw("method and error", '{"jsonrpc":"2.0","id":1,"method":"m","error":{"code":1,"message":"x"}}', failure("Ambiguous")),
|
||||
raw("response with params", '{"jsonrpc":"2.0","id":1,"params":{},"result":{}}', failure("Ambiguous")),
|
||||
raw("duplicate id", '{"jsonrpc":"2.0","id":1,"id":2,"result":{}}', failure("DuplicateField")),
|
||||
raw("duplicate result", '{"jsonrpc":"2.0","id":1,"result":{},"result":{}}', failure("DuplicateField")),
|
||||
raw("method with escapes", '{"jsonrpc":"2.0","method":"pi\\u006eg"}', failure("InvalidMethod")),
|
||||
raw("params as an array", '{"jsonrpc":"2.0","method":"m","params":[1]}', failure("InvalidParams")),
|
||||
raw("result as an array", '{"jsonrpc":"2.0","id":1,"result":[]}', failure("InvalidResult")),
|
||||
raw("error code 1e3", '{"jsonrpc":"2.0","id":1,"error":{"code":1e3,"message":"x"}}', failure("InvalidErrorObject")),
|
||||
raw("error without a message", '{"jsonrpc":"2.0","id":1,"error":{"code":-32600}}', failure("InvalidErrorObject")),
|
||||
raw("1,000 levels of nesting", deep(1000), response("1")),
|
||||
raw("5,000 levels of nesting", deep(5000), failure("TooDeep")),
|
||||
raw("unknown fields are ignored", '{"jsonrpc":"2.0","id":3,"extra":{"id":9},"result":{"resultType":"input_required"}}', response("3", "input_required")),
|
||||
];
|
||||
|
||||
const bigString = "x".repeat(8 * 1024 * 1024);
|
||||
const manyFields = Array.from({ length: 100_000 }, (_, i) => `"f${i}":${i}`).join(",");
|
||||
const oversizedCases: Case[] = [
|
||||
raw("8 MiB string in a result", `{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"${bigString}"}]}}`, response("1")),
|
||||
raw("100,000 result fields", `{"jsonrpc":"2.0","id":2,"result":{${manyFields}}}`, response("2")),
|
||||
raw("8 MiB notification params", `{"jsonrpc":"2.0","method":"notifications/message","params":{"data":"${bigString}"}}`, '{"kind":"notification","method":"notifications/message"}'),
|
||||
];
|
||||
|
||||
function wireScenario(name: string, description: string, cases: () => Case[], minimum: number, timeoutMs?: number): Scenario {
|
||||
let selected: Case[] = [];
|
||||
return {
|
||||
name,
|
||||
unit: "U01",
|
||||
description,
|
||||
needs: ["wire_lab"],
|
||||
setup: (ctx) => {
|
||||
selected = cases();
|
||||
const parts = selected.flatMap((c) => [typeof c.input === "string" ? Buffer.from(c.input) : c.input, Buffer.from("\n")]);
|
||||
writeFileSync(join(ctx.caseDir, "input.jsonl"), Buffer.concat(parts));
|
||||
},
|
||||
command: (ctx) => [ctx.bin.wire_lab, join(ctx.caseDir, "input.jsonl")],
|
||||
expect: {
|
||||
exit: 0,
|
||||
traces: [],
|
||||
stdout: (text) => {
|
||||
if (selected.length < minimum) return `only ${selected.length} cases, expected at least ${minimum}`;
|
||||
const got = text.split("\n").filter((l) => l.length > 0);
|
||||
if (got.length !== selected.length) return `${got.length} output lines for ${selected.length} cases`;
|
||||
for (const [i, c] of selected.entries()) {
|
||||
if (got[i] !== c.expected) return `${c.source}: got ${got[i].slice(0, 200)}, expected ${c.expected}`;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
},
|
||||
timeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
// The frozen snapshot holds 36 such examples (four more use "..." and aren't JSON).
|
||||
wireScenario("u01-spec-examples", "Every JSON-RPC example in the spec snapshot decodes as JSON.parse reads it", specExamples, 36),
|
||||
wireScenario("u01-ids", "String and number ids: which ones match the client's, and which are rejected", () => idCases, idCases.length),
|
||||
wireScenario("u01-malformed", "Malformed, ambiguous, and too-deep messages fail with the exact error", () => malformedCases, malformedCases.length),
|
||||
wireScenario("u01-oversized", "Multi-megabyte messages and 100,000 fields decode without a hidden limit", () => oversizedCases, oversizedCases.length, 60_000),
|
||||
];
|
||||
@@ -0,0 +1,34 @@
|
||||
// U02 request lifecycle, step 6 at the core level: the real Zig core, driven
|
||||
// by tests/tla/drivers/mcp/labs/request_lab.zig with seeded random events (late, duplicate,
|
||||
// and unknown-id messages included), must produce traces that TLC accepts as
|
||||
// behaviors of spec/Request.tla. The planted-bug case is a known-bad control.
|
||||
// Transport-level u02 scenarios (over stdio) follow once U03 exists.
|
||||
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const requestTrace = { machine: "request", instance: "lab", specDir: "tests/tla/mcp", wrapper: "RequestTrace" } as const;
|
||||
|
||||
const seeded = (seed: number, steps: number, slots: number): Scenario => ({
|
||||
name: `u02-core-seed-${seed}`,
|
||||
unit: "U02",
|
||||
description: `Random core run: seed ${seed}, ${steps} steps, ${slots} slots; the trace is a behavior of Request.tla`,
|
||||
needs: ["request_lab"],
|
||||
command: (ctx) => [ctx.bin.request_lab, ctx.tracePath, "--seed", String(seed), "--steps", String(steps), "--slots", String(slots)],
|
||||
expect: { exit: 0, traces: [{ ...requestTrace, verdict: "ACCEPTED" }] },
|
||||
});
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
seeded(1, 200, 4),
|
||||
seeded(2, 200, 4),
|
||||
seeded(3, 300, 2),
|
||||
seeded(4, 300, 8),
|
||||
seeded(5, 500, 4),
|
||||
{
|
||||
name: "u02-core-planted-late-delivery",
|
||||
unit: "U02",
|
||||
description: "Known-bad control: a late response is reported as delivered again",
|
||||
needs: ["request_lab"],
|
||||
command: (ctx) => [ctx.bin.request_lab, ctx.tracePath, "--seed", "1", "--steps", "200", "--plant-bug", "late-delivery"],
|
||||
expect: { exit: 0, traces: [{ ...requestTrace, verdict: "REJECTED" }] },
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,85 @@
|
||||
// U02 request lifecycle over a real transport: requests are encoded by U01,
|
||||
// tracked by the U02 request table, and carried by the U03 stdio connection
|
||||
// to the scripted fake server. Each run must produce request traces that TLC
|
||||
// accepts as behaviors of spec/Request.tla, and stdio traces it accepts as
|
||||
// behaviors of spec/StdioConn.tla.
|
||||
|
||||
import { expectFinal, labScenario, type LabCase, outcomes } from "../lab";
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const overStdio = (c: Omit<LabCase, "unit" | "requestTrace">): Scenario => labScenario({ ...c, unit: "U02", requestTrace: true });
|
||||
const received = (log: { dir: string; line: string }[], method: string) => log.filter((l) => l.dir === "in" && l.line.includes(`"method":"${method}"`));
|
||||
const want = (got: Record<string, string>, expected: Record<string, string>): string | null => {
|
||||
for (const [id, outcome] of Object.entries(expected)) if (got[id] !== outcome) return `request ${id} ended ${got[id]}, expected ${outcome}`;
|
||||
return null;
|
||||
};
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
overStdio({
|
||||
name: "u02-stdio-responses",
|
||||
description: "Three requests end with their own outcomes, including an error from the server",
|
||||
rules: [
|
||||
{ method: "tools/list", result: { tools: [] }, delayMs: 100 },
|
||||
{ method: "ping", result: {} },
|
||||
{ method: "boom", error: { code: -32602, message: "bad params" } },
|
||||
],
|
||||
script: ["start", "request tools/list", "request ping", "request boom", "outcome 1 result", "outcome 2 result", "outcome 3 peer_error", "stop", "expect stopped"],
|
||||
check: (lines, final, log) => {
|
||||
const problem = want(outcomes(lines), { 1: "result", 2: "result", 3: "peer_error" });
|
||||
if (problem) return problem;
|
||||
// Every request carried its id as the progress token (WIRE-06).
|
||||
if (!received(log, "tools/list").some((l) => l.line.includes('"progressToken":1'))) return "tools/list was sent without its progress token";
|
||||
return expectFinal(final, { state: "stopped", in_flight: 0 });
|
||||
},
|
||||
}),
|
||||
overStdio({
|
||||
name: "u02-stdio-cancel-race",
|
||||
description: "A cancelled request stays cancelled when the server's late response arrives (REQ-01, REQ-02)",
|
||||
rules: [{ method: "slow", result: {}, delayMs: 500 }],
|
||||
script: ["start", "request slow", "cancel 1", "outcome 1 cancelled", "drain 900", "stop", "expect stopped"],
|
||||
check: (lines, _, log) => {
|
||||
if (received(log, "notifications/cancelled").length !== 1) return "the server should get exactly one notifications/cancelled";
|
||||
if (!lines.some((l) => l.seen?.message?.includes('"id":1'))) return "the late response should arrive and be ignored";
|
||||
return want(outcomes(lines), { 1: "cancelled" });
|
||||
},
|
||||
}),
|
||||
overStdio({
|
||||
name: "u02-stdio-timeout",
|
||||
description: "A request the server never answers times out, and the server is told to stop (REQ-03)",
|
||||
rules: [{ method: "hang", noReply: true }],
|
||||
labFlags: ["--idle-ms", "300", "--hard-ms", "2000"],
|
||||
script: ["start", "request hang", "outcome 1 timed_out 5000", "stop", "expect stopped"],
|
||||
check: (lines, _, log) => {
|
||||
if (received(log, "notifications/cancelled").length !== 1) return "a timeout should send notifications/cancelled";
|
||||
return want(outcomes(lines), { 1: "timed_out" });
|
||||
},
|
||||
}),
|
||||
overStdio({
|
||||
name: "u02-stdio-progress",
|
||||
description: "Progress notifications for a request in flight are accepted before its result (REQ-04)",
|
||||
rules: [
|
||||
{
|
||||
method: "work",
|
||||
before: [
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: 1, progress: 1 } },
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: 1, progress: 2 } },
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: 9, progress: 1 } },
|
||||
],
|
||||
result: {},
|
||||
},
|
||||
],
|
||||
script: ["start", "request work", "outcome 1 result", "stop", "expect stopped"],
|
||||
check: (lines) => want(outcomes(lines), { 1: "result" }),
|
||||
}),
|
||||
overStdio({
|
||||
name: "u02-stdio-crash-lost",
|
||||
description: "A crash ends requests in flight as lost; a request while backing off is lost too; after the restart requests work (STDIO-07)",
|
||||
rules: [
|
||||
{ method: "crashy", noReply: true, then: "crash" },
|
||||
{ method: "ping", result: {} },
|
||||
],
|
||||
labFlags: ["--backoff-ms", "300"],
|
||||
script: ["start", "request crashy", "outcome 1 lost", "expect backoff", "request ping", "outcome 2 lost", "expect idle", "start", "request ping", "outcome 3 result", "stop", "expect stopped"],
|
||||
check: (lines, final) => want(outcomes(lines), { 1: "lost", 2: "lost", 3: "result" }) ?? expectFinal(final, { state: "stopped", generation: 2 }),
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,120 @@
|
||||
// U03 stdio connection, step 6: the real I/O layer (src/mcp/io/process.zig),
|
||||
// run by tests/tla/drivers/mcp/labs/stdio_lab.zig against the scripted fake server, must
|
||||
// produce stdio traces that TLC accepts as behaviors of spec/StdioConn.tla.
|
||||
// The planted case is a known-bad control: the lab allows three failures
|
||||
// while the model is told two, so TLC must reject the trace.
|
||||
|
||||
import { expectFinal, labScenario, type LabCase, messages, PING, PING2 } from "../lab";
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const stdioScenario = (c: Omit<LabCase, "unit">): Scenario => labScenario({ ...c, unit: "U03" });
|
||||
|
||||
const crashLoop = ["start", "expect backoff", "expect idle", "start", "expect backoff", "expect idle", "start"];
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
stdioScenario({
|
||||
name: "u03-exchange-and-stop",
|
||||
description: "A request gets its response; a stop closes stdin and the server exits without signals",
|
||||
rules: [{ method: "ping", result: {} }],
|
||||
script: ["start", `send request ${PING}`, "expect message", "stop", "expect stopped"],
|
||||
check: (lines, final) => (messages(lines).length !== 1 ? "expected one message" : expectFinal(final, { state: "stopped", term_sent: false, kill_sent: false })),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-crash-restart",
|
||||
description: "A server that crashes after each reply backs off, restarts on demand, and answers again",
|
||||
rules: [{ method: "ping", result: {}, then: "crash" }],
|
||||
labFlags: ["--backoff-ms", "50"],
|
||||
script: ["start", `send request ${PING}`, "expect message", "expect backoff", "expect idle", "start", `send request ${PING2}`, "expect message", "expect backoff", "stop"],
|
||||
check: (lines, final) => (messages(lines).length !== 2 ? "expected two messages" : expectFinal(final, { state: "stopped", generation: 2 })),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-server-exits-on-its-own",
|
||||
description: "A server that shuts itself down counts as a failure and backs off",
|
||||
rules: [{ method: "ping", result: {}, then: "exit" }],
|
||||
labFlags: ["--backoff-ms", "50"],
|
||||
script: ["start", `send request ${PING}`, "expect message", "expect backoff", "stop"],
|
||||
check: (_, final) => expectFinal(final, { state: "stopped", attempts: 1 }),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-crash-loop-fails",
|
||||
description: "A server that dies at startup fails after the attempt limit; a reconnect starts counting again",
|
||||
fakeFlags: ["--exit-at-start", "3"],
|
||||
labFlags: ["--max-attempts", "3", "--backoff-ms", "20"],
|
||||
maxAttempts: 3,
|
||||
script: [...crashLoop, "expect failed", "start", "reconnect", "start", "expect backoff", "stop"],
|
||||
check: (lines, final) =>
|
||||
!lines.some((l) => l.did?.action === "start" && l.did.state === "failed") ? "a start while failed should be ignored" : expectFinal(final, { state: "stopped", attempts: 1, generation: 4 }),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-ignores-closed-stdin",
|
||||
description: "A server that keeps running after stdin closes gets SIGTERM after the grace period",
|
||||
fakeFlags: ["--linger"],
|
||||
labFlags: ["--grace-ms", "200", "--term-ms", "5000"],
|
||||
script: ["start", "stop", "expect stopped"],
|
||||
check: (_, final) => expectFinal(final, { state: "stopped", term_sent: true, kill_sent: false }),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-ignores-sigterm",
|
||||
description: "A server that also ignores SIGTERM gets SIGKILL",
|
||||
fakeFlags: ["--linger", "--ignore-term"],
|
||||
labFlags: ["--grace-ms", "200", "--term-ms", "200"],
|
||||
script: ["start", "stop", "expect stopped"],
|
||||
check: (_, final) => expectFinal(final, { state: "stopped", term_sent: true, kill_sent: true }),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-stderr-flood",
|
||||
description: "1 MB of stderr before a reply never stalls the server or changes the state (STDIO-05)",
|
||||
rules: [{ method: "ping", stderr: ["e".repeat(200)], stderrRepeat: 5000, result: {} }],
|
||||
script: ["start", `send request ${PING}`, "expect message 30000", "stop", "expect stopped"],
|
||||
check: (lines, final) => {
|
||||
const stderr = lines.filter((l) => l.seen?.stderr !== undefined).length;
|
||||
if (messages(lines).length !== 1) return "expected the reply after the flood";
|
||||
if (stderr + final.stderr_dropped < 5000) return `only ${stderr} stderr lines seen and ${final.stderr_dropped} dropped`;
|
||||
return expectFinal(final, { state: "stopped", errored: false });
|
||||
},
|
||||
timeoutMs: 60_000,
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-oversized-line",
|
||||
description: "A stdout line over the limit kills the server at once, without SIGTERM",
|
||||
rules: [{ method: "ping", rawBytes: 100_000, result: {} }],
|
||||
labFlags: ["--max-line", "65536"],
|
||||
script: ["start", `send request ${PING}`, "expect killing", "expect backoff", "stop"],
|
||||
check: (_, final) => expectFinal(final, { state: "stopped", errored: true, term_sent: false, kill_sent: true }),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-banner-dropped",
|
||||
description: "A non-MCP line on stdout is dropped and the connection keeps working",
|
||||
rules: [{ method: "ping", result: {} }],
|
||||
fakeFlags: ["--banner", "Server running on stdio"],
|
||||
script: ["start", "expect dropped", `send request ${PING}`, "expect message", "stop", "expect stopped"],
|
||||
check: (lines) => (lines.find((l) => l.seen?.dropped)?.seen?.dropped !== "InvalidJson" ? "expected the banner dropped as InvalidJson" : null),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-crlf-and-empty-lines",
|
||||
description: "CRLF line endings and empty lines are handled; nothing is dropped",
|
||||
rules: [{ method: "ping", raw: '{"jsonrpc":"2.0","id":1,"result":{}}\r' }],
|
||||
fakeFlags: ["--banner", ""],
|
||||
script: ["start", `send request ${PING}`, "expect message", "stop", "expect stopped"],
|
||||
check: (lines) => {
|
||||
if (lines.some((l) => l.seen?.dropped)) return "nothing should be dropped";
|
||||
return messages(lines)[0] === '{"jsonrpc":"2.0","id":1,"result":{}}' ? null : `message was ${JSON.stringify(messages(lines)[0])}`;
|
||||
},
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-spawn-failure",
|
||||
description: "A command that can't start counts as a failure and backs off",
|
||||
server: ["/nonexistent/mcpv2-test-server"],
|
||||
script: ["start", "expect backoff", "stop"],
|
||||
check: (_, final) => (final.spawn_error === null ? "expected a spawn error" : expectFinal(final, { state: "stopped", attempts: 1 })),
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u03-planted-limit-mismatch",
|
||||
description: "Known-bad control: the lab allows three failures but the model is told two",
|
||||
fakeFlags: ["--exit-at-start", "3"],
|
||||
labFlags: ["--max-attempts", "3", "--backoff-ms", "20"],
|
||||
maxAttempts: 2,
|
||||
verdict: "REJECTED",
|
||||
script: ["start", "expect backoff", "expect idle", "start", "expect backoff", "stop"],
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,225 @@
|
||||
// U04 version detection, step 6: the era core (src/mcp/core/era.zig) driven by
|
||||
// tests/tla/drivers/mcp/labs/stdio_lab.zig against the scripted fake server. Every scenario
|
||||
// checks the era, stdio, and request traces against their models, and what
|
||||
// the server actually received, from its own log.
|
||||
|
||||
import type { Rule } from "../fakes/stdio";
|
||||
import { expectFinal, labScenario, outcomes, type LabCase, type Seen } from "../lab";
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const MODERN = "2026-07-28";
|
||||
const NEXT = "2027-01-01";
|
||||
const LEGACY = "2025-11-25";
|
||||
const VERSION_KEY = "io.modelcontextprotocol/protocolVersion";
|
||||
|
||||
const discoverResult = (versions: string[]): Rule => ({
|
||||
method: "server/discover",
|
||||
result: { resultType: "complete", supportedVersions: versions, capabilities: {}, _meta: { "io.modelcontextprotocol/serverInfo": { name: "fake", version: "1" } } },
|
||||
});
|
||||
const unsupported = (supported: string[], extra: Partial<Rule> = {}): Rule => ({
|
||||
method: "server/discover",
|
||||
error: { code: -32022, message: "Unsupported protocol version", data: { supported, requested: MODERN } },
|
||||
...extra,
|
||||
});
|
||||
const initializeResult = (version = LEGACY): Rule => ({
|
||||
method: "initialize",
|
||||
result: { protocolVersion: version, capabilities: {}, serverInfo: { name: "fake", version: "1" } },
|
||||
});
|
||||
// A tools/list that answers only the given era: a version, or null for none sent.
|
||||
const toolsFor = (version: string | null): Rule => ({ method: "tools/list", version, result: { tools: [] } });
|
||||
|
||||
interface Received {
|
||||
method?: string;
|
||||
version: string | null;
|
||||
}
|
||||
|
||||
// What the server received, in order: each message's method and 2026 _meta version.
|
||||
function received(log: { dir: string; line: string }[]): Received[] {
|
||||
return log
|
||||
.filter((l) => l.dir === "in")
|
||||
.map((l) => {
|
||||
const m = JSON.parse(l.line) as { method?: string; params?: { _meta?: Record<string, string> } };
|
||||
return { method: m.method, version: m.params?._meta?.[VERSION_KEY] ?? null };
|
||||
});
|
||||
}
|
||||
|
||||
const methods = (log: { dir: string; line: string }[]) => received(log).map((r) => r.method);
|
||||
const eraLines = (lines: Seen[]) => lines.filter((l) => l.era).map((l) => l.era!);
|
||||
|
||||
function sameList(got: unknown[], want: unknown[], what: string): string | null {
|
||||
return JSON.stringify(got) === JSON.stringify(want) ? null : `${what} were ${JSON.stringify(got)}, expected ${JSON.stringify(want)}`;
|
||||
}
|
||||
|
||||
// Detection with one preferred version unless the case says otherwise.
|
||||
function eraScenario(c: Omit<LabCase, "unit" | "requestTrace" | "era"> & { preferred?: string[]; eraVerdict?: "ACCEPTED" | "REJECTED"; modelPreferred?: number }): Scenario {
|
||||
const preferred = c.preferred ?? [MODERN];
|
||||
return labScenario({
|
||||
...c,
|
||||
unit: "U04",
|
||||
requestTrace: true,
|
||||
era: { preferred: c.modelPreferred ?? preferred.length, verdict: c.eraVerdict },
|
||||
labFlags: ["--preferred", preferred.join(","), ...(c.labFlags ?? ["--probe-ms", "3000"])],
|
||||
});
|
||||
}
|
||||
|
||||
const STOP = ["stop", "expect stopped 5000"];
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
eraScenario({
|
||||
name: "u04-modern",
|
||||
description: "A modern server answers server/discover; requests carry the chosen version (VER-02, WIRE-03)",
|
||||
rules: [discoverResult([MODERN]), toolsFor(MODERN)],
|
||||
script: ["start", "detect", "expect era-modern", "request tools/list", "outcome 2 result", ...STOP],
|
||||
check: (lines, final, log) =>
|
||||
expectFinal(final, { era: "modern", era_version: MODERN }) ??
|
||||
sameList(received(log), [{ method: "server/discover", version: MODERN }, { method: "tools/list", version: MODERN }], "received messages") ??
|
||||
sameList(eraLines(lines), [{ ready: MODERN }], "era lines"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-legacy-method-not-found",
|
||||
description: "A legacy server rejects server/discover; initialize, then initialized, then requests without 2026 _meta (VER-03, VER-06, VER-07)",
|
||||
rules: [initializeResult(), toolsFor(null)],
|
||||
script: ["start", "detect", "expect era-legacy", "request tools/list", "outcome 3 result", ...STOP],
|
||||
check: (_, final, log) =>
|
||||
expectFinal(final, { era: "legacy", era_version: LEGACY }) ??
|
||||
sameList(methods(log), ["server/discover", "initialize", "notifications/initialized", "tools/list"], "received methods") ??
|
||||
(received(log)[3].version === null ? null : "tools/list carried a 2026 version on a legacy server"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-legacy-silent",
|
||||
description: "A legacy server that ignores server/discover is found by the probe timeout; the probe is never cancelled",
|
||||
rules: [{ method: "server/discover", noReply: true }, initializeResult()],
|
||||
labFlags: ["--probe-ms", "300"],
|
||||
script: ["start", "detect", "expect era-legacy", ...STOP],
|
||||
check: (_, final, log) =>
|
||||
expectFinal(final, { era: "legacy" }) ?? sameList(methods(log), ["server/discover", "initialize", "notifications/initialized"], "received methods"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-legacy-error-without-id",
|
||||
description: "An error without an id during the probe means a legacy server",
|
||||
rules: [{ method: "server/discover", raw: '{"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"Invalid Request"}}' }, initializeResult()],
|
||||
script: ["start", "detect", "expect era-legacy", ...STOP],
|
||||
check: (_, final) => expectFinal(final, { era: "legacy" }),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-unsupported-retry",
|
||||
description: "-32022 to the probe retries with the first listed version the client supports (VER-01)",
|
||||
preferred: [NEXT, MODERN],
|
||||
rules: [unsupported([MODERN], { version: NEXT } as Partial<Rule>), { ...discoverResult([MODERN]), version: MODERN }, toolsFor(MODERN)],
|
||||
script: ["start", "detect", "expect era-modern", "request tools/list", "outcome 3 result", ...STOP],
|
||||
check: (_, final, log) =>
|
||||
expectFinal(final, { era: "modern", era_version: MODERN }) ??
|
||||
sameList(
|
||||
received(log),
|
||||
[{ method: "server/discover", version: NEXT }, { method: "server/discover", version: MODERN }, { method: "tools/list", version: MODERN }],
|
||||
"received messages",
|
||||
),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-dual-lists-legacy",
|
||||
description: "-32022 listing only the legacy version falls back to initialize",
|
||||
rules: [unsupported([LEGACY]), initializeResult()],
|
||||
script: ["start", "detect", "expect era-legacy", ...STOP],
|
||||
check: (_, final, log) => expectFinal(final, { era: "legacy" }) ?? sameList(methods(log), ["server/discover", "initialize", "notifications/initialized"], "received methods"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-modern-rejection",
|
||||
description: "-32021 to the probe fails without falling back",
|
||||
rules: [{ method: "server/discover", error: { code: -32021, message: "Missing required client capability", data: { requiredCapabilities: { sampling: {} } } } }],
|
||||
script: ["start", "detect", "expect era-failed", ...STOP],
|
||||
check: (lines, final, log) =>
|
||||
expectFinal(final, { era: "failed", era_failure: "rejected" }) ??
|
||||
sameList(methods(log), ["server/discover"], "received methods") ??
|
||||
sameList(eraLines(lines), [{ failed: "rejected" }], "era lines"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-no-shared-version",
|
||||
description: "A DiscoverResult with no shared version and no legacy version fails without initialize (VER-01)",
|
||||
rules: [discoverResult(["2099-01-01"])],
|
||||
script: ["start", "detect", "expect era-failed", ...STOP],
|
||||
check: (_, final, log) => expectFinal(final, { era: "failed", era_failure: "no_shared_version" }) ?? sameList(methods(log), ["server/discover"], "received methods"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-unsupported-initialize-version",
|
||||
description: "initialize answered with a version other than the legacy one fails; initialized is never sent (VER-09)",
|
||||
rules: [initializeResult("2024-11-05")],
|
||||
script: ["start", "detect", "expect era-failed", ...STOP],
|
||||
check: (_, final, log) =>
|
||||
expectFinal(final, { era: "failed", era_failure: "unsupported_version" }) ?? sameList(methods(log), ["server/discover", "initialize"], "received methods"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-initialize-timeout",
|
||||
description: "A server that answers neither the probe nor initialize fails when initialize times out",
|
||||
rules: [{ method: "server/discover", noReply: true }, { method: "initialize", noReply: true }],
|
||||
labFlags: ["--probe-ms", "200", "--idle-ms", "500"],
|
||||
script: ["start", "detect", "expect era-failed", ...STOP],
|
||||
check: (lines, final) =>
|
||||
expectFinal(final, { era: "failed", era_failure: "initialize_failed" }) ?? (outcomes(lines)[2] === "timed_out" ? null : "initialize should time out"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-modern-went-quiet",
|
||||
description: "A modern server that rejects a version it lists, then stops answering, fails without initialize (VER-03)",
|
||||
rules: [unsupported([MODERN], { times: 1 }), { method: "server/discover", noReply: true }],
|
||||
labFlags: ["--probe-ms", "300"],
|
||||
script: ["start", "detect", "expect era-failed", ...STOP],
|
||||
check: (_, final, log) =>
|
||||
expectFinal(final, { era: "failed", era_failure: "went_quiet" }) ?? sameList(methods(log), ["server/discover", "server/discover"], "received methods"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-fall-forward",
|
||||
description: "A modern server too slow for the probe rejects initialize with -32022; the client probes once more",
|
||||
rules: [discoverResult([MODERN]), { method: "initialize", error: { code: -32022, message: `Unsupported protocol version; supported: ${MODERN}`, data: { supported: [MODERN] } } }],
|
||||
fakeFlags: ["--start-delay", "800"],
|
||||
labFlags: ["--probe-ms", "200"],
|
||||
script: ["start", "detect", "expect era-modern 5000", ...STOP],
|
||||
check: (_, final, log) =>
|
||||
expectFinal(final, { era: "modern", era_version: MODERN }) ?? sameList(methods(log), ["server/discover", "initialize", "server/discover"], "received methods"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-redetect-after-restart",
|
||||
description: "A new process detects again before any request",
|
||||
rules: [discoverResult([MODERN]), { method: "tools/list", times: 1, result: { tools: [] }, then: "crash" }, toolsFor(MODERN)],
|
||||
labFlags: ["--probe-ms", "3000", "--backoff-ms", "20"],
|
||||
script: ["start", "detect", "expect era-modern", "request tools/list", "outcome 2 result", "expect idle", "start", "expect era-modern", "request tools/list", "outcome 4 result", ...STOP],
|
||||
check: (_, final, log) =>
|
||||
expectFinal(final, { era: "modern", generation: 2 }) ?? sameList(methods(log), ["server/discover", "tools/list", "server/discover", "tools/list"], "received methods"),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-requests-wait",
|
||||
description: "A request before detection finishes is refused, and nothing reaches the server (VER-08)",
|
||||
rules: [{ ...discoverResult([MODERN]), delayMs: 300 }, toolsFor(MODERN)],
|
||||
script: ["start", "detect", "request tools/list", "expect era-modern", "request tools/list", "outcome 2 result", ...STOP],
|
||||
check: (lines, final, log) => {
|
||||
if (!lines.some((l) => l.did?.era_error === "NotReady")) return "the early request should be refused";
|
||||
return expectFinal(final, { era: "modern" }) ?? sameList(methods(log), ["server/discover", "tools/list"], "received methods");
|
||||
},
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-operation-version-rejected",
|
||||
description: "-32022 to a request after detection switches to another listed version, never to legacy (VER-01, VER-05)",
|
||||
preferred: [NEXT, MODERN],
|
||||
rules: [
|
||||
discoverResult([NEXT, MODERN]),
|
||||
{ method: "tools/list", version: NEXT, error: { code: -32022, message: "Unsupported protocol version", data: { supported: [MODERN, LEGACY] } } },
|
||||
toolsFor(MODERN),
|
||||
],
|
||||
script: ["start", "detect", "expect era-modern", "request tools/list", "outcome 2 peer_error", "request tools/list", "outcome 3 result", ...STOP],
|
||||
check: (lines, final, log) =>
|
||||
expectFinal(final, { era: "modern", era_version: MODERN }) ??
|
||||
sameList(eraLines(lines), [{ ready: NEXT }, { ready: MODERN }], "era lines") ??
|
||||
sameList(
|
||||
received(log).map((r) => r.version),
|
||||
[NEXT, NEXT, MODERN],
|
||||
"versions received",
|
||||
),
|
||||
}),
|
||||
eraScenario({
|
||||
name: "u04-planted-count-mismatch",
|
||||
description: "Known-bad control: the lab has two preferred versions but the model is told one, so the retry with the second can't be matched",
|
||||
preferred: [NEXT, MODERN],
|
||||
modelPreferred: 1,
|
||||
eraVerdict: "REJECTED",
|
||||
rules: [unsupported([MODERN], { version: NEXT } as Partial<Rule>), { ...discoverResult([MODERN]), version: MODERN }],
|
||||
script: ["start", "detect", "expect era-modern", ...STOP],
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,214 @@
|
||||
// U05 tool catalog, step 6: the catalog core (src/mcp/core/catalog.zig) and
|
||||
// the tool store (src/mcp/protocol/tools.zig), driven by
|
||||
// tests/tla/drivers/mcp/labs/stdio_lab.zig against the scripted fake server after real era
|
||||
// detection. Every scenario checks the catalog, era, stdio, and request
|
||||
// traces against their models, and the requests the server actually received.
|
||||
|
||||
import type { Rule } from "../fakes/stdio";
|
||||
import { expectFinal, labScenario, type LabCase, type Seen } from "../lab";
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const MODERN = "2026-07-28";
|
||||
const LEGACY = "2025-11-25";
|
||||
const LIST_CHANGED = { jsonrpc: "2.0", method: "notifications/tools/list_changed" };
|
||||
|
||||
const discover: Rule = { method: "server/discover", result: { resultType: "complete", supportedVersions: [MODERN], capabilities: { tools: { listChanged: true } } } };
|
||||
const initialize: Rule = { method: "initialize", result: { protocolVersion: LEGACY, capabilities: { tools: { listChanged: true } }, serverInfo: { name: "fake", version: "1" } } };
|
||||
const tool = (name: string) => ({ name, inputSchema: { type: "object" } });
|
||||
const page = (names: string[], extra: Record<string, unknown> = {}) => ({ tools: names.map(tool), ...extra });
|
||||
const ping = (before: unknown[] = []): Rule => ({ method: "ping", before, result: {} });
|
||||
|
||||
// What the server received: each request's method, with the cursor of tools/list requests.
|
||||
function received(log: { dir: string; line: string }[]): string[] {
|
||||
return log
|
||||
.filter((l) => l.dir === "in")
|
||||
.map((l) => JSON.parse(l.line) as { method?: string; params?: { cursor?: string } })
|
||||
.filter((m) => m.method !== undefined && m.method !== "notifications/initialized")
|
||||
.map((m) => (m.method === "tools/list" && m.params?.cursor !== undefined ? `tools/list ${JSON.stringify(m.params.cursor)}` : m.method!));
|
||||
}
|
||||
|
||||
const toolLines = (lines: Seen[]) => lines.filter((l) => l.tools).map((l) => l.tools!);
|
||||
const calls = (lines: Seen[]) => lines.filter((l) => l.call).map((l) => l.call!);
|
||||
const events = (lines: Seen[]) => toolLines(lines).filter((t) => t.event !== "dropped").map((t) => (t.event === "fail" ? "fail" : `${t.event}${t.fresh ? "" : " (not cached)"} ${t.names?.join(",")}`));
|
||||
|
||||
function same(got: unknown, want: unknown, what: string): string | null {
|
||||
return JSON.stringify(got) === JSON.stringify(want) ? null : `${what} were ${JSON.stringify(got)}, expected ${JSON.stringify(want)}`;
|
||||
}
|
||||
|
||||
// Detection, then the catalog, on one preferred modern version.
|
||||
function catalogScenario(c: Omit<LabCase, "unit" | "requestTrace" | "era" | "catalog"> & { maxPages?: number; modelPages?: number; catalogVerdict?: "ACCEPTED" | "REJECTED" }): Scenario {
|
||||
const maxPages = c.maxPages ?? 100;
|
||||
return labScenario({
|
||||
...c,
|
||||
unit: "U05",
|
||||
requestTrace: true,
|
||||
era: { preferred: 1 },
|
||||
catalog: { maxPages: c.modelPages ?? maxPages, verdict: c.catalogVerdict },
|
||||
labFlags: ["--preferred", MODERN, "--probe-ms", "3000", "--max-pages", String(maxPages), ...(c.labFlags ?? [])],
|
||||
});
|
||||
}
|
||||
|
||||
const START = ["start", "detect", "expect era-modern"];
|
||||
const START_LEGACY = ["start", "detect", "expect era-legacy"];
|
||||
const STOP = ["stop", "expect stopped 5000"];
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
catalogScenario({
|
||||
name: "u05-pages-and-cache",
|
||||
description: "Two pages joined in order, an empty-string cursor followed, malformed tools and duplicates dropped, then served from the cache (CAT-01, CAT-04)",
|
||||
rules: [
|
||||
discover,
|
||||
{ method: "tools/list", cursor: null, result: { tools: [tool("a"), { name: "bad" }], nextCursor: "", ttlMs: 60000 } },
|
||||
{ method: "tools/list", cursor: "", result: { tools: [tool("b"), tool("a")], ttlMs: 30000 } },
|
||||
],
|
||||
script: [...START, "tools", "expect catalog-fresh", "tools", ...STOP],
|
||||
check: (lines, final, log) =>
|
||||
same(events(lines), ["publish a,b", "serve a,b"], "catalog results") ??
|
||||
same(toolLines(lines).filter((t) => t.event === "dropped").map((t) => [t.invalid, t.duplicate]), [[1, 0], [0, 1]], "dropped counts") ??
|
||||
same(received(log), ["server/discover", "tools/list", 'tools/list ""'], "requests") ??
|
||||
expectFinal(final, { catalog: "fresh", tools: 2 }),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-zero-ttl",
|
||||
description: "A modern list with ttlMs 0 is never served from the cache (CAT-04)",
|
||||
rules: [discover, { method: "tools/list", result: page(["a"], { ttlMs: 0 }) }],
|
||||
script: [...START, "tools", "expect catalog-idle", "tools", "expect catalog-idle", ...STOP],
|
||||
check: (lines, _, log) => same(events(lines), ["publish (not cached) a", "publish (not cached) a"], "catalog results") ?? same(received(log), ["server/discover", "tools/list", "tools/list"], "requests"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-ttl-expires",
|
||||
description: "A cached modern list goes stale when its shortest page TTL runs out, and is listed again on the next need (CAT-04)",
|
||||
rules: [discover, { method: "tools/list", cursor: null, result: page(["a"], { nextCursor: "p2", ttlMs: 60000 }) }, { method: "tools/list", cursor: "p2", result: page(["b"], { ttlMs: 300 }) }],
|
||||
script: [...START, "tools", "expect catalog-fresh", "expect catalog-stale 3000", "tools", "expect catalog-fresh", ...STOP],
|
||||
check: (lines, _, log) => same(events(lines), ["publish a,b", "publish a,b"], "catalog results") ?? same(received(log).filter((m) => m.startsWith("tools/list")).length, 4, "tools/list requests"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-list-changed",
|
||||
description: "notifications/tools/list_changed makes a fresh list stale at once (CAT-05)",
|
||||
rules: [discover, { method: "tools/list", result: page(["a"], { ttlMs: 60000 }) }, ping([LIST_CHANGED])],
|
||||
script: [...START, "tools", "expect catalog-fresh", "request ping", "outcome 3 result", "expect catalog-stale", "tools", "expect catalog-fresh", ...STOP],
|
||||
check: (lines, _, log) => same(events(lines), ["publish a", "publish a"], "catalog results") ?? same(received(log), ["server/discover", "tools/list", "ping", "tools/list"], "requests"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-changed-during-listing",
|
||||
description: "A list_changed that arrives while a listing runs keeps its answer out of the cache",
|
||||
rules: [discover, { method: "tools/list", times: 1, before: [LIST_CHANGED], result: page(["old"], { ttlMs: 60000 }) }, { method: "tools/list", result: page(["new"], { ttlMs: 60000 }) }],
|
||||
script: [...START, "tools", "expect catalog-idle", "tools", "expect catalog-fresh", "tools", ...STOP],
|
||||
check: (lines) => same(events(lines), ["publish (not cached) old", "publish new", "serve new"], "catalog results"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-legacy-cached",
|
||||
description: "A 2025 list has no TTL but stays fresh until list_changed",
|
||||
rules: [initialize, { method: "tools/list", result: page(["a"]) }, ping([LIST_CHANGED])],
|
||||
script: [...START_LEGACY, "tools", "expect catalog-fresh", "tools", "request ping", "outcome 4 result", "expect catalog-stale", "tools", "expect catalog-fresh", ...STOP],
|
||||
check: (lines, _, log) =>
|
||||
same(events(lines), ["publish a", "serve a", "publish a"], "catalog results") ?? same(received(log), ["server/discover", "initialize", "tools/list", "ping", "tools/list"], "requests"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-endless-cursor",
|
||||
description: "A server that always sends the same cursor is stopped at the page limit",
|
||||
maxPages: 5,
|
||||
rules: [discover, { method: "tools/list", result: page(["t"], { nextCursor: "same", ttlMs: 60000 }) }],
|
||||
script: [...START, "tools", "expect catalog-idle", ...STOP],
|
||||
check: (lines, final, log) =>
|
||||
same(events(lines), ["publish t"], "catalog results") ?? same(received(log).filter((m) => m.startsWith("tools/list")).length, 5, "tools/list requests") ?? expectFinal(final, { tools: 1 }),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-invalid-cursor-restart",
|
||||
description: "-32602 for a continuation page drops the pages so far and lists again from the beginning (CAT-02)",
|
||||
rules: [
|
||||
discover,
|
||||
{ method: "tools/list", cursor: null, result: page(["a"], { nextCursor: "c1", ttlMs: 60000 }) },
|
||||
{ method: "tools/list", cursor: "c1", times: 1, error: { code: -32602, message: "Invalid cursor" } },
|
||||
{ method: "tools/list", cursor: "c1", result: page(["b"], { ttlMs: 60000 }) },
|
||||
],
|
||||
script: [...START, "tools", "expect catalog-fresh", ...STOP],
|
||||
check: (lines, _, log) => same(events(lines), ["publish a,b"], "catalog results") ?? same(received(log), ["server/discover", "tools/list", 'tools/list "c1"', "tools/list", 'tools/list "c1"'], "requests"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-invalid-cursor-twice",
|
||||
description: "A cursor that is invalid again after the restart fails the listing",
|
||||
rules: [discover, { method: "tools/list", cursor: null, result: page(["a"], { nextCursor: "c1" }) }, { method: "tools/list", cursor: "c1", error: { code: -32602, message: "Invalid cursor" } }],
|
||||
script: [...START, "tools", "expect catalog-idle", ...STOP],
|
||||
check: (lines, final, log) =>
|
||||
same(events(lines), ["fail"], "catalog results") ??
|
||||
same(received(log), ["server/discover", "tools/list", 'tools/list "c1"', "tools/list", 'tools/list "c1"'], "requests") ??
|
||||
expectFinal(final, { catalog: "none", tools: 0 }),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-page-error",
|
||||
description: "A page error fails the listing and caches nothing; the next need lists again",
|
||||
rules: [discover, { method: "tools/list", error: { code: -32603, message: "Internal error" } }],
|
||||
script: [...START, "tools", "expect catalog-idle", "tools", "expect catalog-idle", ...STOP],
|
||||
check: (lines, _, log) => same(events(lines), ["fail", "fail"], "catalog results") ?? same(received(log), ["server/discover", "tools/list", "tools/list"], "requests"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-needs-join",
|
||||
description: "A need while a listing runs joins it instead of starting another",
|
||||
rules: [discover, { method: "tools/list", delayMs: 300, result: page(["a"], { ttlMs: 60000 }) }],
|
||||
script: [...START, "tools", "tools", "expect catalog-fresh", ...STOP],
|
||||
check: (lines, _, log) => same(events(lines), ["publish a"], "catalog results") ?? same(received(log), ["server/discover", "tools/list"], "requests"),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-call-answers",
|
||||
description: "Tool call answers: an isError result and structured content go to the model, input_required and malformed results are recognized, and an unknown-tool error makes the list stale (CAT-08)",
|
||||
rules: [
|
||||
discover,
|
||||
{ method: "tools/list", result: page(["echo"], { ttlMs: 60000 }) },
|
||||
{ method: "tools/call", times: 1, result: { content: [{ type: "text", text: "bad date" }], structuredContent: [1, 2], isError: true } },
|
||||
{ method: "tools/call", times: 1, result: { resultType: "input_required", inputRequests: {} } },
|
||||
{ method: "tools/call", times: 1, result: { structuredContent: {} } },
|
||||
{ method: "tools/call", times: 1, error: { code: -32602, message: "Unknown tool: nope" } },
|
||||
],
|
||||
script: [
|
||||
...START,
|
||||
"tools",
|
||||
"expect catalog-fresh",
|
||||
'call echo {"text":"hi"}',
|
||||
"outcome 3 result",
|
||||
"call echo",
|
||||
"outcome 4 result",
|
||||
"call echo",
|
||||
"outcome 5 result",
|
||||
"call nope",
|
||||
"outcome 6 peer_error",
|
||||
"expect catalog-stale",
|
||||
"tools",
|
||||
"expect catalog-fresh",
|
||||
...STOP,
|
||||
],
|
||||
check: (lines, _, log) => {
|
||||
const got = calls(lines).map((c) => [c.kind, c.is_error ?? null, c.structured ?? null, c.code ?? null]);
|
||||
const problem = same(got, [["result", true, "[1,2]", null], ["input_required", null, null, null], ["invalid", null, null, null], ["failure", null, null, -32602]], "call answers");
|
||||
return problem ?? same(received(log), ["server/discover", "tools/list", "tools/call", "tools/call", "tools/call", "tools/call", "tools/list"], "requests");
|
||||
},
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-new-process-relists",
|
||||
description: "A new server process makes the list stale, so it is listed again after detection",
|
||||
rules: [discover, { method: "tools/list", result: page(["a"], { ttlMs: 60000 }) }, { method: "tools/call", result: { content: [] }, then: "crash" }],
|
||||
labFlags: ["--backoff-ms", "20"],
|
||||
script: [...START, "tools", "expect catalog-fresh", "call a", "outcome 3 result", "expect idle", "start", "expect era-modern", "expect catalog-stale", "tools", "expect catalog-fresh", ...STOP],
|
||||
check: (lines, final, log) =>
|
||||
same(events(lines), ["publish a", "publish a"], "catalog results") ??
|
||||
same(received(log), ["server/discover", "tools/list", "tools/call", "server/discover", "tools/list"], "requests") ??
|
||||
expectFinal(final, { generation: 2, catalog: "fresh" }),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-listing-lost",
|
||||
description: "A server that dies during a listing fails it; nothing is cached",
|
||||
rules: [discover, { method: "tools/list", noReply: true, then: "crash" }],
|
||||
labFlags: ["--backoff-ms", "20"],
|
||||
script: [...START, "tools", "expect catalog-idle", "stop", "expect stopped 5000"],
|
||||
check: (lines, final) => same(events(lines), ["fail"], "catalog results") ?? expectFinal(final, { catalog: "none" }),
|
||||
}),
|
||||
catalogScenario({
|
||||
name: "u05-planted-page-limit-mismatch",
|
||||
description: "Known-bad control: the lab stops at four pages but the model is told three, so the fourth page can't be matched",
|
||||
maxPages: 4,
|
||||
modelPages: 3,
|
||||
catalogVerdict: "REJECTED",
|
||||
rules: [discover, { method: "tools/list", result: page(["t"], { nextCursor: "same" }) }],
|
||||
script: [...START, "tools", "expect catalog-idle", ...STOP],
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,90 @@
|
||||
// U10 subscriptions over stdio, step 6: the stdio lab's `listen` and
|
||||
// `unlisten` drive src/mcp/core/subscription.zig against the fake stdio
|
||||
// server, whose listen rules acknowledge and never answer. Every listen
|
||||
// shares the one channel, so notifications are told apart only by their
|
||||
// subscriptionId (SUBS-02). Detection takes request id 1, so listens get 2,
|
||||
// 3, and so on, and later rules can name them.
|
||||
|
||||
import type { Rule } from "../fakes/stdio";
|
||||
import { labScenario, type LabCase, type Seen } from "../lab";
|
||||
import type { Scenario } from "../scenario";
|
||||
|
||||
const MODERN = "2026-07-28";
|
||||
const sid = "io.modelcontextprotocol/subscriptionId";
|
||||
const discover: Rule = { method: "server/discover", result: { resultType: "complete", supportedVersions: [MODERN], capabilities: { tools: { listChanged: true } } } };
|
||||
const ack = (notifications: object) => ({ jsonrpc: "2.0", method: "notifications/subscriptions/acknowledged", params: { _meta: { [sid]: "$id" }, notifications } });
|
||||
const changed = (what: string, id: number) => ({ jsonrpc: "2.0", method: `notifications/${what}/list_changed`, params: { _meta: { [sid]: id } } });
|
||||
const listen = (notifications: object, times?: number): Rule => ({ method: "subscriptions/listen", before: [ack(notifications)], noReply: true, ...(times ? { times } : {}) });
|
||||
|
||||
const START = ["start", "detect", "expect era-modern"];
|
||||
const subs = (lines: Seen[]) => lines.filter((l) => l.sub).map((l) => l.sub!);
|
||||
const sent = (log: { dir: string; line: string }[], method: string) =>
|
||||
log.filter((l) => l.dir === "in").map((l) => JSON.parse(l.line)).filter((m) => m.method === method);
|
||||
|
||||
function stdioScenario(c: Omit<LabCase, "unit">): Scenario {
|
||||
return labScenario({
|
||||
unit: "U10",
|
||||
requestTrace: true,
|
||||
era: { preferred: 1 },
|
||||
subscription: {},
|
||||
labFlags: ["--preferred", MODERN, "--probe-ms", "3000", "--backoff-ms", "200"],
|
||||
...c,
|
||||
});
|
||||
}
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
stdioScenario({
|
||||
name: "u10-stdio-demux",
|
||||
description: "Two subscriptions on one channel: changes for both, interleaved, reach only the subscription whose id they carry and only what it asked for (SUBS-02)",
|
||||
rules: [
|
||||
discover,
|
||||
listen({ toolsListChanged: true }, 1),
|
||||
listen({ promptsListChanged: true }, 1),
|
||||
{ method: "ping", before: [changed("tools", 2), changed("prompts", 3), changed("tools", 3), changed("prompts", 2), changed("tools", 9)], result: {} },
|
||||
],
|
||||
script: [...START, "listen 0 1", "listen 1 2", "drain 300", "request ping", "drain 300", "stop", "expect stopped 5000"],
|
||||
check(lines) {
|
||||
const delivered = subs(lines).filter((s) => s.deliver).map((s) => [s.sub, s.deliver]);
|
||||
const acked = subs(lines).filter((s) => s.acknowledged !== undefined).map((s) => [s.sub, s.acknowledged]);
|
||||
if (JSON.stringify(acked) !== JSON.stringify([[0, 1], [1, 2]])) return `acknowledgments were ${JSON.stringify(acked)}`;
|
||||
return JSON.stringify(delivered) === JSON.stringify([[0, "tools"], [1, "prompts"]]) ? null : `deliveries were ${JSON.stringify(delivered)}`;
|
||||
},
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u10-stdio-cancel",
|
||||
description: "Cancelling a subscription on stdio sends notifications/cancelled naming its listen (SUBS-04)",
|
||||
rules: [discover, listen({ toolsListChanged: true })],
|
||||
script: [...START, "listen 0 1", "drain 300", "unlisten 0", "drain 300", "stop", "expect stopped 5000"],
|
||||
check(lines, _final, log) {
|
||||
const cancels = sent(log, "notifications/cancelled").map((m) => m.params?.requestId);
|
||||
const [l] = sent(log, "subscriptions/listen");
|
||||
if (JSON.stringify(cancels) !== JSON.stringify([l?.id])) return `cancels named ${JSON.stringify(cancels)}, the listen was ${l?.id}`;
|
||||
return subs(lines).some((s) => s.ended) ? "the host's own cancel was reported as an end" : null;
|
||||
},
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u10-stdio-restart",
|
||||
description: "The server process crashes: once the new process is detected, the listen goes out again with a new id and is acknowledged again (SUBS-05)",
|
||||
rules: [discover, listen({ toolsListChanged: true }), { method: "crash", result: {}, then: "crash" }],
|
||||
script: [...START, "listen 0 1", "drain 300", "request crash", "drain 3000", "stop", "expect stopped 5000"],
|
||||
timeoutMs: 30_000,
|
||||
check(lines, _final, log) {
|
||||
const listens = sent(log, "subscriptions/listen").map((m) => m.id);
|
||||
if (listens.length !== 2 || !(listens[1] > listens[0])) return `listens went out with ids ${JSON.stringify(listens)}`;
|
||||
const acked = subs(lines).filter((s) => s.acknowledged !== undefined).length;
|
||||
if (acked !== 2) return `acknowledged ${acked} times`;
|
||||
return subs(lines).some((s) => s.ended) ? "the subscription ended" : null;
|
||||
},
|
||||
}),
|
||||
stdioScenario({
|
||||
name: "u10-stdio-server-cancel",
|
||||
description: "The server's notifications/cancelled for the listen ends the subscription, and it isn't listened to again (SUBS-03)",
|
||||
rules: [discover, listen({ toolsListChanged: true }), { method: "bye", before: [{ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: 2 } }], result: {} }],
|
||||
script: [...START, "listen 0 1", "drain 300", "request bye", "drain 500", "stop", "expect stopped 5000"],
|
||||
check(lines, _final, log) {
|
||||
const ended = subs(lines).filter((s) => s.ended).map((s) => s.ended);
|
||||
if (JSON.stringify(ended) !== JSON.stringify(["server_cancelled"])) return `ends were ${JSON.stringify(ended)}`;
|
||||
return sent(log, "subscriptions/listen").length === 1 ? null : "the listen went out again";
|
||||
},
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,507 @@
|
||||
// U12 engine, step 6: tests/tla/drivers/mcp/labs/engine_lab.zig drives src/mcp/engine.zig
|
||||
// over fake stdio servers (tests/tla/drivers/mcp/fakes/stdio.ts), one per name. Every
|
||||
// server's machines trace into one file: the engine gate under the server's
|
||||
// name, and its stdio core, request table, era, catalog, subscriptions,
|
||||
// rounds, and elicitations under "<name>.". Request ids and logical handles
|
||||
// count per server: detection is x1, the tool list listen x2, and calls
|
||||
// follow, so a call's rounds are the instance after them.
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { HttpRule } from "../fakes/http";
|
||||
import type { Rule } from "../fakes/stdio";
|
||||
import type { Scenario, ScenarioContext, TraceExpectation } from "../scenario";
|
||||
|
||||
const FAKE = join(import.meta.dir, "../fakes/stdio.ts");
|
||||
const FAKE_HTTP = join(import.meta.dir, "../fakes/http.ts");
|
||||
const WITH_SERVER = join(import.meta.dir, "../with-server.ts");
|
||||
const MODERN = "2026-07-28";
|
||||
const LEGACY = "2025-11-25";
|
||||
|
||||
// ---- fake servers ----
|
||||
|
||||
const discover: Rule = { method: "server/discover", result: { resultType: "complete", supportedVersions: [MODERN], capabilities: { tools: { listChanged: true } } } };
|
||||
const listen: Rule = { method: "subscriptions/listen", noReply: true };
|
||||
const tool = (name: string) => ({ name, inputSchema: { type: "object" } });
|
||||
const list = (...names: string[]): Rule => ({ method: "tools/list", result: { resultType: "complete", tools: names.map(tool) } });
|
||||
const ok = (text: string, extra: Partial<Rule> = {}): Rule => ({ method: "tools/call", result: { resultType: "complete", content: [{ type: "text", text }] }, ...extra });
|
||||
const initialize: Rule = { method: "initialize", result: { protocolVersion: LEGACY, capabilities: { tools: {} }, serverInfo: { name: "fake", version: "1" } } };
|
||||
const list25 = (...names: string[]): Rule => ({ method: "tools/list", result: { tools: names.map(tool) } });
|
||||
const ok25 = (text: string, extra: Partial<Rule> = {}): Rule => ({ method: "tools/call", result: { content: [{ type: "text", text }] }, ...extra });
|
||||
const crash = (name: string): Rule => ({ method: "tools/call", tool: name, noReply: true, then: "crash" });
|
||||
export const modern = (...extra: Rule[]): Rule[] => [discover, listen, list("echo"), ...extra, ok("fine")];
|
||||
export const legacy = (...extra: Rule[]): Rule[] => [initialize, list25("add"), ...extra, ok25("fine")];
|
||||
|
||||
// ---- the lab ----
|
||||
|
||||
interface Line {
|
||||
did?: { action?: string; expect?: string; met?: boolean; error?: string; line?: string; k?: number };
|
||||
allow?: { server: string; tool: string; ok: boolean };
|
||||
event?: { tag: string; maybe_ran?: boolean; answer?: string | null; names?: string[]; k?: number };
|
||||
status?: { name: string; phase: string; version: string | null; tools: number; last_error: string | null }[];
|
||||
final?: { ok: boolean; logged: { name: string; lines: number }[] };
|
||||
}
|
||||
type Log = { dir: string; line: string }[];
|
||||
type Machine = "engine" | "stdio" | "request" | "era" | "catalog" | "subscription" | "elicitation" | { mrtr: string };
|
||||
|
||||
interface EngineCase {
|
||||
name: string;
|
||||
description: string;
|
||||
/** Each a fake stdio server with these rules, or `argv` run as it is. `{caseDir}` in a flag is the case's directory. */
|
||||
servers: { [name: string]: { rules: Rule[]; flags?: string[]; argv?: string[] } };
|
||||
/** One server on the fake HTTP server, declared first, as "server <name> http {URL}". */
|
||||
http?: { name: string; rules: HttpRule[]; flags?: string[] };
|
||||
/** Errors script steps are meant to hit, in order, such as a refused answer. */
|
||||
errors?: string[];
|
||||
script: string[];
|
||||
labFlags?: string[];
|
||||
/** The engine's tries in a row. */
|
||||
maxAttempts?: number;
|
||||
/** The machines to check per server; a server left out is checked for none. */
|
||||
machines: { [name: string]: Machine[] };
|
||||
/** A planted control: the attempts the engine trace's model is given, and the verdict expected. */
|
||||
control?: { server: string; maxAttempts: number };
|
||||
check?: (lines: Line[], logs: { [name: string]: Log }) => string | null;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
const BASE: Machine[] = ["engine", "stdio", "request", "era"];
|
||||
/** server/discover's answer for a stdio server written in shell. */
|
||||
const STUCK_DISCOVER = JSON.stringify({ jsonrpc: "2.0", id: 1, result: { resultType: "complete", supportedVersions: [MODERN], capabilities: { tools: {} } } });
|
||||
|
||||
function engineScenario(c: EngineCase): Scenario {
|
||||
let caseDir = "";
|
||||
const attempts = c.maxAttempts ?? 8;
|
||||
const traces: TraceExpectation[] = [];
|
||||
for (const [server, machines] of Object.entries(c.machines)) {
|
||||
for (const m of machines) {
|
||||
const remote = `${server}.remote`;
|
||||
if (m === "engine") {
|
||||
const control = c.control?.server === server ? c.control : undefined;
|
||||
traces.push({ machine: "engine", instance: server, specDir: "tests/tla/mcp", wrapper: "EngineTrace", verdict: control ? "REJECTED" : "ACCEPTED", constants: { ObsMaxAttempts: control?.maxAttempts ?? attempts } });
|
||||
} else if (m === "stdio") traces.push({ machine: "stdio", instance: `${server}.stdio`, specDir: "tests/tla/mcp", wrapper: "StdioConnTrace", verdict: "ACCEPTED", constants: { ObsMaxAttempts: 255 } });
|
||||
else if (m === "request") traces.push({ machine: "request", instance: remote, specDir: "tests/tla/mcp", wrapper: "RequestTrace", verdict: "ACCEPTED" });
|
||||
else if (m === "era") traces.push({ machine: "era", instance: remote, specDir: "tests/tla/mcp", wrapper: "EraTrace", verdict: "ACCEPTED", constants: { ObsPreferredCount: 1 } });
|
||||
else if (m === "catalog") traces.push({ machine: "catalog", instance: remote, specDir: "tests/tla/mcp", wrapper: "CatalogTrace", verdict: "ACCEPTED", constants: { ObsMaxPages: 100 } });
|
||||
else if (m === "subscription") traces.push({ machine: "subscription", instance: remote, specDir: "tests/tla/mcp", wrapper: "SubscriptionTrace", verdict: "ACCEPTED", constants: { ObsMaxTries: 8 } });
|
||||
else if (m === "elicitation") traces.push({ machine: "elicitation", instance: remote, specDir: "tests/tla/mcp", wrapper: "ElicitationTrace", verdict: "ACCEPTED", constants: { ObsMaxPending: 16 } });
|
||||
else traces.push({ machine: "mrtr", instance: `${server}.${m.mrtr}`, specDir: "tests/tla/mcp", wrapper: "MrtrTrace", verdict: "ACCEPTED", constants: { ObsMaxRequests: 10, ObsMaxKeys: 16 } });
|
||||
}
|
||||
}
|
||||
return {
|
||||
name: c.name,
|
||||
unit: "U12",
|
||||
description: c.description,
|
||||
needs: ["engine_lab"],
|
||||
setup: (ctx: ScenarioContext) => {
|
||||
caseDir = ctx.caseDir;
|
||||
const declared = Object.entries(c.servers).map(([name, s]) => {
|
||||
const rules = join(ctx.caseDir, `${name}.rules.json`);
|
||||
writeFileSync(rules, JSON.stringify(s.rules));
|
||||
const flags = (s.flags ?? []).map((f) => f.replaceAll("{caseDir}", ctx.caseDir));
|
||||
const argv = s.argv ?? [process.execPath, FAKE, "--script", rules, "--log", join(ctx.caseDir, `${name}.jsonl`), ...flags];
|
||||
return `server ${name} stdio ${JSON.stringify(argv)}`;
|
||||
});
|
||||
if (c.http) {
|
||||
writeFileSync(join(ctx.caseDir, "http.rules.json"), JSON.stringify(c.http.rules));
|
||||
declared.unshift(`server ${c.http.name} http {URL}`);
|
||||
}
|
||||
writeFileSync(join(ctx.caseDir, "script.txt"), [...declared, ...c.script].join("\n") + "\n");
|
||||
},
|
||||
command: (ctx: ScenarioContext) => {
|
||||
const lab = [ctx.bin.engine_lab, ctx.tracePath, join(ctx.caseDir, "script.txt"), "--probe-ms", "3000", "--max-attempts", String(attempts), ...(c.labFlags ?? [])];
|
||||
if (!c.http) return lab;
|
||||
const server = [process.execPath, FAKE_HTTP, "--script", join(ctx.caseDir, "http.rules.json"), "--log", join(ctx.caseDir, `${c.http.name}.jsonl`), ...(c.http.flags ?? [])];
|
||||
// Bun takes a leading "--" as its own, so an --env comes first.
|
||||
return [process.execPath, WITH_SERVER, "--env", "MCPV2_ENGINE=1", "--", ...server, "--", ...lab, "--define", "URL=http://127.0.0.1:{port}/mcp"];
|
||||
},
|
||||
expect: {
|
||||
exit: 0,
|
||||
traces,
|
||||
stdout: (text) => {
|
||||
const lines = text.split("\n").filter(Boolean).map((l) => JSON.parse(l) as Line);
|
||||
const errors = [...(c.errors ?? [])];
|
||||
const unmet = lines.find((l) => l.did?.met === false || (l.did?.error !== undefined && l.did.error !== errors.shift()));
|
||||
if (unmet) return `script step failed: ${JSON.stringify(unmet.did)}`;
|
||||
if (errors.length > 0) return `errors that never came: ${errors.join(", ")}`;
|
||||
if (!lines.at(-1)?.final?.ok) return "no final line, or the script didn't finish";
|
||||
const logs: { [name: string]: Log } = {};
|
||||
for (const name of [...Object.keys(c.servers), ...(c.http ? [c.http.name] : [])]) {
|
||||
const path = join(caseDir, `${name}.jsonl`);
|
||||
logs[name] = existsSync(path) ? readFileSync(path, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) : [];
|
||||
}
|
||||
return c.check ? c.check(lines, logs) : null;
|
||||
},
|
||||
},
|
||||
timeoutMs: c.timeoutMs ?? 60_000,
|
||||
};
|
||||
}
|
||||
|
||||
const tags = (lines: Line[]) => lines.filter((l) => l.event).map((l) => l.event!.tag);
|
||||
const event = (lines: Line[], prefix: string) => lines.find((l) => l.event?.tag.startsWith(prefix))?.event;
|
||||
const lastStatus = (lines: Line[]) => lines.filter((l) => l.status).at(-1)!.status!;
|
||||
const status = (lines: Line[], name: string) => lastStatus(lines).find((s) => s.name === name);
|
||||
const received = (log: Log, method: string) => log.filter((l) => l.dir === "in").map((l) => JSON.parse(l.line)).filter((m) => m.method === method);
|
||||
/** What the fake HTTP server received, by JSON-RPC method. */
|
||||
const posted = (log: Log, method: string) =>
|
||||
(log as unknown as { dir: string; body?: string }[]).filter((l) => l.dir === "in" && l.body).map((l) => JSON.parse(l.body!)).filter((m) => m.method === method);
|
||||
const replies = (log: Log) => log.filter((l) => l.dir === "in").map((l) => JSON.parse(l.line)).filter((m) => m.method === undefined);
|
||||
const first = (...checks: (string | null)[]) => checks.find((x) => x !== null) ?? null;
|
||||
const expectThat = (ok: boolean, what: string) => (ok ? null : what);
|
||||
const same = (got: unknown, want: unknown, what: string) => (JSON.stringify(got) === JSON.stringify(want) ? null : `${what} were ${JSON.stringify(got)}, expected ${JSON.stringify(want)}`);
|
||||
|
||||
const form = { mode: "form", message: "Name?", requestedSchema: { type: "object", properties: { name: { type: "string" } }, required: ["name"] } };
|
||||
const askName = { k: { method: "elicitation/create", params: form } };
|
||||
const FORM = ["--capabilities", JSON.stringify({ elicitation: { form: {} } })];
|
||||
|
||||
export const scenarios: Scenario[] = [
|
||||
engineScenario({
|
||||
name: "u12-two-servers",
|
||||
description: "Two stdio servers at once, a 2026 one and a 2025 one: each starts for its first call, both answer, both list their tools, and the 2025 one's stderr reaches the host's sink",
|
||||
servers: { a: { rules: modern() }, b: { rules: legacy({ ...ok25("from b"), stderr: ["b says hi"] }) } },
|
||||
script: ["status", 'call a echo {"x":1}', "call b add", "expect a.called.1.result", "expect b.called.2.result", "tools a", "expect a.tools.ok", "tools b", "expect b.tools.ok"],
|
||||
machines: { a: [...BASE, "catalog", "subscription"], b: [...BASE, "catalog"] },
|
||||
check(lines) {
|
||||
const before = lines.find((l) => l.status)!.status!;
|
||||
return first(
|
||||
same(before.map((s) => s.phase), ["idle", "idle"], "phases before anything"),
|
||||
same([status(lines, "a")?.version, status(lines, "b")?.version], [MODERN, LEGACY], "versions"),
|
||||
same([status(lines, "a")?.tools, status(lines, "b")?.tools], [1, 1], "tool counts"),
|
||||
same(event(lines, "b.called.2")?.answer, '[{"type":"text","text":"from b"}]', "b's answer"),
|
||||
same(lines.at(-1)!.final!.logged, [{ name: "a", lines: 0 }, { name: "b", lines: 1 }], "stderr lines logged"),
|
||||
);
|
||||
},
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-lazy",
|
||||
description: "A server nothing needs never starts: listing one server's tools leaves the other idle, with no process (LazyConnect)",
|
||||
servers: { a: { rules: modern() }, b: { rules: modern() } },
|
||||
script: ["tools a", "expect a.tools.ok", "drain 300"],
|
||||
machines: { a: [...BASE, "catalog", "subscription"] },
|
||||
check: (lines, logs) => first(same(status(lines, "b")?.phase, "idle", "b's phase"), expectThat(logs.b.length === 0, "b's process saw something"), expectThat(!tags(lines).some((t) => t.startsWith("b.")), "b said something")),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-denied",
|
||||
description: "A call the host refuses ends denied at once, and its server never starts for it (SEC-01)",
|
||||
servers: { a: { rules: modern() } },
|
||||
labFlags: ["--deny", "rm"],
|
||||
script: ['call a rm {"path":"/"}', "expect a.called.1.denied 2000", "drain 300"],
|
||||
machines: { a: ["engine"] },
|
||||
check: (lines, logs) =>
|
||||
first(
|
||||
same(lines.filter((l) => l.allow).map((l) => l.allow), [{ server: "a", tool: "rm", ok: false }], "permission asks"),
|
||||
same(status(lines, "a")?.phase, "idle", "a's phase"),
|
||||
expectThat(logs.a.length === 0, "a's process started"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-one-failing",
|
||||
description: "A server whose process dies at every start gives up after its tries in a row and ends what waits transport_failed, and a later call at once; another server is untouched",
|
||||
servers: { a: { rules: modern(), flags: ["--exit-at-start", "1"] }, b: { rules: modern() } },
|
||||
maxAttempts: 2,
|
||||
script: ["call a echo", "call b echo", "expect b.called.2.result", "expect a.called.1.transport_failed 15000", "call a echo", "expect a.called.3.transport_failed 2000"],
|
||||
machines: { a: ["engine", "stdio"], b: [...BASE, "subscription"] },
|
||||
check: (lines) =>
|
||||
first(
|
||||
same([status(lines, "a")?.phase, status(lines, "a")?.last_error], ["failed", "process"], "a's status"),
|
||||
same(status(lines, "b")?.phase, "ready", "b's phase"),
|
||||
same(tags(lines).filter((t) => t.startsWith("a.phase")), ["a.phase.connecting", "a.phase.backoff", "a.phase.connecting", "a.phase.failed"], "a's phases"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-reconnect",
|
||||
description: "Reconnect drops a ready server's process and starts a new one, which detects again and serves the next call",
|
||||
servers: { a: { rules: modern() } },
|
||||
script: ["tools a", "expect a.phase.ready", "expect a.tools.ok", "reconnect a", "expect a.phase.idle", "expect a.phase.ready 10000", "call a echo", "expect a.called.1.result"],
|
||||
machines: { a: [...BASE, "catalog", "subscription"] },
|
||||
check: (lines, logs) => first(same(received(logs.a, "server/discover").length, 2, "detections"), expectThat(tags(lines).includes("a.phase.idle"), "no idle phase between")),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-crash-lost",
|
||||
description: "A process that dies during a call loses it, maybe run; the next call waits out the backoff and starts the server again",
|
||||
servers: { a: { rules: modern(crash("boom")) } },
|
||||
script: ["call a boom", "expect a.called.1.lost 10000", "call a echo", "expect a.called.2.result 15000"],
|
||||
machines: { a: [...BASE, "subscription"] },
|
||||
check: (lines, logs) => first(same(event(lines, "a.called.1")?.maybe_ran, true, "maybe_ran"), same(received(logs.a, "server/discover").length, 2, "processes")),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-crash-forgets",
|
||||
description: "A process exit also ends what it had in flight in the request table (U02): once the request timeout has passed, the new process hears no cancellation for the old call, and the host no second end",
|
||||
servers: { a: { rules: modern(crash("boom")) } },
|
||||
labFlags: ["--timeout-ms", "1500"],
|
||||
script: ["call a boom", "expect a.called.1.lost 10000", "call a echo", "expect a.called.2.result 15000", "drain 2500"],
|
||||
machines: { a: [...BASE, "subscription"] },
|
||||
check: (lines, logs) =>
|
||||
first(
|
||||
same(received(logs.a, "notifications/cancelled").length, 0, "cancellations sent"),
|
||||
same(lines.filter((l) => l.event?.tag.startsWith("a.called.1.")).length, 1, "ends of call 1"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-spawn-fails",
|
||||
description: "A command that can't be started fails within its tries instead of looking busy: each failed spawn is a down (found by mcpv2 list --connect), the waiting call ends transport_failed, and the status names the spawn error",
|
||||
servers: { a: { rules: [], argv: ["/nonexistent/mcp-server"] } },
|
||||
maxAttempts: 2,
|
||||
script: ["call a echo", "expect a.called.1.transport_failed 10000", "expect a.phase.failed 1000", "status"],
|
||||
machines: { a: ["engine", "stdio"] },
|
||||
check: (lines) => same(lines.filter((l) => l.status).at(-1)?.status?.[0]?.last_error, "FileNotFound", "last errors"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-stderr-flood",
|
||||
description: "5000 stderr lines during a call all reach the host's sink, written by the reader task, with none dropped",
|
||||
servers: { a: { rules: modern({ ...ok("done"), stderr: ["noise"], stderrRepeat: 5000 }) } },
|
||||
script: ["call a echo", "expect a.called.1.result 20000", "drain 500"],
|
||||
machines: { a: [...BASE, "subscription"] },
|
||||
check: (lines) => same(lines.at(-1)!.final!.logged, [{ name: "a", lines: 5000 }], "stderr lines logged"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-cancel-waiting",
|
||||
description: "A call cancelled while its server is still starting ends cancelled and never reaches the server",
|
||||
servers: { a: { rules: modern(), flags: ["--start-delay", "1500"] } },
|
||||
script: ["call a echo", "cancel 1", "expect a.called.1.cancelled 2000", "expect a.phase.ready 10000", "drain 300"],
|
||||
machines: { a: [...BASE, "subscription"] },
|
||||
check: (_lines, logs) => expectThat(received(logs.a, "tools/call").length === 0, "the call reached the server"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-input-stdio",
|
||||
description: "A 2026 call over stdio that comes back input_required asks the host, and the next round carries its answer under the server's key (MRTR-02, MRTR-03)",
|
||||
servers: { a: { rules: modern({ method: "tools/call", times: 1, result: { resultType: "input_required", inputRequests: askName } }) } },
|
||||
labFlags: FORM,
|
||||
script: ["call a ask", "expect a.input.1", 'respond 1 [{"action":"accept","content":{"name":"Ada"}}]', "expect a.called.1.result"],
|
||||
machines: { a: [...BASE, "subscription", { mrtr: "x3" }] },
|
||||
check: (_lines, logs) => {
|
||||
const calls = received(logs.a, "tools/call");
|
||||
return first(same(calls.length, 2, "tools/call requests"), same(calls[1]?.params?.inputResponses, { k: { action: "accept", content: { name: "Ada" } } }, "the second round's answers"));
|
||||
},
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-elicit-stdio",
|
||||
description: "A 2025 server's elicitation/create over stdio reaches the host, whose answer goes back on stdin as a response (ELIC-09)",
|
||||
servers: { b: { rules: legacy({ ...ok25("asked"), before: [{ jsonrpc: "2.0", id: "e1", method: "elicitation/create", params: form }] }) } },
|
||||
labFlags: FORM,
|
||||
script: ["call b go", "expect b.elicit 5000", 'elicit b 1 accept {"name":"Ada"}', "expect b.called.1.result 5000", "drain 300"],
|
||||
machines: { b: [...BASE, "elicitation"] },
|
||||
check: (_lines, logs) => same(replies(logs.b).find((m) => m.id === "e1")?.result, { action: "accept", content: { name: "Ada" } }, "the elicitation's answer"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-ping-during-initialize",
|
||||
description: "A 2025 stdio server that pings before it answers initialize gets its pong: the connection takes a response before detection ends (STDIO-09, CORE-11)",
|
||||
servers: { b: { rules: [{ ...initialize, before: [{ jsonrpc: "2.0", id: "p1", method: "ping" }] }, list25("add"), ok25("fine")] } },
|
||||
script: ["tools b", "expect b.tools.ok", "drain 300"],
|
||||
machines: { b: BASE },
|
||||
check: (_lines, logs) => same(replies(logs.b).find((m) => m.id === "p1")?.result, {}, "the ping's answer"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-ping-after-initialize",
|
||||
description: "Control: a 2025 stdio server's ping once it's ready gets its pong (STDIO-09)",
|
||||
servers: { b: { rules: [initialize, { ...list25("add"), before: [{ jsonrpc: "2.0", id: "p2", method: "ping" }] }, ok25("fine")] } },
|
||||
script: ["tools b", "expect b.tools.ok", "drain 300"],
|
||||
machines: { b: BASE },
|
||||
check: (_lines, logs) => same(replies(logs.b).find((m) => m.id === "p2")?.result, {}, "the ping's answer"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-elicit-withdrawn",
|
||||
description: "A 2025 server withdraws its elicitation and asks another, which takes the same slot: the host's answer to the first is refused and reaches no one, and its answer to the second reaches the second",
|
||||
servers: {
|
||||
b: {
|
||||
rules: legacy({
|
||||
...ok25("asked"),
|
||||
before: [
|
||||
{ jsonrpc: "2.0", id: "e1", method: "elicitation/create", params: form },
|
||||
{ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: "e1", reason: "asked again" } },
|
||||
{ jsonrpc: "2.0", id: "e2", method: "elicitation/create", params: form },
|
||||
],
|
||||
}),
|
||||
},
|
||||
},
|
||||
labFlags: FORM,
|
||||
errors: ["UnknownElicitation"],
|
||||
script: ["call b go", "expect b.elicit 5000", "expect b.elicit_withdrawn 5000", "expect b.elicit 5000", 'elicit b 1 accept {"name":"for-1"}', 'elicit b 2 accept {"name":"for-2"}', "expect b.called.1.result 5000", "drain 300"],
|
||||
machines: { b: [...BASE, "elicitation"] },
|
||||
check: (lines, logs) =>
|
||||
first(
|
||||
same(replies(logs.b).filter((m) => m.id === "e1" || m.id === "e2"), [{ jsonrpc: "2.0", id: "e2", result: { action: "accept", content: { name: "for-2" } } }], "the elicitations' answers"),
|
||||
same(lines.filter((l) => l.did?.action === "elicit").map((l) => l.did?.error ?? "sent"), ["UnknownElicitation", "sent"], "the host's two answers"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-reload",
|
||||
description: "A reload without a server drops it and keeps the other running on the same process",
|
||||
servers: { a: { rules: modern() }, b: { rules: modern() } },
|
||||
script: ["tools a", "tools b", "expect a.tools.ok", "expect b.tools.ok", "reload a", "drain 300", "call a echo", "expect a.called.1.result"],
|
||||
machines: { a: [...BASE, "catalog", "subscription"], b: [...BASE, "catalog", "subscription"] },
|
||||
check: (lines, logs) => first(same(lastStatus(lines).map((s) => s.name), ["a"], "servers after the reload"), same(received(logs.a, "server/discover").length, 1, "a's detections")),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-subscription-restart",
|
||||
description: "After a crash, the tool list subscription waiting to listen again starts the server by itself once the backoff is over, and listens again (SUBS-05)",
|
||||
servers: { a: { rules: modern(crash("boom")) } },
|
||||
script: ["tools a", "expect a.phase.ready", "expect a.tools.ok", "drain 300", "call a boom", "expect a.called.1.lost 10000", "expect a.phase.ready 15000", "drain 500"],
|
||||
machines: { a: [...BASE, "catalog", "subscription"] },
|
||||
check: (_lines, logs) => first(same(received(logs.a, "server/discover").length, 2, "processes"), same(received(logs.a, "subscriptions/listen").length, 2, "listens")),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-http-and-stdio",
|
||||
description: "An HTTP server and a stdio server behind one engine: both start for their calls and answer; reconnecting the HTTP one drops its detection and detects again, and the stdio one keeps its process",
|
||||
http: { name: "h", rules: [{ method: "server/discover", result: { resultType: "complete", supportedVersions: [MODERN], capabilities: { tools: {} } } }, { method: "tools/list", result: { resultType: "complete", tools: [tool("echo")] } }, { method: "tools/call", result: { resultType: "complete", content: [{ type: "text", text: "from h" }] } }] },
|
||||
servers: { a: { rules: modern() } },
|
||||
labFlags: ["--no-listen"],
|
||||
script: ["call h echo", "call a echo", "expect h.called.1.result", "expect a.called.2.result", "reconnect h", "expect h.phase.idle", "expect h.phase.ready 10000", "tools h", "expect h.tools.ok", "call a echo", "expect a.called.3.result"],
|
||||
machines: { h: ["engine", "request", "era", "catalog"], a: BASE },
|
||||
check: (lines, logs) =>
|
||||
first(
|
||||
same(posted(logs.h, "server/discover").length, 2, "h's detections"),
|
||||
same(received(logs.a, "server/discover").length, 1, "a's detections"),
|
||||
same(event(lines, "h.called.1")?.answer, '[{"type":"text","text":"from h"}]', "h's answer"),
|
||||
same([status(lines, "h")?.phase, status(lines, "a")?.phase], ["ready", "ready"], "phases"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-http-initialized-timeout",
|
||||
description: "A 2025 HTTP server answers initialize but never answers notifications/initialized: after the request timeout the session fails with initialized_timed_out, and the server goes down instead of looking ready with no tools (CORE-6b)",
|
||||
http: {
|
||||
name: "h",
|
||||
flags: ["--session", "--initialized-delay", "30000"],
|
||||
rules: [
|
||||
{ method: "initialize", result: initialize.result },
|
||||
{ method: "tools/list", result: { tools: [tool("echo")] } },
|
||||
],
|
||||
},
|
||||
servers: {},
|
||||
labFlags: ["--no-listen", "--timeout-ms", "800"],
|
||||
script: ["tools h", "expect h.phase.ready 5000", "expect h.phase.backoff 5000", "status"],
|
||||
machines: { h: ["engine", "request", "era", "catalog"] },
|
||||
check: (lines, logs) => {
|
||||
const raw = logs.h as unknown as { dir: string; body?: string }[];
|
||||
return first(
|
||||
same(status(lines, "h")?.last_error, "initialized_timed_out", "h's last error"),
|
||||
raw.some((l) => l.dir === "in" && l.body?.includes('"tools/list"')) ? "tools/list went out before initialized was answered" : null,
|
||||
);
|
||||
},
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-http-reinit-timeout",
|
||||
description: "A 2025 HTTP session ends after a call, and the new initialize for the next call never answers: that call times out, the initialize times out too, its stream is closed, and the session fails; the server goes down and starts again by itself, and the third call works",
|
||||
http: {
|
||||
name: "h",
|
||||
flags: ["--session"],
|
||||
rules: [
|
||||
{ method: "initialize", result: initialize.result, times: 1 },
|
||||
{ method: "initialize", result: initialize.result, sse: true, hang: true, times: 1 },
|
||||
{ method: "initialize", result: initialize.result },
|
||||
{ method: "tools/list", result: { tools: [tool("echo")] } },
|
||||
{ method: "tools/call", result: { content: [{ type: "text", text: "first" }] }, endSession: true, times: 1 },
|
||||
{ method: "tools/call", result: { content: [{ type: "text", text: "again" }] } },
|
||||
],
|
||||
},
|
||||
servers: {},
|
||||
labFlags: ["--no-listen", "--timeout-ms", "800"],
|
||||
script: ["call h echo", "expect h.called.1.result 5000", "call h echo", "expect h.called.2.cancelled 5000", "expect h.phase.backoff 5000", "call h echo", "expect h.called.3.result 10000", "status"],
|
||||
machines: { h: ["engine", "request", "era", "catalog"] },
|
||||
check: (lines, logs) => {
|
||||
// The hung initialize's stream must close before the restart's
|
||||
// initialize, not when the lab exits.
|
||||
const raw = logs.h as unknown as { t: number; dir: string; id?: number; body?: string }[];
|
||||
const inits = raw.filter((l) => l.dir === "in" && l.body && JSON.parse(l.body).method === "initialize");
|
||||
const hung = inits[1] && JSON.parse(inits[1].body!).id;
|
||||
const closedAt = raw.find((l) => l.dir === "closed" && l.id === hung)?.t;
|
||||
return first(
|
||||
same(inits.length, 3, "initialize posts"),
|
||||
closedAt !== undefined && closedAt < inits[2].t ? null : "the second initialize's stream was not closed before the restart",
|
||||
same(event(lines, "h.called.3")?.answer, '[{"type":"text","text":"again"}]', "the third call's answer"),
|
||||
);
|
||||
},
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-stopped-reading",
|
||||
description: "A stdio server stops reading stdin and gets a call bigger than a pipe holds: only that server's stdin writer waits, so another server's call still answers and the stuck call times out",
|
||||
servers: {
|
||||
// Answers server/discover, then never reads stdin again; a Bun fake
|
||||
// keeps reading into its own buffer, so this is a shell.
|
||||
a: { rules: [], argv: ["/bin/sh", "-c", `read -r _; printf '%s\\n' '${STUCK_DISCOVER}'; exec sleep 60`] },
|
||||
b: { rules: modern() },
|
||||
},
|
||||
labFlags: ["--no-listen", "--timeout-ms", "800"],
|
||||
script: [`call a echo {"text":"${"x".repeat(300_000)}"}`, "call b echo", "expect b.called.2.result 5000", "expect a.called.1.cancelled 5000"],
|
||||
machines: { a: BASE, b: BASE },
|
||||
check: (lines) =>
|
||||
first(
|
||||
same(event(lines, "b.called.2")?.answer, '[{"type":"text","text":"fine"}]', "b's answer"),
|
||||
same(event(lines, "a.called.1")?.timed_out, true, "the stuck call timed out"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-retry-recovers",
|
||||
description: "A server that crashes once after it is up, with nothing needing it, starts again by itself once its backoff is over, and answers the next call",
|
||||
servers: { a: { rules: [{ ...discover, then: "crash" }, list("echo"), ok("fine")], flags: ["--once", "{caseDir}/a.crashed"] } },
|
||||
labFlags: ["--no-listen"],
|
||||
script: ["tools a", "expect a.phase.ready 5000", "expect a.phase.backoff 5000", "expect a.phase.ready 10000", "call a echo", "expect a.called.1.result 5000"],
|
||||
machines: { a: BASE },
|
||||
check: (_lines, logs) => same(received(logs.a, "server/discover").length, 2, "processes started"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-retry-gives-up",
|
||||
description: "A server that dies right after every start starts again by itself, with nothing needing it, and gives up after its tries in a row",
|
||||
servers: { a: { rules: [{ ...discover, then: "crash" }, list("echo")] } },
|
||||
maxAttempts: 3,
|
||||
labFlags: ["--no-listen"],
|
||||
script: ["tools a", "expect a.phase.failed 20000"],
|
||||
machines: { a: BASE },
|
||||
check: (_lines, logs) => same(received(logs.a, "server/discover").length, 3, "processes started"),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-retry-stable",
|
||||
description: "A server that stays up past --stable-ms before each crash starts its tries over each time, so it never gives up",
|
||||
servers: { a: { rules: [discover, list("echo")], flags: ["--exit-after", "800"] } },
|
||||
maxAttempts: 2,
|
||||
labFlags: ["--no-listen", "--stable-ms", "200"],
|
||||
script: ["tools a", "drain 6000", "status"],
|
||||
machines: { a: BASE },
|
||||
check: (lines, logs) =>
|
||||
first(
|
||||
same(received(logs.a, "server/discover").length >= 3, true, "at least 3 processes started"),
|
||||
same(status(lines, "a")?.phase === "failed", false, "given up"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-cancel-sent",
|
||||
description: "Cancelling a call in flight over stdio sends notifications/cancelled naming its request, and the call ends cancelled",
|
||||
servers: { a: { rules: modern({ ...ok("late"), delayMs: 2000 }) } },
|
||||
script: ["tools a", "expect a.tools.ok", "call a slow", "drain 300", "cancel 1", "expect a.called.1.cancelled 2000", "drain 2500"],
|
||||
machines: { a: [...BASE, "catalog", "subscription"] },
|
||||
check: (_lines, logs) => {
|
||||
const [call] = received(logs.a, "tools/call");
|
||||
return same(received(logs.a, "notifications/cancelled").map((m) => m.params?.requestId), [call?.id], "cancellations");
|
||||
},
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-version-switch",
|
||||
description: "A call a 2026 server rejects with -32022 after detection goes out once more with the version the error lists (VER-05)",
|
||||
servers: {
|
||||
a: {
|
||||
rules: [
|
||||
{ method: "server/discover", result: { resultType: "complete", supportedVersions: [MODERN, "2026-09-09"], capabilities: { tools: {} } } },
|
||||
{ method: "tools/call", version: MODERN, error: { code: -32022, message: "Unsupported protocol version", data: { supported: ["2026-09-09"] } } },
|
||||
{ method: "tools/call", version: "2026-09-09", result: { resultType: "complete", content: [{ type: "text", text: "switched" }] } },
|
||||
],
|
||||
},
|
||||
},
|
||||
labFlags: ["--no-listen", "--preferred", `${MODERN},2026-09-09`],
|
||||
script: ["call a echo", "expect a.called.1.result"],
|
||||
machines: { a: ["engine", "stdio", "request"] },
|
||||
check: (lines, logs) =>
|
||||
first(
|
||||
same(received(logs.a, "tools/call").map((m) => m.params?._meta?.["io.modelcontextprotocol/protocolVersion"]), [MODERN, "2026-09-09"], "the call's versions"),
|
||||
same(event(lines, "a.called.1")?.answer, '[{"type":"text","text":"switched"}]', "the answer"),
|
||||
),
|
||||
}),
|
||||
engineScenario({
|
||||
name: "u12-control-attempts",
|
||||
description: "Planted control: the engine trace of a server that gives up after 2 tries, checked against a model told 3, is rejected",
|
||||
servers: { a: { rules: modern(), flags: ["--exit-at-start", "1"] } },
|
||||
maxAttempts: 2,
|
||||
script: ["call a echo", "expect a.called.1.transport_failed 15000"],
|
||||
machines: { a: ["engine"] },
|
||||
control: { server: "a", maxAttempts: 3 },
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,83 @@
|
||||
// Runs an HTTP server and a client together, as one command for the driver
|
||||
// (U06). The server is started first, the client runs against it, and the
|
||||
// server is stopped before this exits, with the client's exit code. Nothing
|
||||
// outlives the run.
|
||||
//
|
||||
// Usage: bun tests/tla/drivers/mcp/with-server.ts [--env K=V ...] [--port <n>] -- <server argv...> -- <client argv...>
|
||||
//
|
||||
// The server either prints a JSON line {"listening": <port>} (the fake HTTP
|
||||
// server does), or, when an argument is `{port}`, gets a free port there, or
|
||||
// listens on the fixed `--port`; the last two are polled until they accept
|
||||
// connections. Client arguments may contain `{port}`.
|
||||
|
||||
function freePort(): number {
|
||||
const server = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } });
|
||||
const port = server.port;
|
||||
server.stop(true);
|
||||
return port;
|
||||
}
|
||||
|
||||
async function accepts(port: number): Promise<boolean> {
|
||||
try {
|
||||
const socket = await Bun.connect({ hostname: "127.0.0.1", port, socket: { data() {} } });
|
||||
socket.end();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const args = process.argv.slice(2);
|
||||
const env: Record<string, string> = {};
|
||||
let fixed = 0;
|
||||
while (args[0] === "--env" || args[0] === "--port") {
|
||||
if (args[0] === "--port") fixed = Number(args[1]);
|
||||
else {
|
||||
const [k, ...v] = args[1].split("=");
|
||||
env[k] = v.join("=");
|
||||
}
|
||||
args.splice(0, 2);
|
||||
}
|
||||
if (args[0] !== "--") throw new Error("usage: with-server.ts [--env K=V ...] -- <server argv...> -- <client argv...>");
|
||||
const split = args.indexOf("--", 1);
|
||||
let serverArgv = args.slice(1, split);
|
||||
let clientArgv = args.slice(split + 1);
|
||||
|
||||
let port = fixed;
|
||||
const assigned = fixed !== 0 || serverArgv.some((a) => a.includes("{port}"));
|
||||
if (assigned && fixed === 0) {
|
||||
port = freePort();
|
||||
serverArgv = serverArgv.map((a) => a.replaceAll("{port}", String(port)));
|
||||
}
|
||||
const server = Bun.spawn(serverArgv, { stdout: "pipe", stderr: "inherit", env: process.env });
|
||||
try {
|
||||
const deadline = Date.now() + 30_000;
|
||||
if (assigned) {
|
||||
while (!(await accepts(port))) {
|
||||
if (Date.now() > deadline || server.exitCode !== null) throw new Error("server never accepted connections");
|
||||
await Bun.sleep(100);
|
||||
}
|
||||
} else {
|
||||
const reader = server.stdout.getReader();
|
||||
let text = "";
|
||||
while (port === 0) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) throw new Error("server exited before listening");
|
||||
text += new TextDecoder().decode(value);
|
||||
const match = text.match(/\{"listening":(\d+)\}/);
|
||||
if (match) port = Number(match[1]);
|
||||
}
|
||||
reader.releaseLock();
|
||||
}
|
||||
clientArgv = clientArgv.map((a) => a.replaceAll("{port}", String(port)));
|
||||
const client = Bun.spawn(clientArgv, { stdout: "inherit", stderr: "inherit", env: { ...process.env, ...env } });
|
||||
return await client.exited;
|
||||
} finally {
|
||||
server.kill("SIGTERM");
|
||||
const ended = await Promise.race([server.exited.then(() => true), Bun.sleep(3000).then(() => false)]);
|
||||
if (!ended) server.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(await main());
|
||||
@@ -0,0 +1,151 @@
|
||||
// Credentials: real `fx mcp list --json`, `fx mcp logout` and `fx mcp login`
|
||||
// runs against a scratch HOME (FX_DISABLE_KEYCHAIN=1, so the file backend),
|
||||
// each command's effect on ~/.fx/mcp-credentials read back, and the trace
|
||||
// checked against Credentials.tla through CredentialsTrace.tla. No server is
|
||||
// contacted: list never connects, these logins have no revocation endpoint,
|
||||
// and a login on an unreadable store stops before it opens a listener.
|
||||
import { expect, test } from "bun:test";
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { artifacts, binary } from "../lib/context";
|
||||
import { fixture, fxEnv, observeStores, runFx, save, v1Entry, v2Path, writeProfile, writeV1, type Fixture } from "./driver";
|
||||
import { checkTrace, type TraceRecord } from "./trace";
|
||||
|
||||
// Tests marked NEXT_MCP check the MCP host fixes from the next MCP-v2
|
||||
// version, which src/core/mcp_host/ does not have yet. They run only with
|
||||
// FX_TLA_NEXT_MCP=1.
|
||||
const NEXT_MCP = process.env.FX_TLA_NEXT_MCP === "1";
|
||||
|
||||
|
||||
const URL = "http://127.0.0.1:9/mcp";
|
||||
const UNREADABLE = "fx can't read the saved MCP sign-ins in ~/.fx/mcp-credentials/credentials-v2.json. A newer fx saved them, or they're damaged; fix or delete them, then try again.\n";
|
||||
|
||||
function setup(name: string) {
|
||||
const f = fixture(artifacts, name);
|
||||
writeProfile(f, { web: { type: "http", url: URL } });
|
||||
return f;
|
||||
}
|
||||
|
||||
class Session {
|
||||
records: TraceRecord[] = [];
|
||||
constructor(readonly f: Fixture) {}
|
||||
observe() {
|
||||
const o = observeStores(this.f);
|
||||
this.records.push({ event: "observe", data: o, at: Date.now() });
|
||||
return o;
|
||||
}
|
||||
async run(cmd: "get" | "logout" | "put", args: string[], server = "") {
|
||||
this.records.push({ event: "run", data: { cmd, server }, at: Date.now(), note: `fx ${args.join(" ")}` });
|
||||
const ran = await runFx(binary, this.f, args, fxEnv(this.f), 30_000);
|
||||
const o = this.observe();
|
||||
return { ran, o };
|
||||
}
|
||||
fault(cmd: "unreadable" | "repair" | "delete") {
|
||||
this.records.push({ event: "run", data: { cmd, server: "" }, at: Date.now() });
|
||||
}
|
||||
async check(initKinds: string[], initEntries: string[], initImported = [false, true]) {
|
||||
const verdict = await checkTrace({
|
||||
module: "Credentials", records: this.records, dir: join(this.f.base, "trace"),
|
||||
constants: {
|
||||
ObsNames: ["web"], ObsFaults: ["unreadable", "repair", "delete"], ObsInitKinds: initKinds, ObsInitEntries: initEntries,
|
||||
ObsInitImported: initImported, ObsInitMarked: [false], ObsInitV1: ["web"],
|
||||
},
|
||||
});
|
||||
save(join(this.f.base, "verdict.json"), verdict);
|
||||
return verdict;
|
||||
}
|
||||
}
|
||||
|
||||
/** A newer fx's document: version 2, which this fx can't read. */
|
||||
function rewriteAsVersion(f: Fixture, version: number) {
|
||||
const doc = JSON.parse(readFileSync(v2Path(f), "utf8"));
|
||||
writeFileSync(v2Path(f), JSON.stringify({ ...doc, version }), { mode: 0o600 });
|
||||
}
|
||||
|
||||
test.skipIf(!NEXT_MCP)("control: v1's login is taken over once, and a logout stays (trace ACCEPTED)", async () => {
|
||||
const s = new Session(setup("cred-control"));
|
||||
writeV1(s.f, [v1Entry("web", URL)]);
|
||||
expect(s.observe()).toEqual({ kind: "absent", entries: [], imported: false, marked: false, v1: ["web"] });
|
||||
const list = await s.run("get", ["mcp", "list", "--json"]);
|
||||
expect(list.ran.code).toBe(0);
|
||||
expect(list.o).toEqual({ kind: "ok", entries: ["web"], imported: true, marked: true, v1: ["web"] });
|
||||
const out = await s.run("logout", ["mcp", "logout", "web"], "web");
|
||||
expect(out.ran.code).toBe(0);
|
||||
expect(out.ran.stdout).toBe("Signed out of web.\n");
|
||||
expect(out.o.entries).toEqual([]);
|
||||
const again = await s.run("get", ["mcp", "list", "--json"]);
|
||||
expect(again.o).toEqual({ kind: "ok", entries: [], imported: true, marked: true, v1: ["web"] });
|
||||
const verdict = await s.check(["absent"], []);
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
}, 120_000);
|
||||
|
||||
test.skipIf(!NEXT_MCP)("a newer fx's store is never written over; sign-in and sign-out say why they can't, and the login is there once it's readable (trace ACCEPTED)", async () => {
|
||||
const s = new Session(setup("cred-unreadable"));
|
||||
const credential = JSON.stringify({ access: "kept-by-newer-fx", refresh: "r", expires_at_ms: null, token_endpoint: "http://127.0.0.1:9/token", revocation_endpoint: null, resource: URL, client_id: "c", client_secret: null, method: "none" });
|
||||
mkdirSync(join(s.f.home, ".fx", "mcp-credentials"), { recursive: true, mode: 0o700 });
|
||||
writeFileSync(v2Path(s.f), JSON.stringify({ version: 2, imported_v1: true, servers: [{ name: "web", url: URL, credential }] }), { mode: 0o600 });
|
||||
const before = readFileSync(v2Path(s.f), "utf8");
|
||||
save(join(s.f.base, "store-before.json"), before);
|
||||
expect(s.observe()).toEqual({ kind: "unreadable", entries: ["web"], imported: true, marked: false, v1: [] });
|
||||
|
||||
// `fx mcp list` never connects; it reads web's credential to start the host.
|
||||
const list = await s.run("get", ["mcp", "list", "--json"]);
|
||||
expect(list.ran.code).toBe(0);
|
||||
const logout = await s.run("logout", ["mcp", "logout", "web"], "web");
|
||||
expect(logout.ran.code).toBe(1);
|
||||
expect(logout.ran.stderr).toBe(`Can't sign out of web: ${UNREADABLE}`);
|
||||
const login = await s.run("put", ["mcp", "login", "--no-browser", "web"], "web");
|
||||
expect(login.ran.code).toBe(1);
|
||||
expect(login.ran.stderr).toBe(`Can't sign in to web: ${UNREADABLE}`);
|
||||
// Independent oracle: the file is byte for byte what the newer fx wrote.
|
||||
save(join(s.f.base, "store-after.json"), readFileSync(v2Path(s.f), "utf8"));
|
||||
expect(readFileSync(v2Path(s.f), "utf8")).toBe(before);
|
||||
expect(login.o).toEqual({ kind: "unreadable", entries: ["web"], imported: true, marked: false, v1: [] });
|
||||
|
||||
// Readable again (version 1), the login is still there and still used.
|
||||
rewriteAsVersion(s.f, 1);
|
||||
s.fault("repair");
|
||||
s.observe();
|
||||
const back = await s.run("get", ["mcp", "list", "--json"]);
|
||||
expect(back.o).toEqual({ kind: "ok", entries: ["web"], imported: true, marked: true, v1: [] });
|
||||
expect(readFileSync(v2Path(s.f), "utf8")).toContain("kept-by-newer-fx");
|
||||
const verdict = await s.check(["unreadable"], ["web"], [true]);
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
}, 120_000);
|
||||
|
||||
test.skipIf(!NEXT_MCP)("a logout stays when the store becomes unreadable (trace ACCEPTED)", async () => {
|
||||
const s = new Session(setup("cred-undone"));
|
||||
writeV1(s.f, [v1Entry("web", URL)]);
|
||||
s.observe();
|
||||
await s.run("get", ["mcp", "list", "--json"]);
|
||||
const out = await s.run("logout", ["mcp", "logout", "web"], "web");
|
||||
expect(out.ran.stdout).toBe("Signed out of web.\n");
|
||||
expect(out.o.entries).toEqual([]);
|
||||
rewriteAsVersion(s.f, 2);
|
||||
s.fault("unreadable");
|
||||
const before = readFileSync(v2Path(s.f), "utf8");
|
||||
expect(s.observe().kind).toBe("unreadable");
|
||||
const back = await s.run("get", ["mcp", "list", "--json"]);
|
||||
const verdict = await s.check(["absent"], []);
|
||||
// Independent oracle: nothing was imported or written.
|
||||
expect(back.o).toEqual({ kind: "unreadable", entries: [], imported: true, marked: true, v1: ["web"] });
|
||||
expect(readFileSync(v2Path(s.f), "utf8")).toBe(before);
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
}, 120_000);
|
||||
|
||||
test.skipIf(!NEXT_MCP)("deleting the v2 file doesn't bring back a login signed out of (trace ACCEPTED)", async () => {
|
||||
const s = new Session(setup("cred-deleted"));
|
||||
writeV1(s.f, [v1Entry("web", URL)]);
|
||||
s.observe();
|
||||
await s.run("get", ["mcp", "list", "--json"]);
|
||||
await s.run("logout", ["mcp", "logout", "web"], "web");
|
||||
rmSync(v2Path(s.f));
|
||||
s.fault("delete");
|
||||
s.observe();
|
||||
const back = await s.run("get", ["mcp", "list", "--json"]);
|
||||
const verdict = await s.check(["absent"], []);
|
||||
// Independent oracle: v1's copy stayed out, and only the note remains.
|
||||
expect(back.o.entries).toEqual([]);
|
||||
expect(back.o).toEqual({ kind: "absent", entries: [], imported: false, marked: true, v1: ["web"] });
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
}, 120_000);
|
||||
@@ -0,0 +1,147 @@
|
||||
// What the mcp-v2-model scenarios share: private fixture folders outside
|
||||
// every repository, the real fx binary with a scratch HOME, fixture logs,
|
||||
// the credential files, loopback probes, and evidence files.
|
||||
import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { TraceRecord } from "./trace";
|
||||
|
||||
export const FIXTURE = join(import.meta.dirname, "fixtures", "stdio-server.ts");
|
||||
/** MCP-v2's fake Streamable HTTP server with fake OAuth, used read-only. */
|
||||
export const HTTP_FAKE = join(import.meta.dirname, "../mcp/fakes/http.ts");
|
||||
export const BUN = process.execPath;
|
||||
|
||||
export interface Fixture { base: string; home: string; workspace: string; control: string; serverLog: string; traceLog: string }
|
||||
|
||||
export function fixture(root: string, name: string): Fixture {
|
||||
const base = mkdtempSync(join(root, `${name}-`));
|
||||
const home = join(base, "home");
|
||||
const workspace = join(base, "workspace");
|
||||
const control = join(base, "control");
|
||||
for (const dir of [join(home, ".fx"), workspace, control]) mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
writeFileSync(join(home, ".fx", "settings.json"), "{}");
|
||||
return { base, home, workspace, control, serverLog: join(base, "servers.jsonl"), traceLog: join(base, "fx-trace.log") };
|
||||
}
|
||||
|
||||
/** fx's environment: scratch HOME, MCP-v2, no Keychain, no browser, no sound, a fake gateway key. */
|
||||
export function fxEnv(f: Fixture, extra: Record<string, string> = {}): Record<string, string> {
|
||||
return {
|
||||
PATH: process.env.PATH ?? "/usr/bin:/bin",
|
||||
HOME: f.home,
|
||||
TMPDIR: process.env.TMPDIR ?? "/tmp",
|
||||
FX_MCP_ENGINE: "v2",
|
||||
FX_DISABLE_KEYCHAIN: "1",
|
||||
FX_NO_OPEN_BROWSER: "1",
|
||||
FX_AUTO_UPGRADE: "0",
|
||||
FX_SOUND: "0",
|
||||
FX_SKIP_ONBOARDING: "1",
|
||||
FX_E2E_DISABLE_DOTENV: "1",
|
||||
FX_TRACE_LOG: f.traceLog,
|
||||
AI_GATEWAY_API_KEY: "fake-mcp-v2-model-key",
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
/** A stdio server entry for mcp.json running the fixture. */
|
||||
export function stdioServer(f: Fixture, name: string, extra: Record<string, string> = {}) {
|
||||
return { command: BUN, args: [FIXTURE], env: { FIXTURE_NAME: name, FIXTURE_LOG: f.serverLog, FIXTURE_CONTROL: f.control, ...extra } };
|
||||
}
|
||||
|
||||
export function writeProfile(f: Fixture, servers: Record<string, unknown>) {
|
||||
writeFileSync(join(f.home, ".fx", "mcp.json"), JSON.stringify({ mcpServers: servers }, null, 2), { mode: 0o600 });
|
||||
}
|
||||
export function writeProject(f: Fixture, servers: Record<string, unknown>) {
|
||||
writeFileSync(join(f.workspace, ".mcp.json"), JSON.stringify({ mcpServers: servers }, null, 2));
|
||||
}
|
||||
|
||||
export interface Ran { args: string[]; code: number; stdout: string; stderr: string; ms: number }
|
||||
|
||||
/** Runs the binary once, with an explicit timeout; every run is appended to `runs.jsonl`. */
|
||||
export async function runFx(binary: string, f: Fixture, args: string[], env: Record<string, string>, timeoutMs = 30_000): Promise<Ran> {
|
||||
const started = Date.now();
|
||||
const child = Bun.spawn([binary, ...args], { cwd: f.workspace, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
||||
const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
||||
clearTimeout(timer);
|
||||
const ran = { args, code, stdout, stderr, ms: Date.now() - started };
|
||||
appendFileSync(join(f.base, "runs.jsonl"), JSON.stringify(ran) + "\n");
|
||||
return ran;
|
||||
}
|
||||
|
||||
export interface ServerEvent { t: number; name: string; pid: number; ev: string; [k: string]: unknown }
|
||||
|
||||
export function serverEvents(f: Fixture): ServerEvent[] {
|
||||
if (!existsSync(f.serverLog)) return [];
|
||||
return readFileSync(f.serverLog, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l) as ServerEvent);
|
||||
}
|
||||
|
||||
export async function waitFor<T>(what: string, probe: () => T | undefined | null | false, timeoutMs: number, everyMs = 50): Promise<T> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
for (;;) {
|
||||
const got = probe();
|
||||
if (got) return got as T;
|
||||
if (Date.now() > deadline) throw new Error(`timed out after ${timeoutMs} ms waiting for ${what}`);
|
||||
await Bun.sleep(everyMs);
|
||||
}
|
||||
}
|
||||
|
||||
/** A GET to a loopback port: "open" when something answered, "closed" when the connection was refused. */
|
||||
export async function probe(port: number, path = "/favicon.ico"): Promise<{ result: "open" | "closed" | "hang"; status?: number }> {
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}${path}`, { signal: AbortSignal.timeout(2000) });
|
||||
await response.text();
|
||||
return { result: "open", status: response.status };
|
||||
} catch (error) {
|
||||
const text = String((error as { code?: string }).code ?? error);
|
||||
if (/ECONNREFUSED|ConnectionRefused|Unable to connect/i.test(text)) return { result: "closed" };
|
||||
if (/Timeout|abort/i.test(text)) return { result: "hang" };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- credentials ----
|
||||
|
||||
export const credDir = (f: Fixture) => join(f.home, ".fx", "mcp-credentials");
|
||||
export const v2Path = (f: Fixture) => join(credDir(f), "credentials-v2.json");
|
||||
export const v1Path = (f: Fixture) => join(credDir(f), "credentials.json");
|
||||
/** The note that v1's logins were taken over. */
|
||||
export const notePath = (f: Fixture) => join(credDir(f), "v1-imported");
|
||||
|
||||
/** A v1 login as v1's store keeps it (mcp_auth_store.zig, its own test's shape). */
|
||||
export function v1Entry(name: string, url: string) {
|
||||
return {
|
||||
server_identity: name, token_type: "Bearer", endpoint: url, resource: url, issuer: "http://127.0.0.1:9",
|
||||
client_id: "c1", client_secret: null, access_token: `v1-access-${name}`, refresh_token: `v1-refresh-${name}`, scope: "read",
|
||||
token_endpoint_auth_method: "none", authorization_endpoint: "http://127.0.0.1:9/authorize",
|
||||
token_endpoint: "http://127.0.0.1:9/token", revocation_endpoint: null, expires_at_ms: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeV1(f: Fixture, entries: object[]) {
|
||||
mkdirSync(credDir(f), { recursive: true, mode: 0o700 });
|
||||
writeFileSync(v1Path(f), JSON.stringify({ version: 1, credentials: entries }), { mode: 0o600 });
|
||||
}
|
||||
|
||||
/** The documents as the model sees them: the v2 one's kind, entries, and flag; v1's logins. */
|
||||
export function observeStores(f: Fixture) {
|
||||
let kind = "absent", entries: string[] = [], imported = false;
|
||||
if (existsSync(v2Path(f))) {
|
||||
const text = readFileSync(v2Path(f), "utf8");
|
||||
try {
|
||||
const doc = JSON.parse(text);
|
||||
entries = (doc.servers ?? []).map((s: { name: string }) => s.name).sort();
|
||||
imported = doc.imported_v1 === true;
|
||||
kind = doc.version === 1 && Array.isArray(doc.servers) ? "ok" : "unreadable";
|
||||
} catch { kind = "unreadable"; }
|
||||
}
|
||||
const v1 = existsSync(v1Path(f)) ? (JSON.parse(readFileSync(v1Path(f), "utf8")).credentials ?? []).map((c: { server_identity: string }) => c.server_identity).sort() : [];
|
||||
return { kind, entries, imported, marked: existsSync(notePath(f)), v1 };
|
||||
}
|
||||
|
||||
export function save(path: string, value: unknown) {
|
||||
writeFileSync(path, typeof value === "string" ? value : JSON.stringify(value, null, 2));
|
||||
}
|
||||
|
||||
/** Server log lines as trace records, in time order with the harness's own. */
|
||||
export function merge(...lists: TraceRecord[][]): TraceRecord[] {
|
||||
return lists.flat().map((r, k) => ({ r, k })).sort((a, b) => (a.r.at ?? 0) - (b.r.at ?? 0) || a.k - b.k).map((x) => x.r);
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// ElicitationRouting: a real `fx ask` in a terminal (tmux; fx ask asks MCP
|
||||
// questions only with a terminal) whose fake-gateway model calls the 2025
|
||||
// fixture's `elicit` tool. The fixture asks question 1, and once the harness
|
||||
// sees it on screen, withdraws it (notifications/cancelled) and asks question
|
||||
// 2, which the client's slot table puts in question 1's slot. The harness
|
||||
// then answers what it saw: "for-1". The fixture's log says which question
|
||||
// got which answer. Checked against ElicitationRouting.tla.
|
||||
//
|
||||
// An earlier fx gave question 2 the answer "for-1". Now it is the contract:
|
||||
// the answer to the withdrawn question reaches no one, and question 2 gets its
|
||||
// own, which the model with its guard accepts.
|
||||
import { expect, test } from "bun:test";
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { artifacts, binary, shellQuote, targetHelpers } from "../lib/context";
|
||||
import { fixture, fxEnv, save, serverEvents, stdioServer, waitFor, writeProfile } from "./driver";
|
||||
import { checkTrace, type TraceRecord } from "./trace";
|
||||
|
||||
// Tests marked NEXT_MCP check the MCP host fixes from the next MCP-v2
|
||||
// version, which src/core/mcp_host/ does not have yet. They run only with
|
||||
// FX_TLA_NEXT_MCP=1.
|
||||
const NEXT_MCP = process.env.FX_TLA_NEXT_MCP === "1";
|
||||
|
||||
|
||||
const TOOL = "mcp_fixture_elicit";
|
||||
|
||||
test.skipIf(!NEXT_MCP)("an answer to a withdrawn 2025 question reaches no one, and question 2 gets its own (trace ACCEPTED with the guard)", async () => {
|
||||
const f = fixture(artifacts, "elicit-slot-reuse");
|
||||
writeProfile(f, { fixture: stdioServer(f, "fixture") });
|
||||
const { TmuxSession, startFakeGateway, fakeGatewayToolCall, fakeGatewayFinalText } = await targetHelpers();
|
||||
const gateway = startFakeGateway([
|
||||
fakeGatewayToolCall("select", "mcp_select_tool", { name: TOOL }),
|
||||
fakeGatewayToolCall("call", TOOL, {}),
|
||||
fakeGatewayFinalText("ELICIT_TURN_DONE"),
|
||||
], { models: [{ id: "openai/gpt-5", type: "language", tags: ["tool-use"] }] });
|
||||
const env = fxEnv(f, { FX_GATEWAY_BASE_URL: gateway.baseUrl, FX_GATEWAY_CHAT_URL: gateway.chatUrl, FX_MODEL: "openai/gpt-5", FX_PERMISSION_MODE: "full-access" });
|
||||
const tui = await TmuxSession.create({
|
||||
cmd: `${shellQuote(binary)} ask --no-save ${shellQuote("Use the fixture's elicit tool.")}`,
|
||||
// No stderrPath: fx ask declares elicitation only when stdin and stderr are a terminal.
|
||||
cwd: f.workspace, env, isolated: true, width: 160, height: 40, remainOnExit: true,
|
||||
});
|
||||
const records: TraceRecord[] = [];
|
||||
const panes: { at: number; label: string; pane: string }[] = [];
|
||||
const snap = async (label: string) => { const pane = await tui.capturePane(); panes.push({ at: Date.now(), label, pane }); return pane; };
|
||||
const ev = (name: string) => serverEvents(f).filter((e) => e.ev === name);
|
||||
let two = false;
|
||||
try {
|
||||
await waitFor("question 1 sent", () => ev("ask").length >= 1, 30_000);
|
||||
await tui.waitForText(/Answer 1|Question 1|question 1/, 15_000);
|
||||
records.push({ event: "shown", data: { q: 1 }, at: Date.now() });
|
||||
await snap("question 1 on screen");
|
||||
writeFileSync(join(f.control, "go"), "go\n");
|
||||
await waitFor("question 2 sent", () => ev("ask").length >= 2, 10_000);
|
||||
await Bun.sleep(700);
|
||||
await snap("after the withdrawal and question 2");
|
||||
await tui.sendText("for-1");
|
||||
// fx reviews the form before it goes: Submit is choice 1.
|
||||
await tui.waitForText("Choose Submit, Edit, Decline, or Cancel.", 10_000);
|
||||
await snap("question 1 review");
|
||||
records.push({ event: "answer", data: { q: 1 }, at: Date.now() });
|
||||
await tui.sendText("1");
|
||||
// fx may show question 2 next; answer what is on screen, naming it.
|
||||
two = await tui.waitForText(/Answer 2|Question 2|question 2/, 6_000).then(() => true, () => false);
|
||||
if (two) {
|
||||
records.push({ event: "shown", data: { q: 2 }, at: Date.now() });
|
||||
await snap("question 2 on screen");
|
||||
await tui.sendText("for-2");
|
||||
await tui.waitForText(/Answer 2: "for-2"[\s\S]*Choose Submit/, 10_000).catch(() => undefined);
|
||||
records.push({ event: "answer", data: { q: 2 }, at: Date.now() });
|
||||
await tui.sendText("1");
|
||||
}
|
||||
await waitFor("the call's result", () => ev("call_end").length === 1, 40_000);
|
||||
await tui.waitForText("ELICIT_TURN_DONE", 20_000).catch(() => undefined);
|
||||
await snap("end");
|
||||
} finally {
|
||||
save(join(f.base, "panes.json"), panes);
|
||||
await tui.kill();
|
||||
gateway.stop();
|
||||
}
|
||||
const events = serverEvents(f);
|
||||
for (const e of events) {
|
||||
if (e.ev === "ask") records.push({ event: "ask", data: { q: e.q as number }, at: e.t });
|
||||
if (e.ev === "withdraw") records.push({ event: "withdraw", data: { q: e.q as number }, at: e.t });
|
||||
if (e.ev === "answered") {
|
||||
const target = e.id === "e1" ? 1 : 2;
|
||||
const content = (e.result as { action?: string; content?: { answer?: string } } | null);
|
||||
const from = /^for-(\d)$/.exec(content?.content?.answer ?? "")?.[1];
|
||||
records.push({ event: "delivered", data: { target, from: from ? Number(from) : target }, at: e.t, note: JSON.stringify(e.result) });
|
||||
}
|
||||
if (e.ev === "call_end") records.push({ event: "call_end", at: e.t });
|
||||
}
|
||||
records.sort((a, b) => (a.at ?? 0) - (b.at ?? 0));
|
||||
const verdict = await checkTrace({ module: "ElicitationRouting", records, dir: join(f.base, "trace"), constants: { ObsVersions: ["2025"], ObsMaxQ: 2, ObsSlots: 2, ObsGuard: true } });
|
||||
save(join(f.base, "verdict.json"), verdict);
|
||||
// Independent oracle: what reached question 2 on the wire.
|
||||
const toTwo = events.find((e) => e.ev === "answered" && e.id === "e2");
|
||||
save(join(f.base, "wire.json"), { toTwo, all: events.filter((e) => ["ask", "withdraw", "answered", "call_end"].includes(e.ev)) });
|
||||
expect(two).toBe(true);
|
||||
expect(toTwo).toBeDefined();
|
||||
expect((toTwo!.result as { content?: { answer?: string } }).content?.answer).toBe("for-2");
|
||||
expect(events.filter((e) => e.ev === "answered" && e.id === "e1")).toEqual([]);
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
}, 180_000);
|
||||
@@ -0,0 +1,109 @@
|
||||
// A 2025-11-25 stdio MCP server for the mcp-v2-model case. It logs every
|
||||
// lifecycle event to FIXTURE_LOG as JSONL ({t, name, pid, ev, ...}):
|
||||
// start, listed (answered tools/list), eof (stdin ended), sigterm, recv, sent.
|
||||
// Env:
|
||||
// FIXTURE_NAME the server's name in its log lines
|
||||
// FIXTURE_LOG the JSONL log (appended synchronously)
|
||||
// FIXTURE_INIT_DELAY_MS hold initialize's answer this long (a slow server)
|
||||
// FIXTURE_CONTROL a folder; the elicit tool waits for "go" in it
|
||||
// Tools: echo (answers at once), elicit (2025 elicitations, see below).
|
||||
import { appendFileSync, existsSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const name = process.env.FIXTURE_NAME ?? "fixture";
|
||||
const logPath = process.env.FIXTURE_LOG;
|
||||
const initDelay = Number(process.env.FIXTURE_INIT_DELAY_MS ?? "0");
|
||||
const control = process.env.FIXTURE_CONTROL;
|
||||
const log = (ev: string, extra: object = {}) => {
|
||||
if (logPath) appendFileSync(logPath, JSON.stringify({ t: Date.now(), name, pid: process.pid, ev, ...extra }) + "\n");
|
||||
};
|
||||
const send = (message: object) => {
|
||||
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + "\n");
|
||||
log("sent", { message });
|
||||
};
|
||||
log("start");
|
||||
process.on("SIGTERM", () => {
|
||||
log("sigterm");
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
const tools = [
|
||||
{ name: "echo", description: "Echo text back", inputSchema: { type: "object", properties: { text: { type: "string" } } } },
|
||||
{ name: "elicit", description: "Ask the user two questions, withdrawing the first", inputSchema: { type: "object" } },
|
||||
];
|
||||
const form = (n: number) => ({
|
||||
mode: "form",
|
||||
message: `Question ${n}`,
|
||||
requestedSchema: { type: "object", properties: { answer: { type: "string", title: `Answer ${n}`, description: `What do you say to question ${n}?` } }, required: ["answer"] },
|
||||
});
|
||||
const waiters = new Map<string, (message: { result?: { action?: string; content?: { answer?: string } } }) => void>();
|
||||
|
||||
async function elicit(callId: unknown) {
|
||||
// Question 1, then (once the harness says go) its withdrawal and question 2,
|
||||
// which the client's slot table puts where question 1 was.
|
||||
const answers: Record<string, unknown> = {};
|
||||
const answered = (id: string) => new Promise<void>((resolve) => waiters.set(id, (m) => { answers[id] = m.result ?? null; resolve(); }));
|
||||
const first = answered("e1");
|
||||
send({ id: "e1", method: "elicitation/create", params: form(1) });
|
||||
log("ask", { q: 1, id: "e1" });
|
||||
const go = control ? join(control, "go") : null;
|
||||
const deadline = Date.now() + 60_000;
|
||||
let firstDone = false;
|
||||
first.then(() => { firstDone = true; });
|
||||
while (go && !existsSync(go) && !firstDone && Date.now() < deadline) await Bun.sleep(25);
|
||||
if (!firstDone) {
|
||||
send({ method: "notifications/cancelled", params: { requestId: "e1", reason: "withdrawn by the fixture" } });
|
||||
log("withdraw", { q: 1, id: "e1" });
|
||||
const second = answered("e2");
|
||||
send({ id: "e2", method: "elicitation/create", params: form(2) });
|
||||
log("ask", { q: 2, id: "e2" });
|
||||
await Promise.race([second, Bun.sleep(30_000)]);
|
||||
// Long enough for fx to show what it holds next.
|
||||
await Bun.sleep(Number(process.env.FIXTURE_AFTER_MS ?? "2500"));
|
||||
}
|
||||
send({ id: callId, result: { content: [{ type: "text", text: `ELICIT_DONE ${JSON.stringify(answers)}` }] } });
|
||||
log("call_end", { answers });
|
||||
}
|
||||
|
||||
let buffer = "";
|
||||
process.stdin.on("data", (chunk) => {
|
||||
buffer += chunk.toString();
|
||||
for (let at = buffer.indexOf("\n"); at >= 0; at = buffer.indexOf("\n")) {
|
||||
const line = buffer.slice(0, at).trim();
|
||||
buffer = buffer.slice(at + 1);
|
||||
if (line) void handle(JSON.parse(line));
|
||||
}
|
||||
});
|
||||
process.stdin.on("end", () => {
|
||||
log("eof");
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
async function handle(m: { id?: unknown; method?: string; params?: { name?: string }; result?: unknown; error?: unknown }) {
|
||||
log("recv", { message: m });
|
||||
if (m.method === undefined) {
|
||||
const waiter = waiters.get(String(m.id));
|
||||
if (waiter) {
|
||||
waiters.delete(String(m.id));
|
||||
log("answered", { id: m.id, result: m.result ?? null, error: m.error ?? null });
|
||||
waiter(m as never);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (m.id === undefined) return; // a notification
|
||||
switch (m.method) {
|
||||
case "initialize":
|
||||
if (initDelay > 0) await Bun.sleep(initDelay);
|
||||
return send({ id: m.id, result: { protocolVersion: "2025-11-25", capabilities: { tools: {} }, serverInfo: { name, version: "1.0.0" } } });
|
||||
case "tools/list":
|
||||
send({ id: m.id, result: { tools } });
|
||||
return log("listed");
|
||||
case "tools/call":
|
||||
if (m.params?.name === "elicit") return elicit(m.id);
|
||||
return send({ id: m.id, result: { content: [{ type: "text", text: `ECHO ${name}` }] } });
|
||||
case "ping":
|
||||
return send({ id: m.id, result: {} });
|
||||
default:
|
||||
return send({ id: m.id, error: { code: -32601, message: "Method not found" } });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// HostLifecycle: the real fx shell in tmux with fixture stdio servers. The
|
||||
// trace is the fixtures' own log (start, listed, eof, sigterm: one process
|
||||
// each, by pid) merged in time order with what the harness did (`/mcp`
|
||||
// verbs, Ctrl+C) and the status line once it stopped changing. Checked
|
||||
// against HostLifecycle.tla through HostLifecycleTrace.tla.
|
||||
import { expect, test } from "bun:test";
|
||||
import { join } from "node:path";
|
||||
import { artifacts } from "../lib/context";
|
||||
import { fixture, merge, save, serverEvents, stdioServer, waitFor, writeProfile, writeProject, type Fixture } from "./driver";
|
||||
import { serverRecords, Shell } from "./shell";
|
||||
import { checkTrace } from "./trace";
|
||||
|
||||
// Tests marked NEXT_MCP check the MCP host fixes from the next MCP-v2
|
||||
// version, which src/core/mcp_host/ does not have yet. They run only with
|
||||
// FX_TLA_NEXT_MCP=1.
|
||||
const NEXT_MCP = process.env.FX_TLA_NEXT_MCP === "1";
|
||||
|
||||
|
||||
function alive(pid: number) { try { process.kill(pid, 0); return true; } catch { return false; } }
|
||||
const started = (f: Fixture, name: string) => serverEvents(f).filter((e) => e.name === name && e.ev === "start");
|
||||
const listed = (f: Fixture, name: string) => serverEvents(f).filter((e) => e.name === name && e.ev === "listed");
|
||||
|
||||
async function finish(shell: Shell, constants: Record<string, unknown>) {
|
||||
const events = serverEvents(shell.f);
|
||||
const quitAt = shell.records.find((r) => r.event === "quit")?.at ?? null;
|
||||
const records = merge(shell.records.filter((r) => r.event !== "release"), serverRecords(events, quitAt));
|
||||
// A release goes right before the exit of the process its host's end stops
|
||||
// (the pane is read after the fact, and one log line can share its millisecond).
|
||||
for (const release of shell.records.filter((r) => r.event === "release")) {
|
||||
const pid = release.data?.beforeExitOf;
|
||||
const at = records.findIndex((r) => r.event === "stop" && r.note?.startsWith(`pid ${pid} `));
|
||||
if (at < 0) throw new Error(`harness: no exit of pid ${pid} to place the release before`);
|
||||
records.splice(at, 0, { event: "release", at: records[at]!.at, note: release.note });
|
||||
}
|
||||
const verdict = await checkTrace({ module: "HostLifecycle", records, constants: constants as never, dir: join(shell.f.base, "trace") });
|
||||
save(join(shell.f.base, "verdict.json"), verdict);
|
||||
return { verdict, events };
|
||||
}
|
||||
|
||||
function setup(name: string, profile: string[], project: string[], slow: string[] = []) {
|
||||
const f = fixture(artifacts, name);
|
||||
writeProfile(f, Object.fromEntries(profile.map((n) => [n, stdioServer(f, n, slow.includes(n) ? { FIXTURE_INIT_DELAY_MS: "6000" } : {})])));
|
||||
if (project.length) writeProject(f, Object.fromEntries(project.map((n) => [n, stdioServer(f, n)])));
|
||||
return f;
|
||||
}
|
||||
|
||||
test("first frame starts the profile servers, the project server waits unspawned, and quitting SIGTERMs all without waiting (trace ACCEPTED)", async () => {
|
||||
const shell = new Shell(setup("hl-first-frame", ["a", "b"], ["p"]));
|
||||
try {
|
||||
await shell.start();
|
||||
await waitFor("a and b listed", () => listed(shell.f, "a").length && listed(shell.f, "b").length, 20_000);
|
||||
const line = await shell.status({ k: "attn", who: "p", why: "waiting" });
|
||||
expect(line).toEqual({ k: "attn", who: "p", why: "waiting", r: 0, e: 0 });
|
||||
const pids = [...started(shell.f, "a"), ...started(shell.f, "b")].map((e) => e.pid);
|
||||
const quitMs = await shell.quit();
|
||||
await waitFor("every server gone", () => pids.every((pid) => !alive(pid)), 5_000);
|
||||
const { verdict, events } = await finish(shell, { ObsServers: ["a", "b", "p"], ObsProject: ["p"], ObsReachable: ["a", "b", "p"], ObsNeedsAuth: [], ObsMaxHosts: 1, ObsMaxLeases: 1 });
|
||||
expect(started(shell.f, "p")).toEqual([]);
|
||||
expect(events.filter((e) => e.ev === "sigterm").map((e) => e.name).sort()).toEqual(["a", "b"]);
|
||||
// A graceful stop would wait 1.5 s on the closed stdin before SIGTERM.
|
||||
expect(quitMs).toBeLessThan(1_000);
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
} finally { await shell.stop(); }
|
||||
}, 120_000);
|
||||
|
||||
test("approve reloads: the project server starts and the unchanged servers come back (trace ACCEPTED)", async () => {
|
||||
const shell = new Shell(setup("hl-approve", ["a", "b"], ["p"]));
|
||||
try {
|
||||
await shell.start();
|
||||
await waitFor("a and b listed", () => listed(shell.f, "a").length && listed(shell.f, "b").length, 20_000);
|
||||
await shell.status({ k: "attn", who: "p", why: "waiting" });
|
||||
await shell.type("/mcp approve p", { event: "reload", data: { server: "p", change: "approved" } });
|
||||
await shell.tui.waitForText("Approved p", 10_000);
|
||||
await waitFor("p, a and b listed again", () => listed(shell.f, "p").length && listed(shell.f, "a").length === 2 && listed(shell.f, "b").length === 2, 20_000);
|
||||
expect(await shell.status({ k: "count", r: 3, e: 3 })).toEqual({ k: "count", who: "", why: "", r: 3, e: 3 });
|
||||
await shell.quit();
|
||||
await Bun.sleep(500);
|
||||
const { verdict, events } = await finish(shell, { ObsServers: ["a", "b", "p"], ObsProject: ["p"], ObsReachable: ["a", "b", "p"], ObsNeedsAuth: [], ObsMaxHosts: 2, ObsMaxLeases: 1 });
|
||||
// The first a and b stopped at their stdin's end before their successors started.
|
||||
expect(events.filter((e) => e.ev === "eof").map((e) => e.name).sort()).toEqual(["a", "b"]);
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
} finally { await shell.stop(); }
|
||||
}, 150_000);
|
||||
|
||||
test("reject and remove reload: the project server never starts and the removed one stops for good (trace ACCEPTED)", async () => {
|
||||
const shell = new Shell(setup("hl-reject-remove", ["a", "b"], ["p"]));
|
||||
try {
|
||||
await shell.start();
|
||||
await waitFor("a and b listed", () => listed(shell.f, "a").length && listed(shell.f, "b").length, 20_000);
|
||||
await shell.status({ k: "attn", who: "p", why: "waiting" });
|
||||
await shell.type("/mcp reject p", { event: "reload", data: { server: "p", change: "rejected" } });
|
||||
await shell.tui.waitForText("Rejected p", 10_000);
|
||||
await waitFor("a and b listed again", () => listed(shell.f, "a").length === 2 && listed(shell.f, "b").length === 2, 20_000);
|
||||
expect(await shell.status({ k: "count", r: 2, e: 2 })).toEqual({ k: "count", who: "", why: "", r: 2, e: 2 });
|
||||
await shell.type("/mcp remove b", { event: "reload", data: { server: "b", change: "removed" } });
|
||||
await shell.tui.waitForText("Removed b", 10_000);
|
||||
await waitFor("a listed a third time", () => listed(shell.f, "a").length === 3, 20_000);
|
||||
expect(await shell.status({ k: "count", r: 1, e: 1 })).toEqual({ k: "count", who: "", why: "", r: 1, e: 1 });
|
||||
await shell.quit();
|
||||
await Bun.sleep(500);
|
||||
const { verdict } = await finish(shell, { ObsServers: ["a", "b", "p"], ObsProject: ["p"], ObsReachable: ["a", "b", "p"], ObsNeedsAuth: [], ObsMaxHosts: 3, ObsMaxLeases: 1 });
|
||||
expect(started(shell.f, "p")).toEqual([]);
|
||||
expect(started(shell.f, "b").length).toBe(2);
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
} finally { await shell.stop(); }
|
||||
}, 150_000);
|
||||
|
||||
test.skipIf(!NEXT_MCP)("finding: a reload while `/mcp show` holds the old host runs two owners and two processes of one server (trace VIOLATES AtMostOneOwner, OneProcessPerServer)", async () => {
|
||||
const shell = new Shell(setup("hl-two-owners", ["s"], ["p"], ["s"]));
|
||||
try {
|
||||
await shell.start();
|
||||
await waitFor("s started", () => started(shell.f, "s").length === 1, 20_000);
|
||||
// `/mcp show s` waits for s, which takes 6 s to answer, holding the host it leased.
|
||||
await shell.type("/mcp show s", { event: "lease" });
|
||||
await Bun.sleep(500);
|
||||
await shell.type("/mcp approve p", { event: "reload", data: { server: "p", change: "approved" } });
|
||||
await shell.tui.waitForText("Approved p", 10_000);
|
||||
const both = await waitFor("a second s while the first runs", () => {
|
||||
const s = started(shell.f, "s");
|
||||
return s.length === 2 && s.every((e) => alive(e.pid)) ? s : null;
|
||||
}, 10_000);
|
||||
save(join(shell.f.base, "two-processes.json"), { at: Date.now(), pids: both.map((e) => e.pid), alive: both.map((e) => alive(e.pid)) });
|
||||
// The show prints once the first s answers; its lease ends and the old host goes.
|
||||
await shell.tui.waitForText("status ready", 20_000);
|
||||
const seen = Date.now();
|
||||
const eof = await waitFor("the first s stopped", () => serverEvents(shell.f).find((e) => e.name === "s" && e.ev === "eof" && e.pid === both[0]!.pid), 10_000);
|
||||
// The show prints once the first s answered, then its lease ends and the
|
||||
// old host goes: `finish` puts the release right before that s's exit.
|
||||
shell.records.push({ event: "release", at: seen, note: `seen ${seen}, first s eof ${eof.t}; placed before pid ${eof.pid}'s exit`, data: { beforeExitOf: eof.pid } });
|
||||
await waitFor("p and the second s listed", () => listed(shell.f, "p").length && listed(shell.f, "s").length === 2, 20_000);
|
||||
await shell.status({ k: "count", r: 2, e: 2 });
|
||||
await shell.quit();
|
||||
await Bun.sleep(500);
|
||||
const { verdict } = await finish(shell, { ObsServers: ["s", "p"], ObsProject: ["p"], ObsReachable: ["s", "p"], ObsNeedsAuth: [], ObsMaxHosts: 2, ObsMaxLeases: 1 });
|
||||
expect(both.length).toBe(2);
|
||||
expect(verdict.verdict).toBe("VIOLATES");
|
||||
expect(verdict.broken).toEqual(expect.arrayContaining(["AtMostOneOwner", "OneProcessPerServer"]));
|
||||
} finally { await shell.stop(); }
|
||||
}, 150_000);
|
||||
@@ -0,0 +1,95 @@
|
||||
// Model gate for the fx-layer MCP-v2 models: each model's smoke config
|
||||
// (safety and liveness), its stress config, every planted bug, the stated
|
||||
// properties the code is expected to break (findings), and repair witnesses.
|
||||
//
|
||||
// Verdicts, as MCP-v2's tools/run.py gives them:
|
||||
// PASS exit 0, the completed marker, an empty queue
|
||||
// CAUGHT a planted bug: exit 12 naming its invariant, or exit 13 for its property
|
||||
// FINDING a findings config: exit 12 naming the stated invariant (13 for a property)
|
||||
// Anything else is FAIL. Usage: bun model.ts OUTPUT_DIR [--model NAME]
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { configWith, runTlc, sha256, sha256Text, tlaDir, tools, type TlcRun } from "./tlc";
|
||||
|
||||
interface Mutant { description: string; old: string; new: string; invariant?: string; property?: string; config?: string }
|
||||
interface Entry {
|
||||
module: string; smoke: string; stress?: string;
|
||||
findings: { config: string; invariant?: string; property?: string; only?: boolean }[];
|
||||
repair: string[];
|
||||
}
|
||||
export interface GateResult {
|
||||
module: string; label: string; kind: "smoke" | "stress" | "mutant" | "finding" | "repair";
|
||||
verdict: "PASS" | "CAUGHT" | "FINDING" | "FAIL"; expected?: string; description?: string;
|
||||
exit: number; seconds: number; generated?: number; distinct?: number; queued?: number; log: string;
|
||||
modelSha256: string; configSha256: string;
|
||||
}
|
||||
|
||||
const catalog = () => JSON.parse(readFileSync(join(tlaDir, "catalog.json"), "utf8")) as Entry[];
|
||||
|
||||
function verdictOf(run: TlcRun, expect?: { invariant?: string; property?: string }): "PASS" | "CAUGHT" | "FAIL" {
|
||||
if (!expect) return run.exit === 0 && run.completed && run.queued === 0 ? "PASS" : "FAIL";
|
||||
if (expect.invariant) return run.exit === 12 && run.violated === expect.invariant ? "CAUGHT" : "FAIL";
|
||||
return run.exit === 13 && run.temporal && run.violated === undefined ? "CAUGHT" : "FAIL";
|
||||
}
|
||||
|
||||
export async function runModels(outDir: string, only?: string): Promise<GateResult[]> {
|
||||
mkdirSync(outDir, { recursive: true, mode: 0o700 });
|
||||
const { java, jar, jarSha256 } = tools();
|
||||
const results: GateResult[] = [];
|
||||
const save = () => writeFileSync(join(outDir, "gate.json"), JSON.stringify({ java, jar, jarSha256, results }, null, 2));
|
||||
for (const entry of catalog()) {
|
||||
if (only && entry.module !== only) continue;
|
||||
const model = join(tlaDir, `${entry.module}.tla`);
|
||||
const record = (kind: GateResult["kind"], run: TlcRun, verdict: GateResult["verdict"], cfgText: string, extra: Partial<GateResult> = {}) => {
|
||||
const r: GateResult = {
|
||||
module: entry.module, label: run.label, kind, verdict, exit: run.exit, seconds: run.seconds,
|
||||
generated: run.generated, distinct: run.distinct, queued: run.queued, log: run.log,
|
||||
modelSha256: sha256(model), configSha256: sha256Text(cfgText), ...extra,
|
||||
};
|
||||
results.push(r);
|
||||
save();
|
||||
console.log(`${r.verdict.padEnd(8)} ${entry.module.padEnd(19)} ${r.label.padEnd(44)} exit=${r.exit} distinct=${r.distinct ?? "-"} ${r.expected ?? ""}`);
|
||||
return r;
|
||||
};
|
||||
for (const [kind, cfg] of [["smoke", entry.smoke], ["stress", entry.stress]] as const) {
|
||||
if (!cfg) continue;
|
||||
const cfgText = readFileSync(join(tlaDir, cfg), "utf8");
|
||||
const run = await runTlc({ outDir, label: `${entry.module}-${kind}`, module: entry.module, cfgText, timeoutMs: 1_200_000 });
|
||||
record(kind, run, verdictOf(run), cfgText);
|
||||
}
|
||||
const mutants = JSON.parse(readFileSync(join(tlaDir, `${entry.module}.mutants.json`), "utf8")) as Mutant[];
|
||||
const source = readFileSync(model, "utf8");
|
||||
for (const [index, m] of mutants.entries()) {
|
||||
const count = source.split(m.old).length - 1;
|
||||
const label = `${entry.module}-mutant-${index + 1}`;
|
||||
if (count !== 1) throw new Error(`${label}: 'old' matches ${count} times, needs exactly 1`);
|
||||
const base = readFileSync(join(tlaDir, m.config ?? entry.smoke), "utf8");
|
||||
const cfgText = m.property ? configWith(base, { onlyProperty: m.property }) : configWith(base, { onlyProperty: "" });
|
||||
const run = await runTlc({ outDir, label, module: entry.module, cfgText, files: { [`${entry.module}.tla`]: source.replace(m.old, m.new) } });
|
||||
record("mutant", run, verdictOf(run, m), cfgText, { expected: m.invariant ?? m.property, description: m.description });
|
||||
}
|
||||
for (const [index, f] of entry.findings.entries()) {
|
||||
const base = readFileSync(join(tlaDir, f.config), "utf8");
|
||||
const cfgText = f.only && f.invariant ? configWith(base, { invariants: [f.invariant] }) : base;
|
||||
const run = await runTlc({ outDir, label: `${entry.module}-finding-${index + 1}`, module: entry.module, cfgText });
|
||||
const v = verdictOf(run, f);
|
||||
record("finding", run, v === "CAUGHT" ? "FINDING" : "FAIL", cfgText, { expected: f.invariant ?? f.property });
|
||||
}
|
||||
for (const cfg of entry.repair) {
|
||||
const cfgText = readFileSync(join(tlaDir, cfg), "utf8");
|
||||
const run = await runTlc({ outDir, label: `${entry.module}-repair-${cfg.split(".")[1]}`, module: entry.module, cfgText });
|
||||
record("repair", run, verdictOf(run), cfgText);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const out = process.argv[2];
|
||||
if (!out) throw new Error("Usage: bun model.ts NEW_OUTPUT_DIRECTORY [--model NAME]");
|
||||
const at = process.argv.indexOf("--model");
|
||||
const results = await runModels(resolve(out), at > 0 ? process.argv[at + 1] : undefined);
|
||||
const failed = results.filter((r) => r.verdict === "FAIL");
|
||||
console.log(`${results.length} runs, ${failed.length} failed`);
|
||||
process.exitCode = failed.length ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// The fx-layer models through TLC (smoke with liveness, stress, every
|
||||
// planted bug, the stated properties the code breaks, repair witnesses), and
|
||||
// the trace bridge qualified both ways: a known-good trace per model is
|
||||
// ACCEPTED and a known-bad one REJECTED at the record that breaks it.
|
||||
import { expect, test } from "bun:test";
|
||||
import { join } from "node:path";
|
||||
import { artifacts } from "../lib/context";
|
||||
import { runModels } from "./model";
|
||||
import { save } from "./driver";
|
||||
import { checkTrace, type TraceRecord } from "./trace";
|
||||
|
||||
test("TLC: every model's smoke and stress pass, every planted bug is caught, stated-property findings and repair witnesses hold", async () => {
|
||||
const results = await runModels(join(artifacts, "tlc"));
|
||||
save(join(artifacts, "gate-summary.json"), results.map(({ module, label, kind, verdict, expected, distinct, exit }) => ({ module, label, kind, verdict, expected, distinct, exit })));
|
||||
expect(results.filter((r) => r.verdict === "FAIL").map((r) => r.label)).toEqual([]);
|
||||
const count = (kind: string, verdict: string) => results.filter((r) => r.kind === kind && r.verdict === verdict).length;
|
||||
// SharedLogins joined the four.
|
||||
expect(count("smoke", "PASS")).toBe(5);
|
||||
expect(count("stress", "PASS")).toBe(5);
|
||||
expect(count("mutant", "CAUGHT")).toBe(results.filter((r) => r.kind === "mutant").length);
|
||||
// Credentials' two findings and its repair witness became the code.
|
||||
// HostLifecycle's OneProcessPerServer did too: it
|
||||
// is now an invariant its smoke and stress configs check, leases included,
|
||||
// so it is no longer an expected finding and needs no separate witness.
|
||||
// SignIn's three did too, with the hold while another host is alive: its smoke and stress
|
||||
// configs now check IndicatorImpliesListener, OneSignInPerServer and
|
||||
// IndicatorClears.
|
||||
expect(count("finding", "FINDING")).toBe(2);
|
||||
expect(count("repair", "PASS")).toBe(1);
|
||||
}, 1_800_000);
|
||||
|
||||
const controls: { name: string; module: string; constants: Record<string, never> | Record<string, unknown>; good: TraceRecord[]; bad: TraceRecord[]; badAt: number }[] = [
|
||||
{
|
||||
name: "credentials", module: "Credentials",
|
||||
constants: { ObsNames: ["web"], ObsFaults: ["unreadable"], ObsInitKinds: ["absent"], ObsInitEntries: [], ObsInitImported: [false], ObsInitMarked: [false], ObsInitV1: ["web"] },
|
||||
good: [
|
||||
{ event: "observe", data: { kind: "absent", entries: [], imported: false, marked: false, v1: ["web"] } },
|
||||
{ event: "run", data: { cmd: "get", server: "" } },
|
||||
{ event: "observe", data: { kind: "ok", entries: ["web"], imported: true, marked: true, v1: ["web"] } },
|
||||
],
|
||||
// A read that took over v1's login but didn't save the flag.
|
||||
bad: [
|
||||
{ event: "observe", data: { kind: "absent", entries: [], imported: false, marked: false, v1: ["web"] } },
|
||||
{ event: "run", data: { cmd: "get", server: "" } },
|
||||
{ event: "observe", data: { kind: "ok", entries: ["web"], imported: false, marked: true, v1: ["web"] } },
|
||||
],
|
||||
badAt: 2,
|
||||
},
|
||||
{
|
||||
name: "host lifecycle", module: "HostLifecycle",
|
||||
constants: { ObsServers: ["a", "p"], ObsProject: ["p"], ObsReachable: ["a", "p"], ObsNeedsAuth: [], ObsMaxHosts: 1, ObsMaxLeases: 1 },
|
||||
good: [
|
||||
{ event: "spawn", data: { server: "a" } },
|
||||
{ event: "ready", data: { server: "a" } },
|
||||
{ event: "status", data: { k: "attn", who: "p", why: "waiting", r: 0, e: 0 } },
|
||||
{ event: "quit" },
|
||||
{ event: "term", data: { server: "a" } },
|
||||
],
|
||||
// The project server waiting for approval starts anyway.
|
||||
bad: [
|
||||
{ event: "spawn", data: { server: "a" } },
|
||||
{ event: "spawn", data: { server: "p" } },
|
||||
],
|
||||
badAt: 1,
|
||||
},
|
||||
{
|
||||
name: "sign-in", module: "SignIn",
|
||||
constants: { ObsMaxLogins: 1, ObsMaxHosts: 1, ObsMaxLeases: 0, ObsBridge: false },
|
||||
good: [
|
||||
{ event: "login" }, { event: "authorize", data: { login: 1 } }, { event: "probe", data: { login: 1, result: "open" } },
|
||||
{ event: "callback", data: { login: 1, state: 1, kind: "get" } }, { event: "redeem", data: { login: 1 } },
|
||||
{ event: "signed_in", data: { login: 1 } }, { event: "probe", data: { login: 1, result: "closed" } },
|
||||
],
|
||||
// A code redeemed for a callback whose state fx never made.
|
||||
bad: [
|
||||
{ event: "login" }, { event: "authorize", data: { login: 1 } },
|
||||
{ event: "callback", data: { login: 1, state: 0, kind: "get" } }, { event: "redeem", data: { login: 1 } },
|
||||
],
|
||||
badAt: 3,
|
||||
},
|
||||
{
|
||||
name: "elicitation routing (2026)", module: "ElicitationRouting",
|
||||
constants: { ObsVersions: ["2026"], ObsMaxQ: 2, ObsSlots: 2, ObsGuard: false },
|
||||
good: [
|
||||
{ event: "ask", data: { q: 1 } }, { event: "shown", data: { q: 1 } }, { event: "answer", data: { q: 1 } },
|
||||
{ event: "delivered", data: { target: 1, from: 1 } }, { event: "call_end" },
|
||||
],
|
||||
// A 2026 answer reaching another round: handles aren't reused, so no match.
|
||||
bad: [
|
||||
{ event: "ask", data: { q: 1 } }, { event: "shown", data: { q: 1 } }, { event: "ask", data: { q: 2 } },
|
||||
{ event: "answer", data: { q: 1 } }, { event: "delivered", data: { target: 2, from: 1 } },
|
||||
],
|
||||
badAt: 4,
|
||||
},
|
||||
];
|
||||
|
||||
for (const c of controls) {
|
||||
test(`bridge control (${c.name}): the known-good trace is ACCEPTED and the known-bad one REJECTED at record ${c.badAt}`, async () => {
|
||||
const dir = join(artifacts, "bridge", c.module);
|
||||
const good = await checkTrace({ module: c.module, records: c.good, constants: c.constants as never, dir: join(dir, "good") });
|
||||
const bad = await checkTrace({ module: c.module, records: c.bad, constants: c.constants as never, dir: join(dir, "bad") });
|
||||
expect(good.verdict).toBe("ACCEPTED");
|
||||
expect(bad.verdict).toBe("REJECTED");
|
||||
expect(bad.firstBad?.index).toBe(c.badAt);
|
||||
}, 300_000);
|
||||
}
|
||||
|
||||
test("bridge control: an empty trace is INCONCLUSIVE, never a pass", async () => {
|
||||
const v = await checkTrace({ module: "SignIn", records: [], constants: { ObsMaxLogins: 1, ObsMaxHosts: 1, ObsMaxLeases: 0, ObsBridge: false }, dir: join(artifacts, "bridge", "empty") });
|
||||
expect(v.verdict).toBe("INCONCLUSIVE");
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
// The real fx shell in tmux for the mcp-v2-model scenarios: start it with a
|
||||
// fake gateway, read its status line once it settles, type `/mcp` verbs, quit.
|
||||
import { join } from "node:path";
|
||||
import { binary, shellQuote, targetHelpers } from "../lib/context";
|
||||
import { fxEnv, save, type Fixture, type ServerEvent } from "./driver";
|
||||
import type { TraceRecord } from "./trace";
|
||||
|
||||
export type Line = { k: string; who: string; why: string; r: number; e: number };
|
||||
|
||||
/** The status line's MCP text, from the last lines of the pane (menu.zig statusLine). */
|
||||
export function parseStatus(pane: string): Line | null {
|
||||
const tail = pane.split("\n").filter((l) => l.trim()).slice(-4).join("\n");
|
||||
let m: RegExpExecArray | null;
|
||||
if ((m = /Signing in to (\S+) · \/mcp/.exec(tail))) return { k: "signing", who: m[1]!, why: "", r: 0, e: 0 };
|
||||
if ((m = /(\S+) is waiting for approval · \/mcp/.exec(tail))) return { k: "attn", who: m[1]!, why: "waiting", r: 0, e: 0 };
|
||||
if ((m = /(\S+) needs login · \/mcp/.exec(tail))) return { k: "attn", who: m[1]!, why: "login", r: 0, e: 0 };
|
||||
if ((m = /(\S+) failed · \/mcp/.exec(tail))) return { k: "attn", who: m[1]!, why: "failed", r: 0, e: 0 };
|
||||
if ((m = /(\d+) MCP servers need you · \/mcp/.exec(tail))) return { k: "many", who: "", why: "", r: Number(m[1]), e: 0 };
|
||||
if ((m = /\bmcp (\d+)\/(\d+)\b/.exec(tail))) return { k: "count", who: "", why: "", r: Number(m[1]), e: Number(m[2]) };
|
||||
return null;
|
||||
}
|
||||
|
||||
export class Shell {
|
||||
records: TraceRecord[] = [];
|
||||
panes: { at: number; label: string; pane: string }[] = [];
|
||||
gateway: { stop(): void; baseUrl: string; chatUrl: string } | null = null;
|
||||
tui: any = null;
|
||||
constructor(readonly f: Fixture) {}
|
||||
|
||||
async start(extra: Record<string, string> = {}) {
|
||||
const { TmuxSession, startFakeGateway } = await targetHelpers();
|
||||
this.gateway = startFakeGateway([], { models: [{ id: "openai/gpt-5", type: "language", tags: ["tool-use"] }] });
|
||||
this.tui = await TmuxSession.create({
|
||||
cmd: shellQuote(binary), cwd: this.f.workspace, isolated: true, width: 160, height: 40, remainOnExit: true,
|
||||
stderrPath: join(this.f.base, "stderr.txt"),
|
||||
env: fxEnv(this.f, { FX_GATEWAY_BASE_URL: this.gateway!.baseUrl, FX_GATEWAY_CHAT_URL: this.gateway!.chatUrl, FX_MODEL: "openai/gpt-5", ...extra }),
|
||||
});
|
||||
await this.tui.waitForComposer(20_000);
|
||||
}
|
||||
|
||||
async snap(label: string) {
|
||||
const pane = await this.tui.capturePane();
|
||||
this.panes.push({ at: Date.now(), label, pane });
|
||||
return pane;
|
||||
}
|
||||
|
||||
/** Waits until the status line has read `want` (or anything, without one) for `stableMs`, and records it. */
|
||||
async status(want: Partial<Line> | null, timeoutMs = 20_000, stableMs = 800): Promise<Line | null> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let last: string | null = null, since = Date.now(), line: Line | null = null;
|
||||
while (Date.now() < deadline) {
|
||||
line = parseStatus(await this.tui.capturePane());
|
||||
const key = JSON.stringify(line);
|
||||
if (key !== last) { last = key; since = Date.now(); }
|
||||
const fits = want === null || (line !== null && Object.entries(want).every(([k, v]) => (line as never)[k] === v));
|
||||
if (fits && line && Date.now() - since >= stableMs) break;
|
||||
await Bun.sleep(100);
|
||||
}
|
||||
await this.snap(`status ${JSON.stringify(want)}`);
|
||||
if (line && line.k !== "signing") this.records.push({ event: "status", data: line, at: Date.now() });
|
||||
if (line && line.k === "signing") this.records.push({ event: "status", data: { signing: true }, at: Date.now(), note: line.who });
|
||||
return line;
|
||||
}
|
||||
|
||||
async type(text: string, record?: TraceRecord) {
|
||||
if (record) this.records.push({ ...record, at: Date.now() });
|
||||
await this.tui.sendText(text);
|
||||
}
|
||||
|
||||
async quit(): Promise<number> {
|
||||
this.records.push({ event: "quit", at: Date.now() });
|
||||
const started = Date.now();
|
||||
await this.tui.sendKeys("C-c");
|
||||
await Bun.sleep(300);
|
||||
const second = Date.now();
|
||||
await this.tui.sendKeys("C-c");
|
||||
while (!this.tui.paneStatus().dead && Date.now() - started < 10_000) await Bun.sleep(20);
|
||||
return Date.now() - second;
|
||||
}
|
||||
|
||||
async stop() {
|
||||
try { if (this.tui) { save(join(this.f.base, "final-pane.txt"), await this.tui.capturePane().catch(() => "")); await this.tui.kill(); } } catch {}
|
||||
this.gateway?.stop();
|
||||
save(join(this.f.base, "panes.json"), this.panes);
|
||||
save(join(this.f.base, "records-harness.json"), this.records);
|
||||
}
|
||||
}
|
||||
|
||||
/** Fixture log lines as model records: exits after quit are its SIGTERM's. */
|
||||
export function serverRecords(events: ServerEvent[], quitAt: number | null): TraceRecord[] {
|
||||
const out: TraceRecord[] = [];
|
||||
for (const e of events) {
|
||||
const data = { server: e.name };
|
||||
if (e.ev === "start") out.push({ event: "spawn", data, at: e.t, note: `pid ${e.pid}` });
|
||||
else if (e.ev === "listed") out.push({ event: "ready", data, at: e.t, note: `pid ${e.pid}` });
|
||||
else if (e.ev === "eof" || e.ev === "sigterm") {
|
||||
const afterQuit = quitAt !== null && e.t >= quitAt;
|
||||
out.push({ event: afterQuit ? "term" : "stop", data, at: e.t, note: `pid ${e.pid} ${e.ev}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
// SignIn: real sign-ins against MCP-v2's fake Streamable HTTP server with
|
||||
// fake OAuth (../mcp/fakes/http.ts): discovery,
|
||||
// registration, an authorization URL the harness follows itself (no browser),
|
||||
// the loopback callback, and the code redemption the fake logs. Checked
|
||||
// against SignIn.tla through SignInTrace.tla.
|
||||
import { expect, test } from "bun:test";
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { artifacts, binary } from "../lib/context";
|
||||
import { BUN, HTTP_FAKE, fixture, fxEnv, probe, save, stdioServer, waitFor, writeProfile, type Fixture } from "./driver";
|
||||
import { Shell } from "./shell";
|
||||
import { checkTrace, type TraceRecord } from "./trace";
|
||||
|
||||
// Tests marked NEXT_MCP check the MCP host fixes from the next MCP-v2
|
||||
// version, which src/core/mcp_host/ does not have yet. They run only with
|
||||
// FX_TLA_NEXT_MCP=1.
|
||||
const NEXT_MCP = process.env.FX_TLA_NEXT_MCP === "1";
|
||||
|
||||
|
||||
const MODERN = "2026-07-28";
|
||||
const RULES = [
|
||||
{ method: "server/discover", result: { resultType: "complete", supportedVersions: [MODERN], capabilities: { tools: {} } } },
|
||||
{ method: "tools/list", result: { resultType: "complete", tools: [{ name: "echo", inputSchema: { type: "object" } }] } },
|
||||
];
|
||||
|
||||
async function startFake(f: Fixture) {
|
||||
if (!existsSync(HTTP_FAKE)) throw new Error(`harness: MCP-v2's fake HTTP server is missing at ${HTTP_FAKE}`);
|
||||
const rules = join(f.base, "rules.json"), auth = join(f.base, "auth.json"), log = join(f.base, "http-fake.jsonl");
|
||||
writeFileSync(rules, JSON.stringify(RULES));
|
||||
writeFileSync(auth, JSON.stringify({ servers: [{}], challengeScope: "mcp:read" }));
|
||||
const child = Bun.spawn([BUN, HTTP_FAKE, "--script", rules, "--log", log, "--auth", auth], { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
||||
const reader = child.stdout.getReader();
|
||||
let text = "";
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (!text.includes("\n") && Date.now() < deadline) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
text += new TextDecoder().decode(value);
|
||||
}
|
||||
const port = JSON.parse(text.split("\n")[0]!).listening as number;
|
||||
const entries = () => (existsSync(log) ? readFileSync(log, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) : []) as { t: number; dir?: string; path?: string; form?: Record<string, string>; json?: { redirect_uris?: string[] } }[];
|
||||
return { url: `http://127.0.0.1:${port}/mcp`, entries, stop: () => child.kill() };
|
||||
}
|
||||
|
||||
const redemptions = (entries: ReturnType<Awaited<ReturnType<typeof startFake>>["entries"]>) =>
|
||||
entries.filter((e) => e.dir === "auth" && e.path?.endsWith("/token") && e.form?.grant_type === "authorization_code");
|
||||
|
||||
/** `fx mcp login --no-browser web` with stdin left open, its output read as it comes. */
|
||||
async function cliLogin(f: Fixture) {
|
||||
const child = Bun.spawn([binary, "mcp", "login", "--no-browser", "web"], { cwd: f.workspace, env: fxEnv(f), stdin: "pipe", stdout: "pipe", stderr: "pipe" });
|
||||
let out = "", err = "";
|
||||
void (async () => { for await (const c of child.stdout) out += new TextDecoder().decode(c); })();
|
||||
void (async () => { for await (const c of child.stderr) err += new TextDecoder().decode(c); })();
|
||||
const killer = setTimeout(() => child.kill("SIGKILL"), 60_000);
|
||||
return {
|
||||
child, out: () => out, err: () => err,
|
||||
async done() { const code = await child.exited; clearTimeout(killer); await Bun.sleep(50); return code; },
|
||||
};
|
||||
}
|
||||
|
||||
async function signInScenario(name: string, tamper: boolean) {
|
||||
const f = fixture(artifacts, name);
|
||||
const fake = await startFake(f);
|
||||
writeProfile(f, { web: { type: "http", url: fake.url } });
|
||||
const records: TraceRecord[] = [];
|
||||
const at = () => Date.now();
|
||||
try {
|
||||
records.push({ event: "login", at: at() });
|
||||
const login = await cliLogin(f);
|
||||
const url = await waitFor("the authorization URL", () => /\n\s+(http\S+)\n/.exec(login.out())?.[1], 30_000);
|
||||
records.push({ event: "authorize", data: { login: 1 }, at: at(), note: url });
|
||||
const authorize = new URL(url);
|
||||
const port = Number(new URL(authorize.searchParams.get("redirect_uri")!).port);
|
||||
const before = await probe(port);
|
||||
expect(before).toEqual({ result: "open", status: 404 });
|
||||
records.push({ event: "probe", data: { login: 1, result: "open" }, at: at() });
|
||||
// A form posted to the loopback listener (not a bridged sign-in) is refused, and the wait goes on.
|
||||
records.push({ event: "callback", data: { login: 1, state: 0, kind: "form_foreign" }, at: at() });
|
||||
const form = await fetch(`http://127.0.0.1:${port}/callback`, { method: "POST", headers: { "content-type": "application/x-www-form-urlencoded" }, body: "code=x&state=y" });
|
||||
expect(form.status).toBe(404);
|
||||
const redirect = await fetch(url, { redirect: "manual" });
|
||||
const location = new URL(redirect.headers.get("location")!);
|
||||
const state = location.searchParams.get("state");
|
||||
expect(state).toBe(authorize.searchParams.get("state"));
|
||||
if (tamper) location.searchParams.set("state", "not-the-state-fx-made");
|
||||
// Recorded when sent: fx answers the page and redeems before this fetch returns.
|
||||
records.push({ event: "callback", data: { login: 1, state: tamper ? 0 : 1, kind: "get" }, at: at() });
|
||||
const back = await fetch(location.toString());
|
||||
save(join(f.base, "callback.json"), { location: location.toString(), status: back.status, body: (await back.text()).slice(0, 400) });
|
||||
const code = await login.done();
|
||||
const ended = at();
|
||||
const tokens = redemptions(fake.entries());
|
||||
for (const t of tokens) records.push({ event: "redeem", data: { login: 1 }, at: Math.min(t.t, ended - 1) });
|
||||
records.push({ event: code === 0 ? "signed_in" : "failed", data: { login: 1 }, at: ended });
|
||||
const after = await probe(port);
|
||||
records.push({ event: "probe", data: { login: 1, result: after.result }, at: at() });
|
||||
save(join(f.base, "login-output.json"), { code, stdout: login.out(), stderr: login.err() });
|
||||
save(join(f.base, "http-fake-entries.json"), fake.entries());
|
||||
records.sort((a, b) => (a.at ?? 0) - (b.at ?? 0));
|
||||
const verdict = await checkTrace({ module: "SignIn", records, dir: join(f.base, "trace"), constants: { ObsMaxLogins: 1, ObsMaxHosts: 1, ObsMaxLeases: 0, ObsBridge: false } });
|
||||
save(join(f.base, "verdict.json"), verdict);
|
||||
return { f, code, tokens, verdict, after, out: login.out(), err: login.err() };
|
||||
} finally { fake.stop(); }
|
||||
}
|
||||
|
||||
test("control: a sign-in redeems the code of the callback whose state matches, then its listener closes (trace ACCEPTED)", async () => {
|
||||
const r = await signInScenario("signin-ok", false);
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.out).toContain("Signed in to web.");
|
||||
expect(r.tokens.length).toBe(1);
|
||||
expect(r.after.result).toBe("closed");
|
||||
expect(readFileSync(join(r.f.home, ".fx", "mcp-credentials", "credentials-v2.json"), "utf8")).toContain("\"name\":\"web\"");
|
||||
expect(r.verdict.verdict).toBe("ACCEPTED");
|
||||
}, 120_000);
|
||||
|
||||
test("control: a callback whose state isn't the live sign-in's is never redeemed (trace ACCEPTED)", async () => {
|
||||
const r = await signInScenario("signin-wrong-state", true);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toContain("Couldn't sign in to web");
|
||||
expect(r.tokens).toEqual([]);
|
||||
expect(r.verdict.verdict).toBe("ACCEPTED");
|
||||
}, 120_000);
|
||||
|
||||
// Fixed by stopping the sign-in and saying so. Before the fix this case was
|
||||
// a finding: the shell kept saying "Signing in" and the trace broke
|
||||
// IndicatorImpliesListener. It fails that way on the code before.
|
||||
test.skipIf(!NEXT_MCP)("contract: a reload during a sign-in closes its listener, and the shell stops saying \"Signing in\" and says why (trace ACCEPTED)", async () => {
|
||||
const f = fixture(artifacts, "signin-reload");
|
||||
const fake = await startFake(f);
|
||||
writeProfile(f, { web: { type: "http", url: fake.url } });
|
||||
const shell = new Shell(f);
|
||||
const records: TraceRecord[] = [];
|
||||
try {
|
||||
await shell.start();
|
||||
records.push({ event: "login", at: Date.now() });
|
||||
await shell.tui.sendText("/mcp login web");
|
||||
await shell.tui.waitForText("Sign in to web in your browser", 20_000);
|
||||
records.push({ event: "authorize", data: { login: 1 }, at: Date.now() });
|
||||
const signing = await shell.status({ k: "signing" }, 10_000);
|
||||
expect(signing?.k).toBe("signing");
|
||||
records.push({ event: "status", data: { signing: true }, at: Date.now() });
|
||||
const registered = await waitFor("the registration's redirect URI", () => fake.entries().find((e) => e.path?.endsWith("/register"))?.json?.redirect_uris?.[0], 10_000);
|
||||
const port = Number(new URL(registered).port);
|
||||
expect((await probe(port)).result).toBe("open");
|
||||
records.push({ event: "probe", data: { login: 1, result: "open" }, at: Date.now() });
|
||||
records.push({ event: "reload", at: Date.now() });
|
||||
await shell.tui.sendText(`/mcp add other -- ${BUN} ${stdioServer(f, "other").args[0]}`);
|
||||
await shell.tui.waitForText("Added other", 10_000);
|
||||
let after = await probe(port);
|
||||
for (let k = 0; k < 30 && after.result === "open"; k++) { await Bun.sleep(100); after = await probe(port); }
|
||||
records.push({ event: "probe", data: { login: 1, result: after.result }, at: Date.now() });
|
||||
await Bun.sleep(3_000);
|
||||
const still = await shell.status(null, 5_000);
|
||||
records.push({ event: "status", data: { signing: still?.k === "signing" }, at: Date.now(), note: JSON.stringify(still) });
|
||||
save(join(f.base, "after-reload.json"), { probe: after, status: still });
|
||||
const verdict = await checkTrace({ module: "SignIn", records, dir: join(f.base, "trace"), constants: { ObsMaxLogins: 1, ObsMaxHosts: 2, ObsMaxLeases: 1, ObsBridge: false } });
|
||||
save(join(f.base, "verdict.json"), verdict);
|
||||
expect(after.result).toBe("closed");
|
||||
expect(still?.k).not.toBe("signing");
|
||||
expect(await shell.tui.captureFullScrollback()).toContain("Couldn't sign in to web: the MCP config changed before the browser came back; run /mcp login web again.");
|
||||
expect(verdict.verdict).toBe("ACCEPTED");
|
||||
await shell.quit();
|
||||
} finally {
|
||||
await shell.stop();
|
||||
save(join(f.base, "http-fake-entries.json"), fake.entries());
|
||||
fake.stop();
|
||||
}
|
||||
}, 150_000);
|
||||
@@ -0,0 +1,93 @@
|
||||
// TLC for the MCP host models in tests/tla/mcp_host/: the TLC that ../lib/tlc.ts
|
||||
// finds, one run per call in a fresh work folder, and the run's verdict. Nothing
|
||||
// here downloads anything or calls a model.
|
||||
import { closeSync, cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { join, resolve } from "node:path";
|
||||
import { TLA_ROOT, tlcToolchain } from "../lib/tlc";
|
||||
|
||||
export const here = import.meta.dirname;
|
||||
export const tlaDir = join(TLA_ROOT, "mcp_host");
|
||||
|
||||
export const sha256 = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
export const sha256Text = (text: string) => createHash("sha256").update(text).digest("hex");
|
||||
|
||||
export function tools() {
|
||||
return tlcToolchain();
|
||||
}
|
||||
|
||||
export interface TlcRun {
|
||||
label: string;
|
||||
module: string;
|
||||
config: string;
|
||||
command: string[];
|
||||
exit: number;
|
||||
seconds: number;
|
||||
log: string;
|
||||
output: string;
|
||||
generated?: number;
|
||||
distinct?: number;
|
||||
queued?: number;
|
||||
violated?: string;
|
||||
temporal: boolean;
|
||||
completed: boolean;
|
||||
}
|
||||
|
||||
/** Runs TLC on `module` with `cfgText` in a fresh folder under `outDir`; `files` replaces model files (planted bugs, TraceObs). */
|
||||
export async function runTlc(opts: {
|
||||
outDir: string; label: string; module: string; cfgText: string;
|
||||
files?: Record<string, string>; extraArgs?: string[]; workers?: number; timeoutMs?: number;
|
||||
}): Promise<TlcRun> {
|
||||
const { java, jar } = tools();
|
||||
const work = join(opts.outDir, opts.label);
|
||||
mkdirSync(join(work, "tmp"), { recursive: true, mode: 0o700 });
|
||||
for (const name of readdirSync(tlaDir)) if (/\.(tla|cfg)$/.test(name)) cpSync(join(tlaDir, name), join(work, name));
|
||||
for (const [name, text] of Object.entries(opts.files ?? {})) writeFileSync(join(work, name), text);
|
||||
const cfgName = `${opts.label}.run.cfg`;
|
||||
writeFileSync(join(work, cfgName), opts.cfgText);
|
||||
const command = [java, "-Xmx3g", "-XX:+UseParallelGC", `-Djava.io.tmpdir=${join(work, "tmp")}`, "-cp", jar, "tlc2.TLC",
|
||||
"-workers", String(opts.workers ?? 4), "-metadir", join(work, "states"), "-config", cfgName, ...(opts.extraArgs ?? []), `${opts.module}.tla`];
|
||||
const log = join(work, "tlc.log");
|
||||
const fd = openSync(log, "w", 0o600);
|
||||
const started = Date.now();
|
||||
let exit: number;
|
||||
try {
|
||||
const child = Bun.spawn(command, { cwd: work, stdin: "ignore", stdout: fd, stderr: fd });
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), opts.timeoutMs ?? 600_000);
|
||||
exit = await child.exited;
|
||||
clearTimeout(timer);
|
||||
} finally { closeSync(fd); }
|
||||
const output = readFileSync(log, "utf8");
|
||||
const stats = /([\d,]+) states generated, ([\d,]+) distinct states found, ([\d,]+) states left on queue/.exec(output);
|
||||
const num = (s?: string) => (s === undefined ? undefined : Number(s.replaceAll(",", "")));
|
||||
return {
|
||||
label: opts.label, module: opts.module, config: cfgName, command, exit, seconds: (Date.now() - started) / 1000, log, output,
|
||||
generated: num(stats?.[1]), distinct: num(stats?.[2]), queued: num(stats?.[3]),
|
||||
violated: /Invariant (\w+) is violated/.exec(output)?.[1],
|
||||
temporal: output.includes("Temporal properties were violated"),
|
||||
completed: output.includes("Model checking completed. No error has been found."),
|
||||
};
|
||||
}
|
||||
|
||||
/** Config text keeping the invariants, with PROPERTY limited to `keep` (or none). */
|
||||
export function configWith(cfgText: string, change: { onlyProperty?: string; invariants?: string[] }): string {
|
||||
const lines = cfgText.split("\n");
|
||||
const out: string[] = [];
|
||||
let section = "";
|
||||
for (const raw of lines) {
|
||||
const words = raw.split("\\*")[0]!.trim().split(/\s+/).filter(Boolean);
|
||||
if (words.length && /^[A-Z]+$/.test(words[0]!) && ["CONSTANTS", "CONSTANT", "SPECIFICATION", "INIT", "NEXT", "INVARIANT", "INVARIANTS", "PROPERTY", "PROPERTIES"].includes(words[0]!)) {
|
||||
section = words[0]!;
|
||||
if (section.startsWith("PROPERT") && change.onlyProperty !== undefined) continue;
|
||||
if (section.startsWith("INVARIANT") && change.invariants !== undefined) continue;
|
||||
out.push(raw);
|
||||
continue;
|
||||
}
|
||||
if (section.startsWith("PROPERT") && change.onlyProperty !== undefined) continue;
|
||||
if (section.startsWith("INVARIANT") && change.invariants !== undefined) continue;
|
||||
out.push(raw);
|
||||
}
|
||||
if (change.invariants?.length) out.push("INVARIANTS", ...change.invariants.map((n) => ` ${n}`));
|
||||
if (change.onlyProperty) out.push(`PROPERTY ${change.onlyProperty}`);
|
||||
return out.join("\n") + "\n";
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
// The trace bridge: turns what a scenario saw of real fx into TraceObs.tla
|
||||
// and checks it against a model's trace wrapper, MCP-v2's way (sentinel
|
||||
// invariant: violated exactly when a complete match exists).
|
||||
//
|
||||
// Verdicts:
|
||||
// ACCEPTED a complete match exists that keeps every stated property
|
||||
// VIOLATES every complete match breaks a stated property (`broken`
|
||||
// names them, from a match TLC prints)
|
||||
// REJECTED no complete match: the real behavior leaves the model;
|
||||
// `firstBad` is the first record no prefix match can take
|
||||
// INCONCLUSIVE an empty trace
|
||||
// TOOL_ERROR anything else (a parse error, an unexpected exit)
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { runTlc, sha256Text } from "./tlc";
|
||||
|
||||
export type Value = string | number | boolean | Value[] | { [k: string]: Value };
|
||||
export interface TraceRecord { event: string; data?: { [k: string]: Value }; at?: number; note?: string }
|
||||
export interface Verdict {
|
||||
module: string; verdict: "ACCEPTED" | "VIOLATES" | "REJECTED" | "INCONCLUSIVE" | "TOOL_ERROR";
|
||||
records: number; broken?: string[]; firstBad?: { index: number; record: TraceRecord }; detail?: string;
|
||||
dir: string; traceSha256: string; runs: { label: string; exit: number; violated?: string; log: string; distinct?: number }[];
|
||||
}
|
||||
|
||||
/** TLA+ for a JSON value: arrays are sets; objects are records. */
|
||||
export function tla(value: Value): string {
|
||||
if (typeof value === "string") return JSON.stringify(value);
|
||||
if (typeof value === "boolean") return value ? "TRUE" : "FALSE";
|
||||
if (typeof value === "number") {
|
||||
if (!Number.isSafeInteger(value) || Math.abs(value) >= 2 ** 31) throw new Error(`TLC integers are 32-bit: ${value}`);
|
||||
return String(value);
|
||||
}
|
||||
if (Array.isArray(value)) return `{${value.map(tla).join(", ")}}`;
|
||||
const keys = Object.keys(value);
|
||||
if (keys.length === 0) return "[none |-> 0]";
|
||||
return `[${keys.map((k) => `${k} |-> ${tla(value[k]!)}`).join(", ")}]`;
|
||||
}
|
||||
|
||||
export function traceObs(records: TraceRecord[], constants: Record<string, Value>): string {
|
||||
const lines = ["---- MODULE TraceObs ----", "EXTENDS Integers, Sequences, FiniteSets", ""];
|
||||
lines.push("ObsTrace == <<");
|
||||
lines.push(records.map((r) => ` [event |-> ${tla(r.event)}, data |-> ${tla(r.data ?? {})}]`).join(",\n"));
|
||||
lines.push(">>", "");
|
||||
for (const [name, value] of Object.entries(constants)) lines.push(`${name} == ${tla(value)}`);
|
||||
lines.push("====", "");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
const substitutions: Record<string, string[]> = {
|
||||
HostLifecycle: ["Servers", "Project", "Reachable", "NeedsAuth", "MaxHosts", "MaxLeases"],
|
||||
Credentials: ["Names", "Faults", "InitKinds", "InitEntries", "InitImported", "InitMarked", "InitV1"],
|
||||
SignIn: ["MaxLogins", "MaxHosts", "MaxLeases", "Bridge"],
|
||||
ElicitationRouting: ["Versions", "MaxQ", "Slots", "Guard"],
|
||||
};
|
||||
|
||||
function wrapperCfg(module: string, invariant: string): string {
|
||||
const names = substitutions[module];
|
||||
if (!names) throw new Error(`no trace wrapper for ${module}`);
|
||||
return ["CONSTANTS", ...names.map((n) => ` ${n} <- Obs${n}`), "INIT TraceInit", "NEXT TraceNext", `INVARIANT ${invariant}`, ""].join("\n");
|
||||
}
|
||||
|
||||
/** `constants` holds Obs<Name> for each wrapper constant. */
|
||||
export async function checkTrace(opts: { module: string; records: TraceRecord[]; constants: Record<string, Value>; dir: string }): Promise<Verdict> {
|
||||
const { module, records, constants, dir } = opts;
|
||||
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
const text = JSON.stringify(records, null, 2);
|
||||
writeFileSync(join(dir, "trace.json"), text);
|
||||
const verdict: Verdict = { module, verdict: "TOOL_ERROR", records: records.length, dir, traceSha256: sha256Text(text), runs: [] };
|
||||
const finish = (v: Partial<Verdict>) => {
|
||||
Object.assign(verdict, v);
|
||||
writeFileSync(join(dir, "verdict.json"), JSON.stringify(verdict, null, 2));
|
||||
return verdict;
|
||||
};
|
||||
if (records.length === 0) return finish({ verdict: "INCONCLUSIVE", detail: "empty trace" });
|
||||
for (const n of substitutions[module] ?? []) if (!(`Obs${n}` in constants)) return finish({ detail: `missing constant Obs${n}` });
|
||||
const run = async (label: string, recs: TraceRecord[], invariant: string) => {
|
||||
const r = await runTlc({
|
||||
outDir: dir, label, module: `${module}Trace`, cfgText: wrapperCfg(module, invariant),
|
||||
files: { "TraceObs.tla": traceObs(recs, constants) }, extraArgs: ["-deadlock"], workers: 2, timeoutMs: 300_000,
|
||||
});
|
||||
verdict.runs.push({ label, exit: r.exit, violated: r.violated, log: r.log, distinct: r.distinct });
|
||||
return r;
|
||||
};
|
||||
const matched = (r: Awaited<ReturnType<typeof run>>, inv: string) => r.exit === 12 && r.violated === inv;
|
||||
const unmatched = (r: Awaited<ReturnType<typeof run>>) => r.exit === 0 && r.completed;
|
||||
|
||||
const whole = await run("match", records, "NotDone");
|
||||
if (unmatched(whole)) {
|
||||
// The longest prefix that matches; the record after it is the first the model can't take.
|
||||
let lo = 0, hi = records.length - 1;
|
||||
while (lo < hi) {
|
||||
const mid = Math.ceil((lo + hi) / 2);
|
||||
const r = mid === 0 ? null : await run(`prefix-${mid}`, records.slice(0, mid), "NotDone");
|
||||
if (r === null || matched(r, "NotDone")) lo = mid; else if (unmatched(r)) hi = mid - 1; else return finish({ detail: `prefix ${mid}: exit ${r.exit}` });
|
||||
}
|
||||
return finish({ verdict: "REJECTED", firstBad: { index: lo, record: records[lo]! } });
|
||||
}
|
||||
if (!matched(whole, "NotDone")) return finish({ detail: `exit ${whole.exit}: ${whole.output.split("\n").filter((l) => /Error|error|Unknown|not/.test(l)).slice(0, 5).join(" | ")}` });
|
||||
const clean = await run("clean", records, "NotDoneClean");
|
||||
if (matched(clean, "NotDoneClean")) return finish({ verdict: "ACCEPTED" });
|
||||
if (!unmatched(clean)) return finish({ detail: `clean: exit ${clean.exit}` });
|
||||
const dirty = await run("dirty", records, "NotDoneDirty");
|
||||
if (!matched(dirty, "NotDoneDirty")) return finish({ detail: `dirty: exit ${dirty.exit}` });
|
||||
const states = dirty.output.split(/^State \d+:/m);
|
||||
const last = states[states.length - 1] ?? "";
|
||||
const broken = [...(/\/\\ bad = \{([^}]*)\}/.exec(last)?.[1] ?? "").matchAll(/"(\w+)"/g)].map((m) => m[1]!);
|
||||
return finish({ verdict: "VIOLATES", broken });
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Connects runs of the real credential edge to TLC.
|
||||
//
|
||||
// 1. The run's events become AuthObserved.tla. AuthTrace replays them through
|
||||
// ProviderAuth's own Next under the scenario's constants. ACCEPTED needs TLC
|
||||
// exit 12 on the NotAllMatched sentinel; a completed search is REJECTED.
|
||||
// 2. After acceptance, each property is checked on the same replay.
|
||||
//
|
||||
// Unreadable, partial or anomalous evidence is never acceptance.
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { observedModule, type Run } from "./driver";
|
||||
import { completed, constantDefinitions, constantOverrides, hashes, PROPERTIES, runTlc, stage, violated } from "./tlc";
|
||||
|
||||
export type TraceVerdict = "ACCEPTED" | "REJECTED" | "INCONCLUSIVE" | "TOOL_ERROR";
|
||||
|
||||
export interface TraceResult {
|
||||
scenario: string;
|
||||
verdict: TraceVerdict;
|
||||
detail: string;
|
||||
events: number;
|
||||
properties: Record<string, "HOLDS" | "VIOLATED" | "NOT_CHECKED" | "TOOL_ERROR">;
|
||||
report: string;
|
||||
hashes: Record<string, string>;
|
||||
}
|
||||
|
||||
export async function checkTrace(run: Run, reportRoot: string): Promise<TraceResult> {
|
||||
mkdirSync(reportRoot, { recursive: true, mode: 0o700 });
|
||||
const root = mkdtempSync(join(reportRoot, `${run.scenario}-`));
|
||||
const properties: TraceResult["properties"] = Object.fromEntries(PROPERTIES.map((p) => [p, "NOT_CHECKED" as const]));
|
||||
const finish = (verdict: TraceVerdict, detail: string): TraceResult => {
|
||||
const result = { scenario: run.scenario, verdict, detail, events: run.events.length, properties, report: root, hashes: hashes() };
|
||||
writeFileSync(join(root, "verdict.json"), JSON.stringify(result, null, 2), { mode: 0o600 });
|
||||
return result;
|
||||
};
|
||||
writeFileSync(join(root, "run.json"), JSON.stringify(run, null, 2), { mode: 0o600 });
|
||||
if (!run.events.length) return finish("INCONCLUSIVE", "empty run");
|
||||
if (run.anomalies.length) return finish("INCONCLUSIVE", `driver anomalies: ${run.anomalies.join("; ")}`);
|
||||
|
||||
const observed = observedModule(run, constantDefinitions(run.setup));
|
||||
const cfg = (invariant: string) => `SPECIFICATION TraceSpec\nINVARIANT ${invariant}\nCONSTANTS\n${constantOverrides}\n`;
|
||||
const conformance = await runTlc(stage(root, "conformance", { "AuthObserved.tla": observed }), "AuthTrace.tla", cfg("NotAllMatched"));
|
||||
if (!(conformance.exitCode === 12 && violated(conformance.log, "NotAllMatched"))) {
|
||||
if (conformance.exitCode === 0 && completed(conformance.log)) {
|
||||
return finish("REJECTED", `the run diverges from the model; see ${join(conformance.dir, "tlc.log")}`);
|
||||
}
|
||||
return finish("TOOL_ERROR", `unexpected TLC exit ${conformance.exitCode}; see ${join(conformance.dir, "tlc.log")}`);
|
||||
}
|
||||
for (const property of PROPERTIES) {
|
||||
const check = await runTlc(stage(root, `property-${property}`, { "AuthObserved.tla": observed }), "AuthTrace.tla", cfg(property));
|
||||
properties[property] = check.exitCode === 0 && completed(check.log) ? "HOLDS"
|
||||
: check.exitCode === 12 && violated(check.log, property) ? "VIOLATED" : "TOOL_ERROR";
|
||||
}
|
||||
return finish("ACCEPTED", `all ${run.events.length} events match the model`);
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
// Runs a target worktree's real credential edge (the authorizer libfx's
|
||||
// transportFetch calls) through scripted scenarios, recording each event in
|
||||
// the vocabulary of ProviderAuth's `ev`. Every endpoint is a local fake; the
|
||||
// fakes follow the model's rules: the issuer rotates refresh tokens and
|
||||
// refuses spent ones, the provider rejects the host's first token when the
|
||||
// scenario says so, and userinfo answers with a token's real account.
|
||||
import { existsSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { Setup } from "./tlc";
|
||||
|
||||
export type AuthEvent =
|
||||
| { k: "ask"; r: string }
|
||||
| { k: "bypass"; r: string }
|
||||
| { k: "redeem"; e: string; rt: number; ok: boolean; acct: string }
|
||||
| { k: "verify"; e: string; at: number; acct: string }
|
||||
| { k: "save"; e: string; at: number }
|
||||
| { k: "fail"; e: string; rt: number }
|
||||
| { k: "send"; r: string; at: number; status: number };
|
||||
|
||||
export const NO_TOKEN = 999;
|
||||
|
||||
export interface Run {
|
||||
scenario: string;
|
||||
setup: Setup;
|
||||
events: AuthEvent[];
|
||||
outcomes: Record<string, string>;
|
||||
/** Anything the driver saw that the model has no event for. */
|
||||
anomalies: string[];
|
||||
}
|
||||
|
||||
type Provider = Setup["provider"];
|
||||
|
||||
/** Which providers the worktree's SDK has, and how to reach its authorizer. */
|
||||
export async function loadEdge(worktree: string) {
|
||||
const sdk = join(worktree, "sdk");
|
||||
if (existsSync(join(sdk, "subscription.js"))) {
|
||||
const { createSubscriptionAuthorizer, createCredentialMemory } = await import(join(sdk, "subscription.js"));
|
||||
const { codexSubscription } = await import(join(sdk, "chatgpt.js"));
|
||||
const { grokSubscription } = await import(join(sdk, "grok.js"));
|
||||
const custom = existsSync(join(sdk, "custom-provider.js")) ? await import(join(sdk, "custom-provider.js")) : null;
|
||||
return {
|
||||
shape: "subscription" as const,
|
||||
providers: ["codex", "grok", ...(custom ? ["custom"] : [])] as Provider[],
|
||||
/** A memory engines can share, when this worktree has one (the follow-up fix). */
|
||||
memory: createCredentialMemory as (() => unknown) | undefined,
|
||||
create(provider: Provider, options: { credentials: Function; onCredentials: Function; fetch: Function; memory?: unknown; refreshLock?: Function }, sessionId: () => string | undefined) {
|
||||
const specs = provider === "custom"
|
||||
? [custom.customProviderSpec(custom.customProviderDefinition({ name: "local", baseUrl: "https://models.example/v1" }), sessionId)]
|
||||
: [codexSubscription, grokSubscription];
|
||||
const auth = createSubscriptionAuthorizer(specs, provider === "custom" ? { credentials: options.credentials, fetch: options.fetch } : options);
|
||||
return (endpoint: string, hostFetch: Function, input: string, init: RequestInit) => {
|
||||
const spec = auth.specFor(endpoint);
|
||||
return spec ? { authorized: true, response: auth.fetch(spec, endpoint, hostFetch, input, init) } : { authorized: false, response: hostFetch(input, init) };
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
const { createChatgptAuthorizer, isCodexRequest } = await import(join(sdk, "chatgpt.js"));
|
||||
// Before #1210 the authorizer took the credentials callback as `chatgpt`.
|
||||
const legacy = (await Bun.file(join(sdk, "chatgpt.js")).text()).includes("createChatgptAuthorizer({ chatgpt,");
|
||||
return {
|
||||
shape: legacy ? ("chatgpt-legacy" as const) : ("chatgpt" as const),
|
||||
providers: ["codex"] as Provider[],
|
||||
memory: undefined as (() => unknown) | undefined,
|
||||
create(_provider: Provider, options: { credentials: Function; onCredentials: Function; fetch: Function }) {
|
||||
const auth = createChatgptAuthorizer(legacy
|
||||
? { chatgpt: options.credentials, onCredentials: options.onCredentials, fetch: options.fetch }
|
||||
: options);
|
||||
return (endpoint: string, hostFetch: Function, input: string, init: RequestInit) =>
|
||||
isCodexRequest(endpoint) ? { authorized: true, response: auth.fetch(hostFetch, input, init) } : { authorized: false, response: hostFetch(input, init) };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const endpoints: Record<Provider, { provider: string; other: string; token?: string; userinfo?: string }> = {
|
||||
codex: { provider: "https://chatgpt.com/backend-api/codex/responses", other: "https://chatgpt.com/backend-api/other", token: "https://auth.openai.com/oauth/token" },
|
||||
grok: { provider: "https://cli-chat-proxy.grok.com/v1/responses", other: "https://cli-chat-proxy.grok.com/v1/other", token: "https://auth.x.ai/oauth2/token", userinfo: "https://auth.x.ai/oauth2/userinfo" },
|
||||
custom: { provider: "https://models.example/v1/chat/completions", other: "https://models.example/v1/models" },
|
||||
};
|
||||
|
||||
const part = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||
const lifetime = (fresh: boolean) => (fresh ? 10 * 24 * 3600 * 1000 : 60 * 1000);
|
||||
|
||||
/** Token n as each provider carries it. */
|
||||
function accessToken(provider: Provider, n: number, acct: string, fresh: boolean): string {
|
||||
if (provider === "codex") {
|
||||
const exp = Math.floor((Date.now() + lifetime(fresh)) / 1000);
|
||||
return `${part({ alg: "none" })}.${part({ exp, "https://api.openai.com/auth": { chatgpt_account_id: `acct-${acct}` }, n })}.sig`;
|
||||
}
|
||||
return provider === "grok" ? `at-${n}` : `sk-${n}`;
|
||||
}
|
||||
|
||||
function tokenNumber(provider: Provider, token: string | null | undefined): number {
|
||||
if (!token) return NO_TOKEN;
|
||||
const raw = token.replace(/^Bearer /, "");
|
||||
if (provider === "codex") {
|
||||
try { return JSON.parse(Buffer.from(raw.split(".")[1]!, "base64url").toString()).n; } catch { return -1; }
|
||||
}
|
||||
const m = /^(?:at|sk)-(\d+)$/.exec(raw);
|
||||
return m ? Number(m[1]) : -1;
|
||||
}
|
||||
|
||||
function hostCredentials(provider: Provider, n: number, acct: string, fresh: boolean) {
|
||||
if (provider === "custom") return { apiKey: accessToken(provider, n, acct, fresh) };
|
||||
const accessTokenValue = accessToken(provider, n, acct, fresh);
|
||||
return provider === "codex"
|
||||
? { accessToken: accessTokenValue, refreshToken: `rt-${n}` }
|
||||
: { accessToken: accessTokenValue, refreshToken: `rt-${n}`, accountId: `acct-${acct}`, expiresAt: Date.now() + lifetime(fresh) };
|
||||
}
|
||||
|
||||
const bodyText = (init: RequestInit = {}) =>
|
||||
init.body === undefined || init.body === null ? "" : typeof init.body === "string" ? init.body : new TextDecoder().decode(init.body as Uint8Array);
|
||||
|
||||
/** A gate a scenario opens to let a held response through. */
|
||||
function gate() {
|
||||
let open!: () => void;
|
||||
const opened = new Promise<void>((resolve) => { open = resolve; });
|
||||
return { opened, open };
|
||||
}
|
||||
|
||||
export interface World {
|
||||
/** Starts request `r` (in `setup.requests`); resolves to its outcome. */
|
||||
send(r: string): Promise<string>;
|
||||
/** Holds the provider's answer to `r` until `release(r)`. */
|
||||
hold(r: string): void;
|
||||
release(r: string): void;
|
||||
/** Resolves once `r`'s request has reached the provider. */
|
||||
arrived(r: string): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `script` against fresh fakes and one real authorizer per engine.
|
||||
* `accounts` lists the account each successful redemption issues, in order.
|
||||
*/
|
||||
export async function runScenario(
|
||||
worktree: string,
|
||||
scenario: string,
|
||||
setup: Setup,
|
||||
script: (world: World) => Promise<void>,
|
||||
options: { accounts?: string[] } = {},
|
||||
): Promise<Run> {
|
||||
const edge = await loadEdge(worktree);
|
||||
if (!edge.providers.includes(setup.provider)) throw new Error(`${setup.provider} is not in this worktree`);
|
||||
const { provider } = setup;
|
||||
const events: AuthEvent[] = [];
|
||||
const anomalies: string[] = [];
|
||||
const outcomes: Record<string, string> = {};
|
||||
const accounts = [...(options.accounts ?? [])];
|
||||
const spent = new Set<number>();
|
||||
const acctOf = new Map<number, string>([[0, "A"]]);
|
||||
let issued = 0;
|
||||
let host = hostCredentials(provider, 0, "A", !setup.expiring);
|
||||
let calling: string | undefined;
|
||||
const holds = new Map<string, ReturnType<typeof gate>>();
|
||||
const arrivals = new Map<string, ReturnType<typeof gate>>();
|
||||
const arrival = (r: string) => { if (!arrivals.has(r)) arrivals.set(r, gate()); return arrivals.get(r)!; };
|
||||
// The redemption a failing refresh task ran, per engine, until its failure is recorded.
|
||||
const failing = new Map<string, number>();
|
||||
const ep = endpoints[provider];
|
||||
|
||||
const authorizers = new Map<string, ReturnType<typeof edge.create>>();
|
||||
if (setup.sharedMemory && !edge.memory) throw new Error("this worktree has no shared credential memory");
|
||||
const shared = setup.sharedMemory ? edge.memory!() : undefined;
|
||||
// One lock for every engine, as servers sharing a host lock would have.
|
||||
let held: Promise<unknown> = Promise.resolve();
|
||||
const refreshLock = (_info: unknown, refresh: () => Promise<void>) => {
|
||||
const run = held.then(refresh);
|
||||
held = run.catch(() => {});
|
||||
return run;
|
||||
};
|
||||
for (const engine of new Set(setup.requests.map((r) => r.engine))) {
|
||||
const fetch = async (input: string | URL | Request, init: RequestInit = {}) => {
|
||||
const url = String((input as Request)?.url ?? input);
|
||||
if (url === ep.token) {
|
||||
const body = bodyText(init);
|
||||
const raw = provider === "codex" ? JSON.parse(body).refresh_token : new URLSearchParams(body).get("refresh_token");
|
||||
const rt = Number(/^rt-(\d+)$/.exec(String(raw))?.[1] ?? -1);
|
||||
if (spent.has(rt) || rt < 0 || rt > issued) {
|
||||
events.push({ k: "redeem", e: engine, rt, ok: false, acct: "none" });
|
||||
failing.set(engine, rt);
|
||||
return Response.json({ error: "invalid_grant", error_description: "refresh token already used" }, { status: 400 });
|
||||
}
|
||||
spent.add(rt);
|
||||
issued += 1;
|
||||
const acct = accounts.shift() ?? "A";
|
||||
acctOf.set(issued, acct);
|
||||
events.push({ k: "redeem", e: engine, rt, ok: true, acct });
|
||||
if (provider === "grok") failing.set(engine, rt);
|
||||
return Response.json({ access_token: accessToken(provider, issued, acct, true), refresh_token: `rt-${issued}`, expires_in: lifetime(true) / 1000, token_type: "Bearer" });
|
||||
}
|
||||
if (url === ep.userinfo) {
|
||||
const at = tokenNumber(provider, new Headers(init.headers).get("authorization"));
|
||||
const acct = acctOf.get(at) ?? "none";
|
||||
events.push({ k: "verify", e: engine, at, acct });
|
||||
// The task fails if the account changed; otherwise it goes on to save.
|
||||
if (acct === "A") failing.delete(engine);
|
||||
return Response.json({ sub: `acct-${acct}`, email: "user@example.com" });
|
||||
}
|
||||
anomalies.push(`authorizer fetched ${url}`);
|
||||
return new Response("unexpected", { status: 599 });
|
||||
};
|
||||
const credentials = () => {
|
||||
if (!calling) anomalies.push("credentials() ran outside a request");
|
||||
events.push({ k: "ask", r: calling ?? "?" });
|
||||
return host;
|
||||
};
|
||||
const onCredentials = (next: any) => {
|
||||
const at = tokenNumber(provider, next.accessToken);
|
||||
events.push({ k: "save", e: engine, at });
|
||||
failing.delete(engine);
|
||||
host = next;
|
||||
};
|
||||
authorizers.set(engine, edge.create(provider, { credentials, onCredentials, fetch, ...(shared ? { memory: shared } : {}), ...(setup.lock ? { refreshLock } : {}) }, () => calling));
|
||||
}
|
||||
|
||||
const providerFetch = async (input: string, init: RequestInit = {}) => {
|
||||
const url = new URL(input);
|
||||
const r = url.searchParams.get("r")!;
|
||||
arrival(r).open();
|
||||
await holds.get(r)?.opened;
|
||||
const at = tokenNumber(provider, new Headers(init.headers).get("authorization"));
|
||||
const status = at === 0 && setup.rejecting ? 401 : 200;
|
||||
events.push({ k: "send", r, at, status });
|
||||
return new Response(status === 200 ? "ok" : JSON.stringify({ error: "rejected" }), { status });
|
||||
};
|
||||
|
||||
const world: World = {
|
||||
async send(r) {
|
||||
const request = setup.requests.find((q) => q.id === r)!;
|
||||
const endpoint = request.url === "provider" ? ep.provider : ep.other;
|
||||
const route = authorizers.get(request.engine)!;
|
||||
calling = r;
|
||||
const { authorized, response } = route(endpoint, providerFetch, `${endpoint}?r=${r}`, { method: "POST", headers: { "content-type": "application/json" } });
|
||||
calling = undefined;
|
||||
if (!authorized) events.push({ k: "bypass", r });
|
||||
try {
|
||||
const answered = await response;
|
||||
outcomes[r] = `status ${answered.status}`;
|
||||
} catch (error) {
|
||||
const rt = failing.get(request.engine);
|
||||
if (rt !== undefined) {
|
||||
events.push({ k: "fail", e: request.engine, rt });
|
||||
failing.delete(request.engine);
|
||||
}
|
||||
outcomes[r] = `error ${(error as Error)?.message ?? error}`;
|
||||
}
|
||||
return outcomes[r]!;
|
||||
},
|
||||
hold(r) { holds.set(r, gate()); },
|
||||
release(r) { holds.get(r)?.open(); },
|
||||
arrived(r) { return arrival(r).opened; },
|
||||
};
|
||||
await script(world);
|
||||
// `bypass` is recorded after the call returns; the model records it on entry.
|
||||
for (let n = 0; n < events.length; n++) {
|
||||
const event = events[n]!;
|
||||
if (event.k !== "bypass") continue;
|
||||
const sendAt = events.findIndex((x, m) => m < n && x.k === "send" && x.r === event.r);
|
||||
if (sendAt >= 0) { events.splice(n, 1); events.splice(sendAt, 0, event); }
|
||||
}
|
||||
return { scenario, setup, events, outcomes, anomalies };
|
||||
}
|
||||
|
||||
/** The observed run as a TLA+ module, with the model's constants fixed. */
|
||||
export function observedModule(run: Run, constants: string): string {
|
||||
const val = (v: unknown) => (typeof v === "string" ? `"${v.replace(/[^a-z0-9_-]/gi, "")}"` : typeof v === "boolean" ? (v ? "TRUE" : "FALSE") : String(v));
|
||||
const record = (event: AuthEvent) => `[${Object.entries(event).map(([key, value]) => `${key} |-> ${val(value)}`).join(", ")}]`;
|
||||
return [
|
||||
"---- MODULE AuthObserved ----",
|
||||
`(* Generated from a real run: ${run.scenario}. *)`,
|
||||
constants,
|
||||
`Observed == << ${run.events.map(record).join(",\n ")} >>`,
|
||||
"====",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Runs every Codex and Grok scenario against a worktree with the follow-up
|
||||
// fix, its engines sharing one memory as one agent's do, and checks what the
|
||||
// real code did: no refresh token redeemed twice, and every request answered.
|
||||
//
|
||||
// bun tests/tla/drivers/provider_auth/fixed-check.ts <report.json> [checkout]
|
||||
//
|
||||
// The fixed code asks the host again after a 401, which the trace model does
|
||||
// not replay, so this checks the recorded events directly instead of through
|
||||
// AuthTrace.
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { REPO_ROOT } from "../lib/tlc";
|
||||
import { runScenario } from "./driver";
|
||||
import { scenarios } from "./scenarios";
|
||||
|
||||
const [report, checkout] = process.argv.slice(2);
|
||||
if (!report) throw new Error("usage: fixed-check.ts <report.json> [checkout]");
|
||||
const worktree = resolve(checkout ?? REPO_ROOT);
|
||||
|
||||
const results = [];
|
||||
let failures = 0;
|
||||
// Two servers: separate memories, one host lock. r2's 401 returns only after r1 refreshed.
|
||||
const twoServers = (provider: "codex" | "grok") => {
|
||||
const requests = [{ id: "r1", engine: "e1", url: "provider" as const }, { id: "r2", engine: "e2", url: "provider" as const }];
|
||||
const base = { provider, requests, expiring: false, rejecting: false, foreignRefresh: false, checkAccount: true, scopeAll: false, noSingleFlight: false, lock: true };
|
||||
return [
|
||||
{ name: `${provider}-two-servers-near-expiry`, setup: { ...base, expiring: true }, script: async (world: any) => { await Promise.all([world.send("r1"), world.send("r2")]); } },
|
||||
{ name: `${provider}-two-servers-rejected-together`, setup: { ...base, rejecting: true }, script: async (world: any) => { await Promise.all([world.send("r1"), world.send("r2")]); } },
|
||||
];
|
||||
};
|
||||
|
||||
for (const provider of ["codex", "grok"] as const) {
|
||||
for (const scenario of [...scenarios(provider).map((s) => ({ ...s, setup: { ...s.setup, sharedMemory: true } })), ...twoServers(provider)]) {
|
||||
const setup = scenario.setup;
|
||||
const run = await runScenario(worktree, scenario.name, setup, scenario.script, { accounts: scenario.accounts });
|
||||
const redeems = run.events.filter((event) => event.k === "redeem");
|
||||
const reused = redeems.filter((event) => event.k === "redeem" && !event.ok).length;
|
||||
const foreign = scenario.name === "grok-refresh-for-another-account";
|
||||
const answered = Object.values(run.outcomes).every((outcome) => outcome === "status 200");
|
||||
// The recheck asks the host from an async continuation, outside the call the driver tracks.
|
||||
const anomalies = run.anomalies.filter((text) => text !== "credentials() ran outside a request");
|
||||
const ok = reused === 0 && anomalies.length === 0 && (foreign ? !answered : answered);
|
||||
if (!ok) failures += 1;
|
||||
results.push({ scenario: scenario.name, ok, redeems: redeems.length, reused, outcomes: run.outcomes, anomalies });
|
||||
console.log(`${ok ? "ok " : "FAIL"} ${scenario.name}: redeems ${redeems.length}, reused ${reused}, ${JSON.stringify(run.outcomes)}`);
|
||||
}
|
||||
}
|
||||
writeFileSync(report, JSON.stringify({ worktree, results }, null, 2));
|
||||
console.log(failures ? `${failures} scenario(s) failed` : "every scenario passed");
|
||||
process.exit(failures ? 1 : 0);
|
||||
@@ -0,0 +1,67 @@
|
||||
// Runs the target worktree's real libfx credential edge (the authorizer that
|
||||
// transportFetch hands Codex, Grok and custom-provider requests to) through
|
||||
// the scenarios in scenarios.ts, then has TLC replay each run
|
||||
// through the ProviderAuth model. No network: every endpoint is a local fake.
|
||||
//
|
||||
// 1. Every run must be a behavior of the model (TLC ACCEPTED), and each
|
||||
// property must reach the verdict its scenario declares.
|
||||
// 2. Controls: a tampered, a truncated and an empty run must not be accepted.
|
||||
import { expect, test } from "bun:test";
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { artifacts, worktree } from "../lib/context";
|
||||
import { checkTrace, type TraceResult } from "./bridge";
|
||||
import { loadEdge, runScenario, type Run } from "./driver";
|
||||
import { scenarios } from "./scenarios";
|
||||
import { PROPERTIES } from "./tlc";
|
||||
|
||||
const TIMEOUT = 900_000;
|
||||
const reports = join(artifacts, "tlc");
|
||||
const edge = await loadEdge(worktree);
|
||||
writeFileSync(join(artifacts, "edge.json"), JSON.stringify({ worktree, shape: edge.shape, providers: edge.providers }, null, 2));
|
||||
|
||||
const runs = new Map<string, Run>();
|
||||
|
||||
// Code with shared credential memory asks the host again after a 401, which
|
||||
// AuthTrace does not replay. There, fixed-check.ts checks the rejection
|
||||
// scenarios from their recorded events instead.
|
||||
const asksAgain = edge.memory !== undefined;
|
||||
const replayable = (name: string) => !(asksAgain && /-(rejected-refreshes-once|late-rejection-after-refresh)$/.test(name));
|
||||
|
||||
for (const provider of edge.providers) {
|
||||
for (const scenario of scenarios(provider)) {
|
||||
test.skipIf(!replayable(scenario.name))(`${scenario.name} is a run of the model`, async () => {
|
||||
const run = await runScenario(worktree, scenario.name, scenario.setup, scenario.script, { accounts: scenario.accounts });
|
||||
runs.set(scenario.name, run);
|
||||
const result: TraceResult = await checkTrace(run, reports);
|
||||
writeFileSync(join(artifacts, `${scenario.name}.json`), JSON.stringify({ run, result }, null, 2));
|
||||
expect(result.verdict, `${result.detail}\nevents: ${JSON.stringify(run.events)}`).toBe("ACCEPTED");
|
||||
for (const property of PROPERTIES) {
|
||||
expect(result.properties[property], `${property} on ${scenario.name}`).toBe(scenario.expect?.[property] ?? "HOLDS");
|
||||
}
|
||||
}, TIMEOUT);
|
||||
}
|
||||
}
|
||||
|
||||
test("controls: tampered, truncated and empty runs are not accepted", async () => {
|
||||
const preferred = asksAgain ? "codex-expiring-refreshes-first" : "codex-rejected-refreshes-once";
|
||||
const name = edge.providers.includes("codex") ? preferred : [...runs.keys()][0]!;
|
||||
const run = runs.get(name) ?? await (async () => {
|
||||
const scenario = scenarios("codex").find((s) => s.name === name)!;
|
||||
return runScenario(worktree, scenario.name, scenario.setup, scenario.script);
|
||||
})();
|
||||
// The resend after the 401 carries the rejected token again.
|
||||
const tampered = structuredClone(run);
|
||||
const lastSend = tampered.events.findLastIndex((event) => event.k === "send");
|
||||
(tampered.events[lastSend] as { at: number }).at = 0;
|
||||
tampered.scenario = `${name}-tampered`;
|
||||
const truncated = structuredClone(run);
|
||||
truncated.events = truncated.events.filter((event) => event.k !== "save");
|
||||
truncated.scenario = `${name}-without-save`;
|
||||
const empty = { ...structuredClone(run), scenario: `${name}-empty`, events: [] };
|
||||
const verdicts = await Promise.all([tampered, truncated, empty].map((r) => checkTrace(r, reports)));
|
||||
writeFileSync(join(artifacts, "controls.json"), JSON.stringify(verdicts, null, 2));
|
||||
expect(verdicts[0]!.verdict).toBe("REJECTED");
|
||||
expect(verdicts[1]!.verdict).toBe("REJECTED");
|
||||
expect(verdicts[2]!.verdict).toBe("INCONCLUSIVE");
|
||||
}, TIMEOUT);
|
||||
@@ -0,0 +1,176 @@
|
||||
// Exhaustive TLC checks of ProviderAuth, grouped by the pull request that
|
||||
// added each provider's credential edge.
|
||||
//
|
||||
// bun tests/tla/drivers/provider_auth/run-all.ts <report-dir> [1210|1211|1212]...
|
||||
//
|
||||
// Each check fixes the model's constants and names the verdict every property
|
||||
// must reach. Designs the code implements must HOLD; planted bugs must be
|
||||
// VIOLATED, which shows each property can fail. A check whose verdicts differ
|
||||
// from the expectation fails the run.
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { checkModel, hashes, PROPERTIES, type Property, type Setup, type Verdict } from "./tlc";
|
||||
|
||||
const base: Omit<Setup, "provider" | "requests"> = {
|
||||
expiring: false, rejecting: false, foreignRefresh: false, checkAccount: true, scopeAll: false, noSingleFlight: false,
|
||||
};
|
||||
const oneEngine = [{ id: "r1", engine: "e1", url: "provider" as const }, { id: "r2", engine: "e1", url: "provider" as const }];
|
||||
const twoEngines = [{ id: "r1", engine: "e1", url: "provider" as const }, { id: "r2", engine: "e2", url: "provider" as const }];
|
||||
const withOther = [{ id: "r1", engine: "e1", url: "provider" as const }, { id: "r2", engine: "e1", url: "other" as const }];
|
||||
const corners = [[false, false], [false, true], [true, false], [true, true]] as const;
|
||||
|
||||
interface Check { pr: string; name: string; setup: Setup; expect: Partial<Record<Property, Verdict>>; why: string }
|
||||
const holdsAll = Object.fromEntries(PROPERTIES.map((p) => [p, "HOLDS"])) as Record<Property, Verdict>;
|
||||
|
||||
function refreshableChecks(pr: string, provider: "codex" | "grok"): Check[] {
|
||||
const checks: Check[] = corners.map(([expiring, rejecting]) => {
|
||||
// Both requests carry the rejected token; the later 401 refreshes the
|
||||
// credential it used, whose refresh token the first refresh already spent.
|
||||
const lateRetry = rejecting && !expiring;
|
||||
return {
|
||||
pr, name: `${provider}-one-engine-${expiring ? "expiring" : "fresh"}-${rejecting ? "rejecting" : "accepting"}`,
|
||||
setup: { ...base, provider, requests: oneEngine, expiring, rejecting },
|
||||
expect: lateRetry ? { ...holdsAll, NoRefreshReuse: "VIOLATED" } : holdsAll,
|
||||
why: lateRetry
|
||||
? "finding: a 401 that arrives after another request's refresh redeems the spent refresh token again"
|
||||
: "the design: two requests on one engine share one refresh",
|
||||
};
|
||||
});
|
||||
checks.push({
|
||||
pr, name: `${provider}-two-engines-expiring`,
|
||||
setup: { ...base, provider, requests: twoEngines, expiring: true },
|
||||
expect: { ...holdsAll, NoRefreshReuse: "VIOLATED" },
|
||||
why: "libfx opens one engine per session, each with its own authorizer: two sessions near expiry each redeem the same refresh token",
|
||||
});
|
||||
checks.push({
|
||||
pr, name: `${provider}-planted-no-single-flight`,
|
||||
setup: { ...base, provider, requests: oneEngine, expiring: true, noSingleFlight: true },
|
||||
expect: { ...holdsAll, NoRefreshReuse: "VIOLATED" },
|
||||
why: "planted bug: without the shared task, NoRefreshReuse must fail",
|
||||
});
|
||||
if (provider === "grok") {
|
||||
checks.push({
|
||||
pr, name: "grok-refresh-for-another-account",
|
||||
setup: { ...base, provider, requests: oneEngine, expiring: true, foreignRefresh: true },
|
||||
expect: { ...holdsAll, NoRefreshReuse: "VIOLATED" },
|
||||
why: "another account's token is refused and never used or saved; the host's refresh token is spent by then, so the next request redeems it again and the user must sign in again",
|
||||
});
|
||||
checks.push({
|
||||
pr, name: "grok-planted-no-account-check",
|
||||
setup: { ...base, provider, requests: oneEngine, expiring: true, foreignRefresh: true, checkAccount: false },
|
||||
expect: { ...holdsAll, NoForeignToken: "VIOLATED" },
|
||||
why: "planted bug: without the account check, another account's token is used and saved",
|
||||
});
|
||||
}
|
||||
return checks;
|
||||
}
|
||||
|
||||
// The follow-up fix: one agent's engines share a memory, and a 401 retries on
|
||||
// newer credentials before it refreshes.
|
||||
function fixedChecks(provider: "codex" | "grok"): Check[] {
|
||||
const fixed = { ...base, provider, sharedMemory: true, recheck: true };
|
||||
const checks: Check[] = [];
|
||||
for (const [topology, requests] of [["one-engine", oneEngine], ["two-engines", twoEngines]] as const) {
|
||||
for (const [expiring, rejecting] of corners) {
|
||||
checks.push({
|
||||
pr: "fixed", name: `${provider}-fixed-${topology}-${expiring ? "expiring" : "fresh"}-${rejecting ? "rejecting" : "accepting"}`,
|
||||
setup: { ...fixed, requests, expiring, rejecting },
|
||||
expect: holdsAll,
|
||||
why: "the fix: no refresh token is redeemed twice by one agent, in any order",
|
||||
});
|
||||
}
|
||||
}
|
||||
checks.push({
|
||||
pr: "fixed", name: `${provider}-recheck-only-late-401`,
|
||||
setup: { ...base, provider, requests: oneEngine, rejecting: true, recheck: true },
|
||||
expect: holdsAll,
|
||||
why: "the 401 fix alone closes the late rejection on one engine",
|
||||
});
|
||||
checks.push({
|
||||
pr: "fixed", name: `${provider}-recheck-only-two-engines-rejecting`,
|
||||
setup: { ...base, provider, requests: twoEngines, rejecting: true, recheck: true },
|
||||
expect: { ...holdsAll, NoRefreshReuse: "VIOLATED" },
|
||||
why: "without a shared memory, two sessions rejected before either saves both refresh; the shared memory closes this",
|
||||
});
|
||||
checks.push({
|
||||
pr: "fixed", name: `${provider}-separate-agents-expiring`,
|
||||
setup: { ...base, provider, requests: twoEngines, expiring: true, recheck: true },
|
||||
expect: { ...holdsAll, NoRefreshReuse: "VIOLATED" },
|
||||
why: "remaining limit: two processes that both start before either saves still redeem the same refresh token",
|
||||
});
|
||||
for (const [expiring, rejecting] of corners) {
|
||||
checks.push({
|
||||
pr: "fixed", name: `${provider}-lock-separate-agents-${expiring ? "expiring" : "fresh"}-${rejecting ? "rejecting" : "accepting"}`,
|
||||
setup: { ...base, provider, requests: twoEngines, expiring, rejecting, recheck: true, lock: true },
|
||||
expect: holdsAll,
|
||||
why: "refreshLock: separate servers that share a lock never redeem one refresh token twice",
|
||||
});
|
||||
}
|
||||
checks.push({
|
||||
pr: "fixed", name: `${provider}-fixed-planted-no-single-flight`,
|
||||
setup: { ...fixed, requests: oneEngine, expiring: true, noSingleFlight: true },
|
||||
expect: { ...holdsAll, NoRefreshReuse: "VIOLATED" },
|
||||
why: "planted bug: the checks still catch a missing shared task",
|
||||
});
|
||||
if (provider === "grok") {
|
||||
checks.push({
|
||||
pr: "fixed", name: "grok-fixed-refresh-for-another-account",
|
||||
setup: { ...fixed, requests: oneEngine, expiring: true, foreignRefresh: true },
|
||||
expect: { ...holdsAll, NoRefreshReuse: "VIOLATED" },
|
||||
why: "unchanged by the fix: after a refused cross-account refresh the user must sign in again",
|
||||
});
|
||||
}
|
||||
return checks;
|
||||
}
|
||||
|
||||
const byPr: Record<string, () => Check[]> = {
|
||||
"fixed": () => [...fixedChecks("codex"), ...fixedChecks("grok")],
|
||||
"1210": () => refreshableChecks("1210", "codex"),
|
||||
"1211": () => [...refreshableChecks("1211", "codex"), ...refreshableChecks("1211", "grok")],
|
||||
"1212": () => [
|
||||
...[false, true].map((rejecting): Check => ({
|
||||
pr: "1212", name: `custom-key-and-other-url-${rejecting ? "rejecting" : "accepting"}`,
|
||||
setup: { ...base, provider: "custom", requests: withOther, rejecting },
|
||||
expect: holdsAll,
|
||||
why: "the design: the key goes to the chat URL only and is never refreshed",
|
||||
})),
|
||||
{
|
||||
pr: "1212", name: "custom-two-engines",
|
||||
setup: { ...base, provider: "custom", requests: twoEngines, rejecting: true },
|
||||
expect: holdsAll,
|
||||
why: "a key has no refresh token, so sessions cannot collide over one",
|
||||
},
|
||||
{
|
||||
pr: "1212", name: "custom-planted-scope-all",
|
||||
setup: { ...base, provider: "custom", requests: withOther, scopeAll: true },
|
||||
expect: { ...holdsAll, ScopedAuth: "VIOLATED" },
|
||||
why: "planted bug: a spec that authorizes every URL must break ScopedAuth",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const [reportArg, ...prArgs] = process.argv.slice(2);
|
||||
if (!reportArg) throw new Error("usage: run-all.ts <report-dir> [1210|1211|1212]...");
|
||||
const report = resolve(reportArg);
|
||||
mkdirSync(report, { recursive: true, mode: 0o700 });
|
||||
const prs = prArgs.length ? prArgs : Object.keys(byPr);
|
||||
|
||||
const results = [];
|
||||
let failures = 0;
|
||||
for (const pr of prs) {
|
||||
for (const check of byPr[pr]!()) {
|
||||
const verdicts: Record<string, { verdict: Verdict; states?: string; log: string }> = {};
|
||||
for (const property of PROPERTIES) {
|
||||
const { verdict, run, states } = await checkModel(report, `${pr}-${check.name}`, check.setup, property);
|
||||
verdicts[property] = { verdict, states, log: join(run.dir, "tlc.log") };
|
||||
}
|
||||
const wrong = PROPERTIES.filter((p) => verdicts[p]!.verdict !== (check.expect[p] ?? "HOLDS"));
|
||||
if (wrong.length) failures += 1;
|
||||
results.push({ ...check, verdicts, ok: wrong.length === 0, wrong });
|
||||
const summary = PROPERTIES.map((p) => `${p}=${verdicts[p]!.verdict}`).join(" ");
|
||||
console.log(`${wrong.length ? "FAIL" : "ok "} #${pr} ${check.name} (${verdicts.TypeOK!.states ?? "?"} states): ${summary}`);
|
||||
}
|
||||
}
|
||||
writeFileSync(join(report, "model-results.json"), JSON.stringify({ hashes: hashes(), results }, null, 2), { mode: 0o600 });
|
||||
console.log(failures ? `${failures} check(s) did not reach their expected verdicts` : "every check reached its expected verdicts");
|
||||
process.exit(failures ? 1 : 0);
|
||||
@@ -0,0 +1,70 @@
|
||||
// Scenarios that drive the real credential edge through the interleavings
|
||||
// ProviderAuth models. Each names the model setup it runs under, so its trace
|
||||
// replays against exactly those constants.
|
||||
import type { Setup } from "./tlc";
|
||||
import type { World } from "./driver";
|
||||
|
||||
type Provider = Setup["provider"];
|
||||
|
||||
export interface Scenario {
|
||||
name: string;
|
||||
setup: Setup;
|
||||
script: (world: World) => Promise<void>;
|
||||
accounts?: string[];
|
||||
/** The property verdicts this trace must reach; omitted ones must hold. */
|
||||
expect?: Record<string, "VIOLATED">;
|
||||
}
|
||||
|
||||
const base = { expiring: false, rejecting: false, foreignRefresh: false, checkAccount: true, scopeAll: false, noSingleFlight: false };
|
||||
const one = (url: "provider" | "other" = "provider") => [{ id: "r1", engine: "e1", url }];
|
||||
const sameEngine = [{ id: "r1", engine: "e1", url: "provider" as const }, { id: "r2", engine: "e1", url: "provider" as const }];
|
||||
const twoEngines = [{ id: "r1", engine: "e1", url: "provider" as const }, { id: "r2", engine: "e2", url: "provider" as const }];
|
||||
const withOther = [{ id: "r1", engine: "e1", url: "provider" as const }, { id: "r2", engine: "e1", url: "other" as const }];
|
||||
|
||||
const alone = async (world: World) => { await world.send("r1"); };
|
||||
const together = async (world: World) => { await Promise.all([world.send("r1"), world.send("r2")]); };
|
||||
const inTurn = async (world: World) => { await world.send("r1"); await world.send("r2"); };
|
||||
|
||||
/** r2 leaves with the rejected token, and its 401 comes back only after r1's refresh saved. */
|
||||
const lateRejection = async (world: World) => {
|
||||
world.hold("r2");
|
||||
const second = world.send("r2");
|
||||
await world.arrived("r2");
|
||||
await world.send("r1");
|
||||
world.release("r2");
|
||||
await second;
|
||||
};
|
||||
|
||||
export function scenarios(provider: Provider): Scenario[] {
|
||||
if (provider === "custom") {
|
||||
return [
|
||||
{ name: "custom-key-on-chat-url", setup: { ...base, provider, requests: one() }, script: alone },
|
||||
{ name: "custom-other-url-bypasses", setup: { ...base, provider, requests: withOther }, script: inTurn },
|
||||
{ name: "custom-rejected-key-not-refreshed", setup: { ...base, provider, requests: one(), rejecting: true }, script: alone },
|
||||
{ name: "custom-two-sessions", setup: { ...base, provider, requests: twoEngines, rejecting: true }, script: together },
|
||||
];
|
||||
}
|
||||
const list: Scenario[] = [
|
||||
{ name: `${provider}-fresh`, setup: { ...base, provider, requests: one() }, script: alone },
|
||||
{ name: `${provider}-expiring-refreshes-first`, setup: { ...base, provider, requests: one(), expiring: true }, script: alone },
|
||||
{ name: `${provider}-rejected-refreshes-once`, setup: { ...base, provider, requests: one(), rejecting: true }, script: alone },
|
||||
{ name: `${provider}-other-url-bypasses`, setup: { ...base, provider, requests: withOther, expiring: true }, script: inTurn },
|
||||
{ name: `${provider}-one-session-shares-refresh`, setup: { ...base, provider, requests: sameEngine, expiring: true }, script: together },
|
||||
{ name: `${provider}-one-session-after-save`, setup: { ...base, provider, requests: sameEngine, expiring: true }, script: inTurn },
|
||||
{
|
||||
name: `${provider}-two-sessions-near-expiry`, setup: { ...base, provider, requests: twoEngines, expiring: true }, script: together,
|
||||
expect: { NoRefreshReuse: "VIOLATED" },
|
||||
},
|
||||
{
|
||||
name: `${provider}-late-rejection-after-refresh`, setup: { ...base, provider, requests: sameEngine, rejecting: true }, script: lateRejection,
|
||||
expect: { NoRefreshReuse: "VIOLATED" },
|
||||
},
|
||||
];
|
||||
if (provider === "grok") {
|
||||
list.push({
|
||||
name: "grok-refresh-for-another-account", setup: { ...base, provider, requests: one(), expiring: true, foreignRefresh: true },
|
||||
script: alone, accounts: ["B"],
|
||||
});
|
||||
}
|
||||
return list;
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// Runs checks of tests/tla/sdk/ProviderAuth.tla with the TLC that
|
||||
// ../lib/tlc.ts finds.
|
||||
// A check is a generated module that fixes the model's constants, a config
|
||||
// naming the properties, and the verdict each property must have.
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { TLA_ROOT, tlcToolchain } from "../lib/tlc";
|
||||
|
||||
const tlaDir = join(TLA_ROOT, "sdk");
|
||||
|
||||
export const PROPERTIES = ["TypeOK", "NoRefreshReuse", "AtMostOneResend", "NoForeignToken", "ScopedAuth", "KeyNeverRefreshed"] as const;
|
||||
export type Property = (typeof PROPERTIES)[number];
|
||||
export type Verdict = "HOLDS" | "VIOLATED" | "TOOL_ERROR";
|
||||
|
||||
/** The model's constants for one check. */
|
||||
export interface Setup {
|
||||
provider: "codex" | "grok" | "custom";
|
||||
/** Request id to engine id and URL kind, in order. */
|
||||
requests: Array<{ id: string; engine: string; url: "provider" | "other" }>;
|
||||
expiring: boolean;
|
||||
rejecting: boolean;
|
||||
foreignRefresh: boolean;
|
||||
checkAccount: boolean;
|
||||
scopeAll: boolean;
|
||||
noSingleFlight: boolean;
|
||||
/** The fixed design: one agent's engines share a memory. */
|
||||
sharedMemory?: boolean;
|
||||
/** The fixed design: a 401 retries on newer credentials before refreshing. */
|
||||
recheck?: boolean;
|
||||
/** The host's refreshLock: refreshes take turns across memories. */
|
||||
lock?: boolean;
|
||||
}
|
||||
|
||||
const sha256 = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
|
||||
const str = (value: string) => {
|
||||
if (!/^[a-z0-9]+$/.test(value)) throw new Error(`refusing unsafe TLA+ string ${JSON.stringify(value)}`);
|
||||
return `"${value}"`;
|
||||
};
|
||||
const bool = (value: boolean) => (value ? "TRUE" : "FALSE");
|
||||
|
||||
/** TLA+ definitions of the constants, named MC*. */
|
||||
export function constantDefinitions(setup: Setup): string {
|
||||
const ids = setup.requests.map((r) => str(r.id)).join(", ");
|
||||
const engines = [...new Set(setup.requests.map((r) => str(r.engine)))].join(", ");
|
||||
const pick = (field: "engine" | "url") =>
|
||||
setup.requests.map((r, n) => `${n === 0 ? "IF" : "ELSE IF"} r = ${str(r.id)} THEN ${str(r[field])}`).join(" ") + ` ELSE "none"`;
|
||||
return [
|
||||
`MCProvider == ${str(setup.provider)}`,
|
||||
`MCEngines == {${engines}}`,
|
||||
`MCRequests == {${ids}}`,
|
||||
`MCEngineOf == [r \\in MCRequests |-> ${pick("engine")}]`,
|
||||
`MCUrlOf == [r \\in MCRequests |-> ${pick("url")}]`,
|
||||
`MCExpiring == ${bool(setup.expiring)}`,
|
||||
`MCRejecting == ${bool(setup.rejecting)}`,
|
||||
`MCForeignRefresh == ${bool(setup.foreignRefresh)}`,
|
||||
`MCCheckAccount == ${bool(setup.checkAccount)}`,
|
||||
`MCScopeAll == ${bool(setup.scopeAll)}`,
|
||||
`MCNoSingleFlight == ${bool(setup.noSingleFlight)}`,
|
||||
`MCSharedMemory == ${bool(setup.sharedMemory ?? false)}`,
|
||||
`MCRecheck == ${bool(setup.recheck ?? false)}`,
|
||||
`MCLock == ${bool(setup.lock ?? false)}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export const constantOverrides = [
|
||||
"Provider <- MCProvider", "Engines <- MCEngines", "Requests <- MCRequests", "EngineOf <- MCEngineOf",
|
||||
"UrlOf <- MCUrlOf", "Expiring <- MCExpiring", "Rejecting <- MCRejecting", "ForeignRefresh <- MCForeignRefresh",
|
||||
"CheckAccount <- MCCheckAccount", "ScopeAll <- MCScopeAll", "NoSingleFlight <- MCNoSingleFlight",
|
||||
"SharedMemory <- MCSharedMemory", "Recheck <- MCRecheck", "Lock <- MCLock",
|
||||
].map((line) => ` ${line}`).join("\n");
|
||||
|
||||
export interface TlcRun { exitCode: number; log: string; dir: string; command: string[] }
|
||||
|
||||
export async function runTlc(dir: string, spec: string, cfg: string, extra: string[] = []): Promise<TlcRun> {
|
||||
writeFileSync(join(dir, "check.cfg"), cfg, { mode: 0o600 });
|
||||
const { java, jar } = tlcToolchain();
|
||||
mkdirSync(join(dir, ".tlc-tmp"), { recursive: true });
|
||||
const command = [java, "-XX:+UseParallelGC", `-Djava.io.tmpdir=${join(dir, ".tlc-tmp")}`, "-cp", jar, "tlc2.TLC", "-workers", "1",
|
||||
"-metadir", join(dir, ".tlc-states"), "-config", "check.cfg", "-deadlock", ...extra, spec];
|
||||
const child = Bun.spawn(command, { cwd: dir, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
||||
const [exitCode, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
||||
const log = `${stderr}\n${stdout}`;
|
||||
writeFileSync(join(dir, "tlc.log"), log, { mode: 0o600 });
|
||||
writeFileSync(join(dir, "result.json"), JSON.stringify({ command, exitCode }, null, 2), { mode: 0o600 });
|
||||
return { exitCode, log, dir, command };
|
||||
}
|
||||
|
||||
/** Copies the model modules into a fresh directory, with any generated modules. */
|
||||
export function stage(root: string, name: string, generated: Record<string, string>): string {
|
||||
const dir = join(root, name);
|
||||
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
for (const module of ["ProviderAuth.tla", "AuthTrace.tla"]) copyFileSync(join(tlaDir, module), join(dir, module));
|
||||
for (const [file, text] of Object.entries(generated)) writeFileSync(join(dir, file), text, { mode: 0o600 });
|
||||
return dir;
|
||||
}
|
||||
|
||||
export const completed = (log: string) => log.includes("Model checking completed. No error has been found.");
|
||||
export const violated = (log: string, name: string) => log.includes(`Invariant ${name} is violated`);
|
||||
|
||||
export function hashes(): Record<string, string> {
|
||||
return {
|
||||
"ProviderAuth.tla": sha256(readFileSync(join(tlaDir, "ProviderAuth.tla"))),
|
||||
"AuthTrace.tla": sha256(readFileSync(join(tlaDir, "AuthTrace.tla"))),
|
||||
"tla2tools.jar": tlcToolchain().jarSha256,
|
||||
};
|
||||
}
|
||||
|
||||
/** Checks one property exhaustively over the model fixed by `setup`. */
|
||||
export async function checkModel(root: string, label: string, setup: Setup, property: Property): Promise<{ verdict: Verdict; run: TlcRun; states?: string }> {
|
||||
const module = ["---- MODULE MC ----", "EXTENDS ProviderAuth", constantDefinitions(setup), "===="].join("\n");
|
||||
const dir = stage(mkdtempSync(join(root, `${label}-${property}-`)), "model", { "MC.tla": module });
|
||||
const run = await runTlc(dir, "MC.tla", `SPECIFICATION Spec\nINVARIANT ${property}\nCONSTANTS\n${constantOverrides}\n`);
|
||||
const states = /(\d+) distinct states found/.exec(run.log)?.[1];
|
||||
if (run.exitCode === 0 && completed(run.log)) return { verdict: "HOLDS", run, states };
|
||||
if (run.exitCode === 12 && violated(run.log, property)) return { verdict: "VIOLATED", run, states };
|
||||
return { verdict: "TOOL_ERROR", run, states };
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
export type Verdict = "ACCEPTED" | "REJECTED" | "INCONCLUSIVE" | "TOOL_ERROR";
|
||||
export type Model = "turns" | "children" | "wiring";
|
||||
export interface RuntimeEvidence {
|
||||
version: 1;
|
||||
sessionsRoot: string;
|
||||
sessionIds: string[];
|
||||
wiringTrace?: string;
|
||||
processes?: Array<{ pid: number; exitCode: number | null; signal: string | null }>;
|
||||
requiredModels: Model[];
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
export interface RuntimeCheck {
|
||||
kind: string;
|
||||
sessionId?: string;
|
||||
verdict: Verdict;
|
||||
expected?: number;
|
||||
matched?: number;
|
||||
exitCode?: number;
|
||||
log?: string;
|
||||
detail?: string;
|
||||
}
|
||||
export interface RuntimeResult {
|
||||
version: 1;
|
||||
verdict: Verdict;
|
||||
checks: RuntimeCheck[];
|
||||
report: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
const exitCodes: Record<Verdict, number> = { ACCEPTED: 0, REJECTED: 1, INCONCLUSIVE: 2, TOOL_ERROR: 3 };
|
||||
const checker = resolve(import.meta.dirname, "check_runtime.py");
|
||||
|
||||
export function decodeResult(stdout: string, exitCode: number, evidence: RuntimeEvidence): RuntimeResult {
|
||||
const result = JSON.parse(stdout) as RuntimeResult;
|
||||
if (result.version !== 1 || !(result.verdict in exitCodes) || !Array.isArray(result.checks) ||
|
||||
typeof result.report !== "string" || exitCodes[result.verdict] !== exitCode) {
|
||||
throw new Error("Invalid TLC bridge result or mismatched process exit");
|
||||
}
|
||||
if (result.verdict === "ACCEPTED") {
|
||||
if (!result.checks.length || result.checks.some(check => check.verdict !== "ACCEPTED" ||
|
||||
!Number.isInteger(check.expected) || check.expected! <= 0 || check.matched !== check.expected || check.exitCode !== 12)) {
|
||||
throw new Error("TLC acceptance lacks complete, nonempty sentinel proof");
|
||||
}
|
||||
for (const model of evidence.requiredModels) {
|
||||
const checks = result.checks.filter(check => check.kind === model);
|
||||
if (!checks.length) throw new Error(`Required ${model} checks absent`);
|
||||
if (model === "turns" && evidence.sessionIds.some(id => !checks.some(check => check.sessionId === id))) {
|
||||
throw new Error("An expected session lacks its turn proof");
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function inspectRuntime(evidence: RuntimeEvidence, reportRoot: string): Promise<RuntimeResult> {
|
||||
mkdirSync(reportRoot, { recursive: true, mode: 0o700 });
|
||||
const invocation = mkdtempSync(join(reportRoot, "tlc-"));
|
||||
const request = join(invocation, "request.json");
|
||||
writeFileSync(request, JSON.stringify(evidence, null, 2), { mode: 0o600 });
|
||||
const command = ["python3", checker, "--request", request, "--report", join(invocation, "report")];
|
||||
const child = Bun.spawn(command, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
||||
const [exitCode, stdout, stderr] = await Promise.all([
|
||||
child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
|
||||
]);
|
||||
writeFileSync(join(invocation, "process.json"), JSON.stringify({ command, exitCode, stdout, stderr }, null, 2), { mode: 0o600 });
|
||||
try {
|
||||
return decodeResult(stdout, exitCode, evidence);
|
||||
} catch (error) {
|
||||
throw new Error(`TLC bridge protocol/tool failure; evidence retained at ${invocation}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function requireAccepted(result: RuntimeResult): void {
|
||||
if (result.verdict !== "ACCEPTED") {
|
||||
throw new Error(`TLC ${result.verdict}; evidence retained at ${result.report}: ${JSON.stringify(result.checks)}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strict runtime-trace checking of real fx sessions against the session manager
|
||||
trace wrappers in tests/tla/session_manager/trace/."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
ARCHIVE = Path(__file__).resolve().parents[2] / 'session_manager/trace'
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'session_manager/rollout'))
|
||||
from run import find_java, sha256 # noqa: E402
|
||||
sys.path.insert(0, str(ARCHIVE))
|
||||
saved = importlib.import_module('check_saved_logs')
|
||||
wiring = importlib.import_module('check_wiring_traces')
|
||||
MAX_EVIDENCE_BYTES = 64 * 1024 * 1024
|
||||
VALID_ID = re.compile(r'(?!\.{1,2}\Z)[A-Za-z0-9._-]{1,160}\Z')
|
||||
|
||||
|
||||
class Inconclusive(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def crc32c(data):
|
||||
crc = 0xffffffff
|
||||
for byte in data:
|
||||
crc ^= byte
|
||||
for _ in range(8):
|
||||
crc = (crc >> 1) ^ (0x82f63b78 if crc & 1 else 0)
|
||||
return crc ^ 0xffffffff
|
||||
|
||||
|
||||
def checked_lines(data):
|
||||
if not data or not data.endswith(b'\n'):
|
||||
raise Inconclusive('empty log or incomplete final JSONL line')
|
||||
lines = []
|
||||
for position, raw in enumerate(data.splitlines(), 1):
|
||||
try:
|
||||
entry = json.loads(raw)
|
||||
except (ValueError, UnicodeDecodeError) as error:
|
||||
raise Inconclusive(f'unreadable JSON at line {position}: {error}') from error
|
||||
if not isinstance(entry, dict) or entry.get('seq') != position:
|
||||
raise Inconclusive(f'non-contiguous seq at line {position}')
|
||||
marker = raw.rfind(b',"crc":"')
|
||||
suffix = raw[marker:] if marker >= 0 else b''
|
||||
if not re.fullmatch(rb',"crc":"[0-9a-f]{8}"}', suffix):
|
||||
raise Inconclusive(f'noncanonical or missing CRC at line {position}')
|
||||
if crc32c(raw[:marker]) != int(entry['crc'], 16):
|
||||
raise Inconclusive(f'CRC32C mismatch at line {position}')
|
||||
lines.append(entry)
|
||||
if lines[0].get('kind') != 'session_created':
|
||||
raise Inconclusive('session_created header missing')
|
||||
return lines
|
||||
|
||||
|
||||
def checked_wiring(text, processes):
|
||||
events = []
|
||||
for line in text.splitlines():
|
||||
if '[wiring]' not in line:
|
||||
continue
|
||||
match = wiring.LINE.fullmatch(line)
|
||||
if not match:
|
||||
raise Inconclusive('malformed wiring line')
|
||||
events.append((int(match.group(4)), match.group(1), match.group(2), match.group(3)))
|
||||
if not events:
|
||||
raise Inconclusive('no wiring events observed')
|
||||
outcomes = {}
|
||||
for process in processes:
|
||||
pid = process.get('pid')
|
||||
if not isinstance(pid, int) or pid <= 0 or pid in outcomes:
|
||||
raise Inconclusive('process outcomes require unique positive PIDs')
|
||||
code, signal = process.get('exitCode'), process.get('signal')
|
||||
if not ((isinstance(code, int) and signal is None) or (code is None and signal in ('SIGKILL', 'SIGTERM'))):
|
||||
raise Inconclusive('missing or unsupported observed process termination')
|
||||
outcomes[pid] = process
|
||||
if set(outcomes) != {event[0] for event in events}:
|
||||
raise Inconclusive('wiring process set does not equal observed process outcomes')
|
||||
for pid, outcome in outcomes.items():
|
||||
mine = [event for event in events if event[0] == pid]
|
||||
if mine[0][1] != 'StoreOpen':
|
||||
raise Inconclusive('startup StoreOpen missing; cannot infer startup')
|
||||
depth = 0
|
||||
for _, action, _, _ in mine:
|
||||
if action == 'StoreOpen':
|
||||
depth += 1
|
||||
elif action == 'StoreClose':
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
raise Inconclusive('store lifecycle nesting is malformed')
|
||||
if outcome['signal'] is None and (depth != 0 or mine[-1][1] != 'StoreClose'):
|
||||
raise Inconclusive('normal process exit lacks complete StoreClose evidence')
|
||||
if outcome['signal'] is not None and depth == 0:
|
||||
raise Inconclusive('crash case did not reach an open-store fault boundary')
|
||||
try:
|
||||
return wiring.steps_for(events)
|
||||
except wiring.Skip as error:
|
||||
raise Inconclusive(str(error)) from error
|
||||
|
||||
|
||||
def trace_verdict(code, output, sentinel, expected):
|
||||
matches = [int(value) for value in re.findall(r'"matched_lines", (\d+)', output)]
|
||||
matched = max(matches, default=0)
|
||||
if code == 12 and f'Invariant {sentinel} is violated' in output and matched == expected and expected > 0:
|
||||
return 'ACCEPTED', matched
|
||||
if code == 0 and 'Model checking completed. No error has been found.' in output and matched < expected:
|
||||
return 'REJECTED', matched
|
||||
return 'TOOL_ERROR', matched
|
||||
|
||||
|
||||
def aggregate(checks):
|
||||
verdicts = {check['verdict'] for check in checks}
|
||||
for verdict in ('TOOL_ERROR', 'INCONCLUSIVE', 'REJECTED'):
|
||||
if verdict in verdicts:
|
||||
return verdict
|
||||
return 'ACCEPTED' if checks and verdicts == {'ACCEPTED'} else 'INCONCLUSIVE'
|
||||
|
||||
|
||||
class RetainedTLC:
|
||||
def __init__(self, report, java, jar):
|
||||
self.report, self.java, self.jar = report, java, jar
|
||||
self.expected = 0
|
||||
self.last = None
|
||||
self.sequence = 0
|
||||
|
||||
def run(self, module, spec_files, generated, cfg, invariant='NotWritten'):
|
||||
self.sequence += 1
|
||||
work = self.report / 'tlc' / f'{self.sequence:03d}-{module}'
|
||||
(work / 'tmp').mkdir(parents=True)
|
||||
models = {}
|
||||
for source in map(Path, spec_files):
|
||||
shutil.copyfile(source, work / source.name)
|
||||
models[source.name] = sha256(source)
|
||||
for name, text in generated.items():
|
||||
(work / name).write_text(text)
|
||||
config = work / (module + '.cfg')
|
||||
config.write_text(cfg)
|
||||
command = [self.java, '-Xmx512m', '-XX:+UseParallelGC', f'-Djava.io.tmpdir={work / "tmp"}',
|
||||
'-cp', str(self.jar), 'tlc2.TLC', '-workers', '1', '-deadlock',
|
||||
'-metadir', str(work / 'states'), '-config', config.name, module]
|
||||
start = datetime.now(timezone.utc)
|
||||
with (work / 'tlc.log').open('w') as output:
|
||||
code = subprocess.run(command, cwd=work, stdout=output, stderr=subprocess.STDOUT).returncode
|
||||
output = (work / 'tlc.log').read_text(errors='replace')
|
||||
verdict, matched = trace_verdict(code, output, invariant, self.expected)
|
||||
self.last = {'verdict': verdict, 'model': module, 'matched': matched, 'expected': self.expected,
|
||||
'exitCode': code, 'command': command, 'cwd': str(work), 'modelHashes': models,
|
||||
'configHash': sha256(config), 'jarHash': sha256(self.jar),
|
||||
'generatedHashes': {name: sha256(work / name) for name in generated},
|
||||
'seconds': (datetime.now(timezone.utc) - start).total_seconds(),
|
||||
'log': str(work / 'tlc.log')}
|
||||
(work / 'result.json').write_text(json.dumps(self.last, indent=2) + '\n')
|
||||
return verdict, matched
|
||||
|
||||
|
||||
def read_regular(path):
|
||||
try:
|
||||
info = path.lstat()
|
||||
except OSError as error:
|
||||
raise Inconclusive(f'missing/unreadable evidence: {path.name}') from error
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_EVIDENCE_BYTES:
|
||||
raise Inconclusive('evidence must be a bounded regular file, not a symlink')
|
||||
try:
|
||||
return path.read_bytes()
|
||||
except OSError as error:
|
||||
raise Inconclusive(f'evidence read failed: {error}') from error
|
||||
|
||||
|
||||
def verify(request, report):
|
||||
ids = request.get('sessionIds', [])
|
||||
required = request.get('requiredModels', ['turns', 'wiring'])
|
||||
if request.get('version') != 1 or not ids or len(ids) != len(set(ids)) or not all(isinstance(s, str) and VALID_ID.fullmatch(s) for s in ids):
|
||||
raise ValueError('version 1 and nonempty unique safe sessionIds required')
|
||||
if not required or not set(required) <= {'turns', 'children', 'wiring'} or len(required) != len(set(required)):
|
||||
raise ValueError('requiredModels must be a nonempty unique supported model set')
|
||||
root = Path(request['sessionsRoot'])
|
||||
snapshot = report / 'input/sessions'
|
||||
snapshot.mkdir(parents=True)
|
||||
checks, observed = [], {}
|
||||
for sid in ids:
|
||||
try:
|
||||
if not stat.S_ISDIR((root / sid).lstat().st_mode):
|
||||
raise Inconclusive('expected session folder is missing or a symlink')
|
||||
data = read_regular(root / sid / 'log.jsonl')
|
||||
destination = snapshot / sid
|
||||
destination.mkdir()
|
||||
(destination / 'log.jsonl').write_bytes(data)
|
||||
lines = checked_lines(data)
|
||||
if lines[0].get('id') != sid:
|
||||
raise Inconclusive('session id does not match its expected folder')
|
||||
for line in lines:
|
||||
if line.get('kind') in saved.CHILD_KINDS:
|
||||
child = line.get('child')
|
||||
if not isinstance(child, str) or not VALID_ID.fullmatch(child):
|
||||
raise Inconclusive('unsafe child identifier')
|
||||
if (root / child / 'log.jsonl').exists() and child not in ids:
|
||||
raise Inconclusive('existing child log omitted from explicit session set')
|
||||
observed[sid] = lines
|
||||
except (Inconclusive, OSError) as error:
|
||||
checks.append({'kind': 'input', 'sessionId': sid, 'verdict': 'INCONCLUSIVE', 'detail': str(error)})
|
||||
java, version = find_java()
|
||||
jar = Path(os.environ.get('TLA2TOOLS_JAR', str(Path.home() / '.local/share/tla/tla2tools.jar'))).resolve()
|
||||
if not jar.is_file():
|
||||
raise RuntimeError(f'missing {jar} (set TLA2TOOLS_JAR)')
|
||||
executor = RetainedTLC(report, java, jar)
|
||||
# The archives expose generator functions with a module-local executor.
|
||||
# Bind that callback only in this new, single-threaded CLI process.
|
||||
saved.run_tlc = executor.run
|
||||
wiring.run_tlc = executor.run
|
||||
children_checked = 0
|
||||
for sid, lines in observed.items():
|
||||
try:
|
||||
if 'turns' in required:
|
||||
target = saved.turn_target(lines)
|
||||
executor.expected = len(target)
|
||||
saved.check_turns(target)
|
||||
checks.append({'kind': 'turns', 'sessionId': sid, **executor.last})
|
||||
if 'children' in required:
|
||||
target, final = saved.child_target(lines, str(snapshot))
|
||||
if target:
|
||||
executor.expected = len(target)
|
||||
saved.check_children(target, final)
|
||||
children_checked += 1
|
||||
checks.append({'kind': 'children', 'sessionId': sid, **executor.last})
|
||||
except (saved.Unreadable, KeyError, TypeError, ValueError) as error:
|
||||
checks.append({'kind': 'projection', 'sessionId': sid, 'verdict': 'INCONCLUSIVE', 'detail': str(error)})
|
||||
if 'children' in required and children_checked == 0:
|
||||
checks.append({'kind': 'children', 'verdict': 'INCONCLUSIVE', 'detail': 'required child work evidence is empty'})
|
||||
if 'wiring' in required:
|
||||
try:
|
||||
data = read_regular(Path(request['wiringTrace']))
|
||||
(report / 'input/wiring.log').write_bytes(data)
|
||||
steps, sessions = checked_wiring(data.decode('utf8'), request.get('processes', []))
|
||||
executor.expected = len(steps)
|
||||
wiring.check(steps, sessions)
|
||||
checks.append({'kind': 'wiring', **executor.last})
|
||||
except (Inconclusive, UnicodeDecodeError) as error:
|
||||
checks.append({'kind': 'wiring', 'verdict': 'INCONCLUSIVE', 'detail': str(error)})
|
||||
return {'version': 1, 'verdict': aggregate(checks), 'checks': checks, 'java': java, 'javaVersion': version,
|
||||
'jarHash': sha256(jar), 'requestHash': sha256(report / 'request.json'),
|
||||
'checkerHash': sha256(Path(__file__)),
|
||||
'projectionHashes': {name: sha256(ARCHIVE / name) for name in ('check_saved_logs.py', 'check_wiring_traces.py')},
|
||||
'scope': 'Observed TurnLifecycle/Subagents/Wiring projections only; not all per-PR model refinement.'}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--request', type=Path, required=True)
|
||||
parser.add_argument('--report', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
report = args.report.resolve()
|
||||
report.mkdir(parents=True, exist_ok=False)
|
||||
checks = []
|
||||
try:
|
||||
raw = read_regular(args.request)
|
||||
(report / 'request.json').write_bytes(raw)
|
||||
result = verify(json.loads(raw), report)
|
||||
except (OSError, ValueError, TypeError, KeyError, RuntimeError, Inconclusive) as error:
|
||||
result = {'version': 1, 'verdict': 'TOOL_ERROR', 'checks': checks, 'detail': str(error)}
|
||||
result['report'] = str(report)
|
||||
(report / 'verdict.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
print(json.dumps(result))
|
||||
return {'ACCEPTED': 0, 'REJECTED': 1, 'INCONCLUSIVE': 2, 'TOOL_ERROR': 3}[result['verdict']]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,189 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { artifacts, binary, repetitions, targetHelpers, worktree } from "../lib/context";
|
||||
import { inspectRuntime, requireAccepted, type Model, type RuntimeEvidence } from "./bridge";
|
||||
|
||||
const { startDynamicFakeGateway, fakeGatewayFinalText, fakeGatewayToolCall, FAKE_GATEWAY_MODEL } = await targetHelpers();
|
||||
type Outcome = { pid: number; exitCode: number | null; signal: string | null; stdout: string; stderr: string; args: string[] };
|
||||
|
||||
function fixture(label: string) {
|
||||
const root = mkdtempSync(join(artifacts, label + "-"));
|
||||
const home = join(root, "home"), workspace = join(root, "workspace"), trace = join(root, "wiring.log");
|
||||
mkdirSync(home, { mode: 0o700 }); mkdirSync(workspace, { mode: 0o700 });
|
||||
const sessionsRoot = join(home, ".fx/sessions/v2");
|
||||
const outcomes: Outcome[] = [];
|
||||
const active = new Set<ChildProcess>();
|
||||
const environment = (gateway: any) => ({
|
||||
PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: home, TMPDIR: root,
|
||||
AI_GATEWAY_API_KEY: "synthetic-tlc-runtime-fixture", FX_DISABLE_KEYCHAIN: "1", FX_E2E_DISABLE_DOTENV: "1",
|
||||
FX_GATEWAY_BASE_URL: gateway.baseUrl, FX_GATEWAY_CHAT_URL: gateway.chatUrl,
|
||||
FX_E2E_GATEWAY_CHAT_URL: gateway.chatUrl, FX_E2E_GATEWAY_MODELS_URL: `${gateway.baseUrl}/coding-agent/v1/models`,
|
||||
FX_MODEL: FAKE_GATEWAY_MODEL, FX_SESSIONS_V2: "1", FX_PERMISSION_MODE: "full-access",
|
||||
FX_AUTO_UPGRADE: "0", FX_SOUND: "0", NO_COLOR: "1", FX_TRACE_LOG: trace, FX_TRACE_SCOPES: "wiring",
|
||||
});
|
||||
function start(gateway: any, args: string[]) {
|
||||
const argv = ["ask", "--json", "--auto", ...args];
|
||||
const child = spawn(binary, argv, { cwd: workspace, env: environment(gateway), stdio: ["ignore", "pipe", "pipe"] });
|
||||
assert(child.pid, "fx did not start"); active.add(child);
|
||||
const stdout: Buffer[] = [], stderr: Buffer[] = [];
|
||||
child.stdout!.on("data", data => stdout.push(Buffer.from(data)));
|
||||
child.stderr!.on("data", data => stderr.push(Buffer.from(data)));
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), 30_000);
|
||||
const finished = new Promise<Outcome>((resolve, reject) => {
|
||||
child.on("error", error => { clearTimeout(timer); active.delete(child); reject(error); });
|
||||
child.on("close", (exitCode, signal) => {
|
||||
clearTimeout(timer); active.delete(child);
|
||||
const result = { pid: child.pid!, exitCode, signal, args: argv, stdout: Buffer.concat(stdout).toString(), stderr: Buffer.concat(stderr).toString() };
|
||||
outcomes.push(result);
|
||||
writeFileSync(join(root, "processes.json"), JSON.stringify(outcomes, null, 2));
|
||||
resolve(result);
|
||||
});
|
||||
});
|
||||
return { child, finished };
|
||||
}
|
||||
const ask = async (gateway: any, args: string[]) => {
|
||||
const outcome = await start(gateway, args).finished;
|
||||
assert.equal(outcome.exitCode, 0, outcome.stderr); assert.equal(outcome.signal, null);
|
||||
assert(!/panic|aborted/i.test(outcome.stderr), outcome.stderr);
|
||||
return { outcome, json: JSON.parse(outcome.stdout) };
|
||||
};
|
||||
const lines = (id: string) => readFileSync(join(sessionsRoot, id, "log.jsonl"), "utf8").trimEnd().split("\n").map(line => JSON.parse(line));
|
||||
const evidence = (ids: string[], models: Model[]): RuntimeEvidence => ({
|
||||
version: 1, sessionsRoot, sessionIds: ids, wiringTrace: trace,
|
||||
processes: outcomes.map(({ pid, exitCode, signal }) => ({ pid, exitCode, signal })), requiredModels: models,
|
||||
metadata: { worktree, binary, binaryHash: createHash("sha256").update(readFileSync(binary)).digest("hex"),
|
||||
head: Bun.spawnSync(["git", "rev-parse", "HEAD"], { cwd: worktree }).stdout.toString().trim() },
|
||||
});
|
||||
async function finish(gateway: any) {
|
||||
for (const child of active) child.kill("SIGKILL");
|
||||
if (active.size) await Promise.all([...active].map(child => new Promise(resolve => child.once("close", resolve))));
|
||||
writeFileSync(join(root, "gateway-requests.json"), JSON.stringify(gateway.requests.map((request: any) => request.body), null, 2));
|
||||
gateway.stop();
|
||||
}
|
||||
return { root, trace, sessionsRoot, outcomes, start, ask, lines, evidence, finish };
|
||||
}
|
||||
|
||||
async function waitFor(check: () => boolean, description: string) {
|
||||
const until = Date.now() + 15_000;
|
||||
while (Date.now() < until) { if (check()) return; await Bun.sleep(25); }
|
||||
throw new Error(`Owned fault boundary not reached: ${description}`);
|
||||
}
|
||||
|
||||
function frame(entry: Record<string, unknown>): string {
|
||||
const { crc: ignored, ...body } = entry;
|
||||
const payload = JSON.stringify(body).slice(0, -1);
|
||||
let crc = 0xffffffff;
|
||||
for (const byte of Buffer.from(payload)) {
|
||||
crc ^= byte;
|
||||
for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ ((crc & 1) ? 0x82f63b78 : 0);
|
||||
}
|
||||
return `${payload},"crc":"${((crc ^ 0xffffffff) >>> 0).toString(16).padStart(8, "0")}"}\n`;
|
||||
}
|
||||
|
||||
test("TLC accepts actual committed/resumed logs and rejects corrupted or incomplete evidence", async () => {
|
||||
for (let iteration = 0; iteration < repetitions; iteration++) {
|
||||
const run = fixture("commit-resume");
|
||||
const gateway = startDynamicFakeGateway((body: string) => fakeGatewayFinalText(body.includes("TLC_SECOND_PROMPT") ? "TLC_SECOND_ANSWER" : "TLC_FIRST_ANSWER"));
|
||||
try {
|
||||
const first = await run.ask(gateway, ["TLC_FIRST_PROMPT"]);
|
||||
expect(first.json.output).toBe("TLC_FIRST_ANSWER"); expect(first.outcome.stderr).toBe("");
|
||||
const id = first.json.session_id;
|
||||
const second = await run.ask(gateway, ["--resume-id", id, "TLC_SECOND_PROMPT"]);
|
||||
expect(second.json.session_id).toBe(id); expect(second.json.output).toBe("TLC_SECOND_ANSWER");
|
||||
expect(gateway.requests.at(-1).body).toContain("TLC_FIRST_ANSWER");
|
||||
expect(run.lines(id).filter(line => line.kind === "turn_committed")).toHaveLength(2);
|
||||
const evidence = run.evidence([id], ["turns", "wiring"]);
|
||||
requireAccepted(await inspectRuntime(evidence, join(run.root, "positive")));
|
||||
|
||||
const badRoot = join(run.root, "synthetic-invalid-log"); mkdirSync(badRoot);
|
||||
cpSync(join(run.sessionsRoot, id), join(badRoot, id), { recursive: true });
|
||||
const invalid = run.lines(id); invalid[1].kind = "turn_committed";
|
||||
writeFileSync(join(badRoot, id, "log.jsonl"), invalid.map(frame).join(""));
|
||||
const rejected = await inspectRuntime({ ...evidence, sessionsRoot: badRoot, requiredModels: ["turns"] }, join(run.root, "negative-log"));
|
||||
expect(rejected.verdict).toBe("REJECTED"); expect(() => requireAccepted(rejected)).toThrow();
|
||||
|
||||
const badTrace = join(run.root, "synthetic-invalid-wiring.log");
|
||||
const text = readFileSync(run.trace, "utf8"); const rows = text.trimEnd().split("\n");
|
||||
const pid = evidence.processes![0]!.pid;
|
||||
rows.splice(1, 0, `1 [wiring] action=EndTurn session=- end=commit pid=${pid}`);
|
||||
writeFileSync(badTrace, rows.join("\n") + "\n");
|
||||
expect((await inspectRuntime({ ...evidence, wiringTrace: badTrace, requiredModels: ["wiring"] }, join(run.root, "negative-wiring"))).verdict).toBe("REJECTED");
|
||||
|
||||
const emptyTrace = join(run.root, "empty.log"); writeFileSync(emptyTrace, "");
|
||||
expect((await inspectRuntime({ ...evidence, wiringTrace: emptyTrace }, join(run.root, "empty-trace"))).verdict).toBe("INCONCLUSIVE");
|
||||
expect((await inspectRuntime({ ...evidence, sessionIds: [id, "missing-session"] }, join(run.root, "missing-session"))).verdict).toBe("INCONCLUSIVE");
|
||||
expect((await inspectRuntime({ ...evidence, requiredModels: ["turns", "children"] }, join(run.root, "missing-child-work"))).verdict).toBe("INCONCLUSIVE");
|
||||
const corrupt = Buffer.from(readFileSync(join(run.sessionsRoot, id, "log.jsonl")));
|
||||
const at = corrupt.indexOf("TLC_FIRST_PROMPT"); assert(at >= 0, "corruption target not present");
|
||||
corrupt[at] ^= 1;
|
||||
writeFileSync(join(badRoot, id, "log.jsonl"), corrupt);
|
||||
const damaged = await inspectRuntime({ ...evidence, sessionsRoot: badRoot, requiredModels: ["turns"] }, join(run.root, "crc-corruption"));
|
||||
expect(damaged.verdict).toBe("INCONCLUSIVE"); expect(damaged.checks[0].detail).toContain("CRC32C mismatch");
|
||||
const invalidRequest = await inspectRuntime({ ...evidence, version: 2 } as unknown as RuntimeEvidence, join(run.root, "invalid-request"));
|
||||
expect(invalidRequest.verdict).toBe("TOOL_ERROR"); expect(() => requireAccepted(invalidRequest)).toThrow();
|
||||
writeFileSync(join(badRoot, id, "log.jsonl"), readFileSync(join(run.sessionsRoot, id, "log.jsonl")).subarray(0, -1));
|
||||
expect((await inspectRuntime({ ...evidence, sessionsRoot: badRoot, requiredModels: ["turns"] }, join(run.root, "torn-tail"))).verdict).toBe("INCONCLUSIVE");
|
||||
} finally { await run.finish(gateway); }
|
||||
}
|
||||
}, Math.max(120_000, repetitions * 120_000));
|
||||
|
||||
test("TLC accepts real owned-process crash followed by interrupted recovery and commit", async () => {
|
||||
for (let iteration = 0; iteration < repetitions; iteration++) {
|
||||
const run = fixture("crash-resume");
|
||||
const gateway = startDynamicFakeGateway((body: string) => body.includes("TLC_RECOVERY_PROMPT")
|
||||
? fakeGatewayFinalText("TLC_RECOVERED_ANSWER")
|
||||
: new Response(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(': held response\n\n')); } }), { headers: { "content-type": "text/event-stream" } }));
|
||||
try {
|
||||
const pending = run.start(gateway, ["TLC_CRASH_PROMPT"]);
|
||||
let id = "";
|
||||
await waitFor(() => {
|
||||
if (!existsSync(run.sessionsRoot) || !gateway.requests.length) return false;
|
||||
for (const name of readdirSync(run.sessionsRoot)) {
|
||||
const path = join(run.sessionsRoot, name, "log.jsonl");
|
||||
if (existsSync(path) && readFileSync(path, "utf8").includes("TLC_CRASH_PROMPT")) { id = name; return true; }
|
||||
}
|
||||
return false;
|
||||
}, "durable prompt and an outstanding local Gateway request");
|
||||
const before = run.lines(id);
|
||||
expect(before.some(line => line.kind === "turn_started")).toBe(true);
|
||||
expect(before.some(line => line.kind === "turn_committed")).toBe(false);
|
||||
writeFileSync(join(run.root, "fault-boundary.json"), JSON.stringify({ pid: pending.child.pid, id, lines: before, gatewayRequests: gateway.requests.length }, null, 2));
|
||||
assert(pending.child.kill("SIGKILL"), "owned process was not killed");
|
||||
const killed = await pending.finished; expect(killed.signal).toBe("SIGKILL"); expect(killed.exitCode).toBe(null);
|
||||
const resumed = await run.ask(gateway, ["--resume-id", id, "TLC_RECOVERY_PROMPT"]);
|
||||
expect(resumed.json.output).toBe("TLC_RECOVERED_ANSWER"); expect(resumed.json.session_id).toBe(id);
|
||||
expect(gateway.requests.at(-1).body).toContain("TLC_CRASH_PROMPT");
|
||||
expect(run.lines(id).some(line => line.kind === "turn_interrupted" && line.reason === "crash")).toBe(true);
|
||||
requireAccepted(await inspectRuntime(run.evidence([id], ["turns", "wiring"]), join(run.root, "proof")));
|
||||
} finally { await run.finish(gateway); }
|
||||
}
|
||||
}, Math.max(120_000, repetitions * 120_000));
|
||||
|
||||
test("TLC accepts actual parent/child work and retains every expected child log", async () => {
|
||||
for (let iteration = 0; iteration < repetitions; iteration++) {
|
||||
const run = fixture("child-work");
|
||||
const gateway = startDynamicFakeGateway((body: string) => {
|
||||
if (!body.includes('"name":"subagent"')) return fakeGatewayFinalText("TLC_CHILD_ANSWER");
|
||||
if (body.includes("TLC_CHILD_ANSWER")) return fakeGatewayFinalText("TLC_PARENT_DONE");
|
||||
return fakeGatewayToolCall("tlc-delegate", "subagent", { request: { action: "run", task: "Report the child marker." } });
|
||||
}, { classifierDecision: "clear" });
|
||||
try {
|
||||
const result = await run.ask(gateway, ["Delegate the TLC marker report."]);
|
||||
expect(result.json.output).toBe("TLC_PARENT_DONE");
|
||||
const parent = result.json.session_id;
|
||||
const childEvents = run.lines(parent).filter(line => line.kind === "child_spawned" || line.kind === "child_finished");
|
||||
expect(childEvents.map(line => [line.kind, line.outcome ?? null])).toEqual([["child_spawned", null], ["child_finished", "ok"]]);
|
||||
const child = childEvents[0].child;
|
||||
expect(childEvents[1].child).toBe(child); expect(childEvents[1].work_id).toBe(childEvents[0].work_id);
|
||||
expect(run.lines(child)[0].parent).toBe(parent); expect(JSON.stringify(run.lines(child))).toContain("TLC_CHILD_ANSWER");
|
||||
const resultProof = await inspectRuntime(run.evidence([parent, child], ["turns", "children", "wiring"]), join(run.root, "proof"));
|
||||
requireAccepted(resultProof);
|
||||
expect(resultProof.checks.filter(check => check.kind === "children")).toHaveLength(1);
|
||||
expect((await inspectRuntime(run.evidence([parent], ["turns", "children"]), join(run.root, "omitted-child"))).verdict).toBe("INCONCLUSIVE");
|
||||
} finally { await run.finish(gateway); }
|
||||
}
|
||||
}, Math.max(120_000, repetitions * 120_000));
|
||||
@@ -0,0 +1,274 @@
|
||||
// Connects behavior observed from the real fx binary to TLC.
|
||||
//
|
||||
// 1. ShellOracle prints the model's emission set and Record(e) for each one.
|
||||
// Every modeled emission must have been observed, or the check is INCONCLUSIVE.
|
||||
// 2. The observations become ShellObserved.tla. ShellTrace replays them through
|
||||
// ShellContract's own Process step. ACCEPTED requires TLC exit 12 on the
|
||||
// NotAllMatched sentinel with every turn matched; a completed search is REJECTED.
|
||||
// 3. After acceptance, each design property is checked on the same replay.
|
||||
//
|
||||
// Strict verdicts: unreadable or missing evidence is never acceptance.
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { join, resolve } from "node:path";
|
||||
import { TLA_ROOT, tlcToolchain } from "../lib/tlc";
|
||||
|
||||
export type Design = "current" | "proposed";
|
||||
export type Verdict = "ACCEPTED" | "REJECTED" | "INCONCLUSIVE" | "TOOL_ERROR";
|
||||
export type PropertyVerdict = "HOLDS" | "VIOLATED" | "NOT_CHECKED" | "TOOL_ERROR";
|
||||
|
||||
export interface ObservedCall {
|
||||
id: string;
|
||||
intent: string;
|
||||
shape: string;
|
||||
/** Exact argument text sent to fx. */
|
||||
sent: string;
|
||||
/** null when the result could not be read. */
|
||||
executed: boolean | null;
|
||||
decoded: string;
|
||||
outcome: string;
|
||||
/** How the call appeared in the next request; absent when fx sent none. */
|
||||
history: { observed: true; input: unknown } | { observed: false };
|
||||
resultBody: string;
|
||||
}
|
||||
|
||||
export interface ObservedTurn {
|
||||
name: string;
|
||||
calls: ObservedCall[];
|
||||
ended: "final" | "guard" | "unknown";
|
||||
}
|
||||
|
||||
export interface Prediction { accepted: boolean; decoded: string; history: string; outcome: string }
|
||||
|
||||
export interface RuntimeResult {
|
||||
version: 1;
|
||||
design: Design;
|
||||
verdict: Verdict;
|
||||
detail: string;
|
||||
turns: number;
|
||||
matchedTurns: number;
|
||||
mismatch?: { turn: string; call?: string; observed?: unknown; predicted?: unknown };
|
||||
/** Every observed call that differs from the model's prediction. Diagnostic only;
|
||||
* the verdict comes from TLC. */
|
||||
divergences?: Array<{ turn: string; call: string; observed: unknown; predicted: unknown }>;
|
||||
properties: Record<string, { verdict: PropertyVerdict; detail?: string }>;
|
||||
report: string;
|
||||
hashes: Record<string, string>;
|
||||
}
|
||||
|
||||
/** Design properties that the runtime trace can decide. AdmissionConsistency needs
|
||||
* what permission admission saw, which the runtime does not expose. */
|
||||
export const RUNTIME_PROPERTIES = ["Exactness", "Completeness", "HistoryFidelity", "ZshForgiving", "GuardBound"] as const;
|
||||
|
||||
const tlaDir = join(TLA_ROOT, "shell_command");
|
||||
const MODULES = ["ShellContract.tla", "ShellTrace.tla", "ShellOracle.tla"];
|
||||
|
||||
const sha256 = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
function word(value: string): string {
|
||||
if (!/^[a-z_:]+$/.test(value)) throw new Error(`refusing unsafe TLA+ string: ${JSON.stringify(value)}`);
|
||||
return `"${value}"`;
|
||||
}
|
||||
|
||||
function tryParse(text: string): { ok: true; value: unknown } | { ok: false } {
|
||||
try { return { ok: true, value: JSON.parse(text) }; } catch { return { ok: false }; }
|
||||
}
|
||||
|
||||
const isObject = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
/** The form an accepted call takes in history under each design. */
|
||||
function canonical(design: Design, input: unknown): boolean {
|
||||
if (!isObject(input)) return false;
|
||||
if (design === "current") {
|
||||
const keys = Object.keys(input);
|
||||
return keys.length === 1 && keys[0] === "request" && isObject(input.request) && typeof input.request.action === "string";
|
||||
}
|
||||
return !("request" in input) && !("action" in input) &&
|
||||
(typeof input.command === "string" || typeof input.session_id === "string");
|
||||
}
|
||||
|
||||
export function historyClass(design: Design, call: ObservedCall): string {
|
||||
if (!call.history.observed) return "unobserved";
|
||||
const input = call.history.input;
|
||||
if (call.executed && canonical(design, input)) return "canonical";
|
||||
// Text that is not JSON cannot be replayed; fx shows {} so providers accept the history.
|
||||
if (!call.executed && call.shape === "invalid_json" && isDeepStrictEqual(input, {})) return "placeholder";
|
||||
const sent = tryParse(call.sent);
|
||||
if (sent.ok && isDeepStrictEqual(input, sent.value)) return `as_sent:${call.shape}`;
|
||||
return `rewrapped:${call.shape}`;
|
||||
}
|
||||
|
||||
export function abstractCall(design: Design, call: ObservedCall) {
|
||||
return {
|
||||
intent: call.intent,
|
||||
shape: call.shape,
|
||||
accepted: call.executed === true,
|
||||
decoded: call.executed ? call.decoded : "none",
|
||||
history: historyClass(design, call),
|
||||
outcome: call.outcome,
|
||||
};
|
||||
}
|
||||
|
||||
export function observedModule(design: Design, turns: ObservedTurn[]): string {
|
||||
const record = (call: ObservedCall) => {
|
||||
const a = abstractCall(design, call);
|
||||
return `[e |-> [intent |-> ${word(a.intent)}, shape |-> ${word(a.shape)}], accepted |-> ${a.accepted ? "TRUE" : "FALSE"}, ` +
|
||||
`decoded |-> ${word(a.decoded)}, history |-> ${word(a.history)}, outcome |-> ${word(a.outcome)}]`;
|
||||
};
|
||||
const turnText = (turn: ObservedTurn) =>
|
||||
` [calls |-> << ${turn.calls.map(record).join(",\n ")} >>,\n ended |-> ${word(turn.ended)}]`;
|
||||
return [
|
||||
"---------------------------- MODULE ShellObserved ----------------------------",
|
||||
`(* Generated by bun/bridge.ts from real fx runs. Design under test: ${design}. *)`,
|
||||
"Observed == <<",
|
||||
turns.map(turnText).join(",\n"),
|
||||
">>",
|
||||
"=============================================================================",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function config(design: Design, body: string): string {
|
||||
return [
|
||||
body,
|
||||
"CONSTANTS",
|
||||
` Design = "${design}"`,
|
||||
' AdmissionReads = "normalized"',
|
||||
" GuessAmbiguous = FALSE",
|
||||
" MaxCalls = 3",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
interface TlcRun { exitCode: number; log: string; dir: string }
|
||||
|
||||
async function runTlc(dir: string, spec: string, cfg: string, extra: string[] = []): Promise<TlcRun> {
|
||||
writeFileSync(join(dir, "check.cfg"), cfg, { mode: 0o600 });
|
||||
const { java, jar } = tlcToolchain();
|
||||
mkdirSync(join(dir, ".tlc-tmp"), { recursive: true });
|
||||
const command = [java, "-XX:+UseParallelGC", `-Djava.io.tmpdir=${join(dir, ".tlc-tmp")}`, "-cp", jar, "tlc2.TLC", "-workers", "1",
|
||||
"-metadir", join(dir, ".tlc-states"), "-config", "check.cfg", ...extra, spec];
|
||||
const child = Bun.spawn(command, { cwd: dir, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
||||
const [exitCode, stdout, stderr] = await Promise.all([
|
||||
child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
|
||||
]);
|
||||
const log = `${stderr}\n${stdout}`;
|
||||
writeFileSync(join(dir, "tlc.log"), log, { mode: 0o600 });
|
||||
writeFileSync(join(dir, "result.json"), JSON.stringify({ command, exitCode }, null, 2), { mode: 0o600 });
|
||||
return { exitCode, log, dir };
|
||||
}
|
||||
|
||||
function stage(root: string, name: string, observed?: string): string {
|
||||
const dir = join(root, name);
|
||||
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
for (const module of MODULES) copyFileSync(join(tlaDir, module), join(dir, module));
|
||||
// Only ShellTrace extends ShellObserved; TLC parses just the root module's imports.
|
||||
if (observed !== undefined) writeFileSync(join(dir, "ShellObserved.tla"), observed, { mode: 0o600 });
|
||||
return dir;
|
||||
}
|
||||
|
||||
const completed = (log: string) => log.includes("Model checking completed. No error has been found.");
|
||||
const violated = (log: string, name: string) => log.includes(`Invariant ${name} is violated`);
|
||||
|
||||
/** The model's emission set and predicted record per emission, from TLC itself. */
|
||||
export async function modelOracle(design: Design, reportRoot: string) {
|
||||
mkdirSync(reportRoot, { recursive: true, mode: 0o700 });
|
||||
const dir = stage(mkdtempSync(join(reportRoot, `oracle-${design}-`)), "oracle");
|
||||
const run = await runTlc(dir, "ShellOracle.tla", config(design, "INIT Init\nNEXT OracleNext"));
|
||||
// TLC wraps long tuples across lines; join them before matching.
|
||||
const text = run.log.replace(/\n\s+/g, " ");
|
||||
const predictions = new Map<string, Prediction>();
|
||||
const line = /<<\s*"FXT_PREDICT",\s*"([a-z_]+)",\s*"([a-z_]+)",\s*(TRUE|FALSE),\s*"([a-z_]+)",\s*"([a-z_:]+)",\s*"([a-z_]+)"\s*>>/g;
|
||||
for (const m of text.matchAll(line)) {
|
||||
predictions.set(`${m[1]}/${m[2]}`, { accepted: m[3] === "TRUE", decoded: m[4]!, history: m[5]!, outcome: m[6]! });
|
||||
}
|
||||
const expected = Number(/<<\s*"FXT_EMITS",\s*(\d+)\s*>>/.exec(text)?.[1] ?? NaN);
|
||||
if (run.exitCode !== 0 || !completed(run.log) || !(expected > 0) || predictions.size !== expected) {
|
||||
throw new Error(`ShellOracle failed (exit ${run.exitCode}, ${predictions.size} of ${expected} predictions parsed); ` +
|
||||
`see ${join(dir, "tlc.log")}`);
|
||||
}
|
||||
return { predictions, emits: [...predictions.keys()].sort(), log: join(dir, "tlc.log") };
|
||||
}
|
||||
|
||||
/** Replays the observations through the model for one design and checks the design properties. */
|
||||
export async function checkRuntime(turns: ObservedTurn[], design: Design, reportRoot: string,
|
||||
options: { requireCoverage?: boolean } = {}): Promise<RuntimeResult> {
|
||||
mkdirSync(reportRoot, { recursive: true, mode: 0o700 });
|
||||
const root = mkdtempSync(join(reportRoot, `${design}-`));
|
||||
const properties: RuntimeResult["properties"] = Object.fromEntries(
|
||||
RUNTIME_PROPERTIES.map((p) => [p, { verdict: "NOT_CHECKED" as PropertyVerdict }]));
|
||||
const hashes: Record<string, string> = Object.fromEntries(
|
||||
[...MODULES.map((m) => [m, sha256(readFileSync(join(tlaDir, m)))]), ["tla2tools.jar", tlcToolchain().jarSha256]]);
|
||||
const finish = (verdict: Verdict, detail: string, extra: Partial<RuntimeResult> = {}): RuntimeResult => {
|
||||
const result: RuntimeResult = { version: 1, design, verdict, detail, turns: turns.length, matchedTurns: 0,
|
||||
properties, report: root, hashes, ...extra };
|
||||
writeFileSync(join(root, "verdict.json"), JSON.stringify(result, null, 2), { mode: 0o600 });
|
||||
return result;
|
||||
};
|
||||
writeFileSync(join(root, "observations.json"), JSON.stringify(turns, null, 2), { mode: 0o600 });
|
||||
hashes["observations.json"] = sha256(readFileSync(join(root, "observations.json")));
|
||||
|
||||
if (!turns.length || turns.some((t) => !t.calls.length)) return finish("INCONCLUSIVE", "empty observation");
|
||||
const unreadable = turns.flatMap((t) => t.calls.filter((c) => c.executed === null).map((c) => `${t.name}/${c.id}`));
|
||||
if (unreadable.length) return finish("INCONCLUSIVE", `unreadable results: ${unreadable.join(", ")}`);
|
||||
const unended = turns.filter((t) => t.ended === "unknown").map((t) => t.name);
|
||||
if (unended.length) return finish("INCONCLUSIVE", `turn end not established: ${unended.join(", ")}`);
|
||||
|
||||
let oracle: Awaited<ReturnType<typeof modelOracle>>;
|
||||
try { oracle = await modelOracle(design, root); } catch (error) { return finish("TOOL_ERROR", String(error)); }
|
||||
if (options.requireCoverage !== false) {
|
||||
const seen = new Set(turns.flatMap((t) => t.calls.map((c) => `${c.intent}/${c.shape}`)));
|
||||
const missing = oracle.emits.filter((key) => !seen.has(key));
|
||||
if (missing.length) return finish("INCONCLUSIVE", `modeled calls never observed: ${missing.join(", ")}`);
|
||||
}
|
||||
|
||||
let observed: string;
|
||||
try { observed = observedModule(design, turns); } catch (error) { return finish("TOOL_ERROR", String(error)); }
|
||||
hashes["ShellObserved.tla"] = sha256(observed);
|
||||
|
||||
const conformance = await runTlc(stage(root, "conformance", observed), "ShellTrace.tla",
|
||||
config(design, "SPECIFICATION TraceSpec\nINVARIANT NotAllMatched"), ["-deadlock"]);
|
||||
const matched = new Set([...conformance.log.matchAll(/<<"FXT_MATCHED", (\d+)>>/g)].map((m) => Number(m[1])));
|
||||
const accepted = conformance.exitCode === 12 && violated(conformance.log, "NotAllMatched") && matched.size === turns.length;
|
||||
if (!accepted) {
|
||||
if (!(conformance.exitCode === 0 && completed(conformance.log))) {
|
||||
return finish("TOOL_ERROR", `unexpected TLC result (exit ${conformance.exitCode}); see ${join(conformance.dir, "tlc.log")}`,
|
||||
{ matchedTurns: matched.size });
|
||||
}
|
||||
const index = matched.size; // turns replay in order; this is the first unmatched one
|
||||
const turn = turns[index]!;
|
||||
const differs = (call: ObservedCall) => {
|
||||
const p = oracle.predictions.get(`${call.intent}/${call.shape}`);
|
||||
const a = abstractCall(design, call);
|
||||
return !p || p.accepted !== a.accepted || p.decoded !== a.decoded || p.outcome !== a.outcome ||
|
||||
(a.history !== "unobserved" && p.history !== a.history);
|
||||
};
|
||||
const divergences = turns.flatMap((t) => t.calls.filter(differs).map((call) => ({
|
||||
turn: t.name, call: `${call.intent}/${call.shape}`, observed: abstractCall(design, call),
|
||||
predicted: oracle.predictions.get(`${call.intent}/${call.shape}`),
|
||||
})));
|
||||
const wrong = turn.calls.find(differs);
|
||||
const mismatch = wrong
|
||||
? { turn: turn.name, call: `${wrong.intent}/${wrong.shape}`, observed: abstractCall(design, wrong),
|
||||
predicted: oracle.predictions.get(`${wrong.intent}/${wrong.shape}`) }
|
||||
: { turn: turn.name, observed: { ended: turn.ended, calls: turn.calls.length } };
|
||||
return finish("REJECTED", `runtime diverges from the ${design} model at turn ${index + 1} (${turn.name}); ` +
|
||||
`${divergences.length} call(s) differ from its predictions`, { matchedTurns: matched.size, mismatch, divergences });
|
||||
}
|
||||
|
||||
for (const property of RUNTIME_PROPERTIES) {
|
||||
const run = await runTlc(stage(root, `property-${property}`, observed), "ShellTrace.tla",
|
||||
config(design, `SPECIFICATION TraceSpec\nINVARIANT ${property}`), ["-deadlock"]);
|
||||
if (run.exitCode === 0 && completed(run.log)) {
|
||||
properties[property] = { verdict: "HOLDS" };
|
||||
} else if (run.exitCode === 12 && violated(run.log, property)) {
|
||||
const last = run.log.lastIndexOf("/\\ calls = ");
|
||||
properties[property] = { verdict: "VIOLATED", detail: last >= 0 ? run.log.slice(last, last + 400).split("\n/\\")[0] : undefined };
|
||||
} else {
|
||||
properties[property] = { verdict: "TOOL_ERROR", detail: `exit ${run.exitCode}; see ${join(run.dir, "tlc.log")}` };
|
||||
}
|
||||
}
|
||||
return finish("ACCEPTED", `all ${turns.length} observed turns match the ${design} model`, { matchedTurns: matched.size });
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
// Runs the real fx binary through every shell call the shell-contract TLA+
|
||||
// model covers, then has TLC replay what fx did through the model
|
||||
// (tests/tla/shell_command/ShellContract.tla). No model calls: a local fake Gateway serves
|
||||
// exact argument bytes.
|
||||
//
|
||||
// 1. The binary must conform to exactly one modeled design, the current or the
|
||||
// proposed one, and the design properties on the observed trace must match the
|
||||
// exhaustive model check of that design.
|
||||
// 2. Controls: tampered, partial and empty evidence must not be accepted.
|
||||
// 3. With FX_TLA_PROPOSED=1: the binary must conform to the proposed design
|
||||
// with every property holding. This fails until fx implements it.
|
||||
import { test } from "bun:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { artifacts, binary, targetHelpers } from "../lib/context";
|
||||
import { checkRuntime, modelOracle, RUNTIME_PROPERTIES, type Design, type ObservedTurn, type RuntimeResult } from "./bridge";
|
||||
import { collectObservations, turnSpecs } from "./scenarios";
|
||||
|
||||
const helpers = await targetHelpers();
|
||||
const reports = join(artifacts, "tlc");
|
||||
const TIMEOUT = 900_000;
|
||||
|
||||
// Property verdicts from run-all.sh's exhaustive check of each design.
|
||||
const EXPECTED_PROPERTIES: Record<Design, Record<string, string>> = {
|
||||
current: { Exactness: "HOLDS", GuardBound: "HOLDS", Completeness: "VIOLATED", HistoryFidelity: "VIOLATED", ZshForgiving: "VIOLATED" },
|
||||
proposed: { Exactness: "HOLDS", GuardBound: "HOLDS", Completeness: "HOLDS", HistoryFidelity: "HOLDS", ZshForgiving: "HOLDS" },
|
||||
};
|
||||
|
||||
let observed: Promise<ObservedTurn[]> | undefined;
|
||||
function observations(): Promise<ObservedTurn[]> {
|
||||
observed ??= (async () => {
|
||||
const oracle = await modelOracle("current", reports);
|
||||
const turns = await collectObservations({ binary, helpers, artifacts: join(artifacts, "fixtures") }, turnSpecs(oracle.emits));
|
||||
writeFileSync(join(artifacts, "observations.json"), JSON.stringify(turns, null, 2), { mode: 0o600 });
|
||||
return turns;
|
||||
})();
|
||||
return observed;
|
||||
}
|
||||
|
||||
const verdicts = new Map<Design, Promise<RuntimeResult>>();
|
||||
function verdict(design: Design): Promise<RuntimeResult> {
|
||||
if (!verdicts.has(design)) {
|
||||
verdicts.set(design, (async () => {
|
||||
const result = await checkRuntime(await observations(), design, reports);
|
||||
writeFileSync(join(artifacts, `${design}-design.json`), JSON.stringify(result, null, 2), { mode: 0o600 });
|
||||
return result;
|
||||
})());
|
||||
}
|
||||
return verdicts.get(design)!;
|
||||
}
|
||||
|
||||
function explain(result: RuntimeResult): string {
|
||||
return `${result.design} ${result.verdict}: ${result.detail}\nmismatch: ${JSON.stringify(result.mismatch)}\n` +
|
||||
`properties: ${JSON.stringify(result.properties)}\nevidence: ${result.report}`;
|
||||
}
|
||||
|
||||
async function acceptedDesign(): Promise<Design> {
|
||||
const [current, proposed] = await Promise.all([verdict("current"), verdict("proposed")]);
|
||||
const accepted = [current, proposed].filter((result) => result.verdict === "ACCEPTED");
|
||||
assert.equal(accepted.length, 1, `expected exactly one design to match\n${explain(current)}\n${explain(proposed)}`);
|
||||
return accepted[0]!.design;
|
||||
}
|
||||
|
||||
test("real fx conforms to one modeled design", async () => {
|
||||
const design = await acceptedDesign();
|
||||
const result = await verdict(design);
|
||||
const turns = await observations();
|
||||
assert.equal(result.matchedTurns, turns.length);
|
||||
for (const property of RUNTIME_PROPERTIES) {
|
||||
assert.equal(result.properties[property]?.verdict, EXPECTED_PROPERTIES[design][property], `${property}: ${explain(result)}`);
|
||||
}
|
||||
writeFileSync(join(artifacts, "conforming-design.txt"), `${design}\n`, { mode: 0o600 });
|
||||
}, TIMEOUT);
|
||||
|
||||
test("bridge rejects tampered, partial and empty evidence", async () => {
|
||||
const design = await acceptedDesign();
|
||||
const turns = await observations();
|
||||
const tampered: ObservedTurn[] = structuredClone(turns);
|
||||
const flip = tampered.find((t) => t.calls.length === 1 && t.calls[0]!.executed === false)!;
|
||||
flip.calls[0]!.executed = true;
|
||||
flip.calls[0]!.decoded = flip.calls[0]!.intent;
|
||||
flip.calls[0]!.outcome = "ok";
|
||||
const rejected = await checkRuntime(tampered, design, join(reports, "control-tampered"));
|
||||
writeFileSync(join(artifacts, "control-tampered.json"), JSON.stringify(rejected, null, 2), { mode: 0o600 });
|
||||
assert.equal(rejected.verdict, "REJECTED", explain(rejected));
|
||||
assert.equal(rejected.mismatch?.turn, flip.name, explain(rejected));
|
||||
|
||||
// Drop the only turn that covers some modeled call, so coverage is incomplete.
|
||||
const key = (call: { intent: string; shape: string }) => `${call.intent}/${call.shape}`;
|
||||
const counts = new Map<string, number>();
|
||||
for (const turn of turns) for (const call of turn.calls) counts.set(key(call), (counts.get(key(call)) ?? 0) + 1);
|
||||
const only = turns.find((t) => t.calls.length === 1 && counts.get(key(t.calls[0]!)) === 1)!;
|
||||
const partial = turns.filter((t) => t !== only);
|
||||
const missing = await checkRuntime(partial, design, join(reports, "control-partial"));
|
||||
assert.equal(missing.verdict, "INCONCLUSIVE", explain(missing));
|
||||
|
||||
const empty = await checkRuntime([], design, join(reports, "control-empty"));
|
||||
assert.equal(empty.verdict, "INCONCLUSIVE", explain(empty));
|
||||
}, TIMEOUT);
|
||||
|
||||
test.skipIf(process.env.FX_TLA_PROPOSED !== "1")("real fx meets the proposed design", async () => {
|
||||
const result = await verdict("proposed");
|
||||
assert.equal(result.verdict, "ACCEPTED", explain(result));
|
||||
for (const property of RUNTIME_PROPERTIES) {
|
||||
assert.equal(result.properties[property]?.verdict, "HOLDS", `${property}: ${explain(result)}`);
|
||||
}
|
||||
}, TIMEOUT);
|
||||
@@ -0,0 +1,271 @@
|
||||
// Drives the real fx binary with every shell call the TLA+ model covers and
|
||||
// records what fx actually did. No model calls: a local fake Gateway serves
|
||||
// exact argument bytes, so shapes such as invalid JSON reach fx unchanged.
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { ObservedCall, ObservedTurn } from "./bridge";
|
||||
|
||||
export interface Emission { intent: string; shape: string }
|
||||
export interface TurnSpec { name: string; calls: Emission[] }
|
||||
export interface RunContext {
|
||||
binary: string;
|
||||
/** The target worktree's tests/e2e/tmux-helpers.ts module. */
|
||||
helpers: any;
|
||||
/** Parent directory for fixtures; they are kept as evidence. */
|
||||
artifacts: string;
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
const SESSION_INTENTS = new Set(["type", "observe", "stop"]);
|
||||
// Prints which shell ran the command. Kept free of quotes so every encoding carries it unchanged.
|
||||
const MARK = "echo FXT:${BASH_VERSION:+bash}${ZSH_VERSION:+zsh}";
|
||||
// Bash-style text that zsh treats differently: an unmatched glob, a =word, and ${arr[0]}.
|
||||
const EQ = "echo before; ls *.fxt-nomatch 2>/dev/null; echo =====; arr=(a b); echo ${arr[0]}; echo after";
|
||||
const EQ_BASH_LINES = ["before", "=====", "a", "after"];
|
||||
const TYPED = "FXT_TYPED\n";
|
||||
const BASH = { kind: "executable", path: "/bin/bash" };
|
||||
|
||||
/** fx's current advertised fields, with action. */
|
||||
function today(intent: string, sid: string): Record<string, unknown> {
|
||||
switch (intent) {
|
||||
case "run": return { action: "run", command: MARK };
|
||||
case "run_eq": return { action: "run", command: EQ };
|
||||
case "run_bash": return { action: "run", command: MARK, shell: BASH };
|
||||
case "run_bash_tty": return { action: "run", command: MARK, shell: BASH, tty: true, yield_time_ms: 2000 };
|
||||
case "type": return { action: "interact", session_id: sid, chars: TYPED, yield_time_ms: 1000 };
|
||||
case "observe": return { action: "interact", session_id: sid };
|
||||
case "stop": return { action: "stop", session_id: sid };
|
||||
}
|
||||
throw new Error(`unknown intent ${intent}`);
|
||||
}
|
||||
|
||||
/** The proposed flat form: no action, nothing nested. */
|
||||
function plan(intent: string, sid: string): Record<string, unknown> {
|
||||
switch (intent) {
|
||||
case "run": return { command: MARK };
|
||||
case "run_eq": return { command: EQ };
|
||||
case "run_bash": return { command: MARK, shell: "bash" };
|
||||
case "run_bash_tty": return { command: MARK, shell: "bash", interactive: true, wait: 2 };
|
||||
case "type": return { session_id: sid, input: TYPED, wait: 1 };
|
||||
case "observe": return { session_id: sid, wait: 1 };
|
||||
case "stop": return { session_id: sid, stop: true };
|
||||
}
|
||||
throw new Error(`unknown intent ${intent}`);
|
||||
}
|
||||
|
||||
/** The current field names without action. */
|
||||
function legacy(intent: string, sid: string): Record<string, unknown> {
|
||||
switch (intent) {
|
||||
case "run": return { command: MARK, yield_time_ms: 30_000 };
|
||||
case "run_eq": return { command: EQ, yield_time_ms: 30_000 };
|
||||
case "run_bash": return { command: MARK, shell: BASH };
|
||||
case "run_bash_tty": return { command: MARK, shell: BASH, tty: true, yield_time_ms: 2000 };
|
||||
case "type": return { session_id: sid, chars: TYPED, yield_time_ms: 1000 };
|
||||
case "observe": return { session_id: sid, yield_time_ms: 1000 };
|
||||
case "stop": return { session_id: sid, force: true };
|
||||
}
|
||||
throw new Error(`unknown intent ${intent}`);
|
||||
}
|
||||
|
||||
/** The proposed flat form plus one default-valued field that does not apply. */
|
||||
function irrelevant(intent: string, sid: string): Record<string, unknown> {
|
||||
const base = plan(intent, sid);
|
||||
if (intent === "observe") return { ...base, cwd: "." };
|
||||
if (intent === "stop") return { ...base, profile: "user" };
|
||||
return { ...base, force: false };
|
||||
}
|
||||
|
||||
/** Exact argument text for one modeled call. */
|
||||
export function encode(e: Emission, sid: string): string {
|
||||
switch (e.shape) {
|
||||
case "flat_new": return JSON.stringify(plan(e.intent, sid));
|
||||
case "flat_action": return JSON.stringify(today(e.intent, sid));
|
||||
case "wrapped": return JSON.stringify({ request: today(e.intent, sid) });
|
||||
case "wrapped_string": return JSON.stringify({ request: JSON.stringify(today(e.intent, sid)) });
|
||||
// A string request whose JSON is cut off before its closing quote and brace.
|
||||
case "wrapped_broken": return JSON.stringify({ request: JSON.stringify(today(e.intent, sid)).slice(0, -2) });
|
||||
// Unescaped quotes inside a string: the whole argument text is not JSON.
|
||||
case "invalid_json": return `${JSON.stringify(today(e.intent, sid)).slice(0, -1)},"note":"say "hi""}`;
|
||||
case "legacy_names": return JSON.stringify(legacy(e.intent, sid));
|
||||
case "with_irrelevant": return JSON.stringify(irrelevant(e.intent, sid));
|
||||
case "both_input": return JSON.stringify({ command: MARK, session_id: sid, input: TYPED, wait: 1 });
|
||||
case "both_bare": return JSON.stringify({ command: MARK, session_id: sid });
|
||||
}
|
||||
throw new Error(`unknown shape ${e.shape}`);
|
||||
}
|
||||
|
||||
const textOf = (value: any): string =>
|
||||
typeof value === "string" ? value
|
||||
: Array.isArray(value) ? value.map(textOf).join("\n")
|
||||
: value && typeof value === "object" ? textOf(value.text ?? value.value ?? value.content ?? "") : "";
|
||||
|
||||
function parts(body: string): any[] {
|
||||
const prompt = JSON.parse(body).prompt ?? [];
|
||||
return prompt.flatMap((m: any) => Array.isArray(m.content) ? m.content : []);
|
||||
}
|
||||
|
||||
/** What fx did, read from the tool result body. */
|
||||
export function interpret(body: string, sid: string | null): Pick<ObservedCall, "executed" | "decoded" | "outcome"> {
|
||||
let value: any;
|
||||
try { value = JSON.parse(body); } catch { return { executed: null, decoded: "unknown", outcome: "unknown" }; }
|
||||
const error = value?.error;
|
||||
if (error?.code === "invalid_shell_request" && error.executed === false) {
|
||||
return { executed: false, decoded: "none", outcome: "rejected" };
|
||||
}
|
||||
if (error?.type === "tool_execution_failed" && /did not run/.test(String(error.message))) {
|
||||
return { executed: false, decoded: "none", outcome: "rejected" };
|
||||
}
|
||||
if (!value || typeof value.state !== "string") return { executed: null, decoded: "unknown", outcome: "unknown" };
|
||||
if (sid && value.session_id === sid) {
|
||||
const decoded = value.state === "stopped" ? "stop" : (value.accepted_bytes ?? 0) > 0 ? "type" : "observe";
|
||||
return { executed: true, decoded, outcome: "ok" };
|
||||
}
|
||||
const output = String(value.output_delta ?? "").replaceAll("\r", "");
|
||||
const lines = output.split("\n").filter(Boolean);
|
||||
if (lines[0] === "before") {
|
||||
const same = lines.length === EQ_BASH_LINES.length && lines.every((line, i) => line === EQ_BASH_LINES[i]);
|
||||
return { executed: true, decoded: "run_eq", outcome: same ? "bash_equivalent" : "zsh_divergent" };
|
||||
}
|
||||
const shell = /FXT:(bash|zsh)\b/.exec(output)?.[1];
|
||||
if (!shell) return { executed: true, decoded: "unknown", outcome: "ok" };
|
||||
// Inside a fx TTY run the command's stdout is not a terminal, so read the backend.
|
||||
const tty = value.backend === "tty";
|
||||
const decoded = shell === "bash" ? (tty ? "run_bash_tty" : "run_bash") : (tty ? "run_tty" : "run");
|
||||
return { executed: true, decoded, outcome: "ok" };
|
||||
}
|
||||
|
||||
function sessionResults(home: string): Map<string, string> {
|
||||
const results = new Map<string, string>();
|
||||
const root = join(home, ".fx", "sessions");
|
||||
for (const id of existsSync(root) ? readdirSync(root) : []) {
|
||||
const events = join(root, id, "events.jsonl");
|
||||
if (!existsSync(events)) continue;
|
||||
for (const line of readFileSync(events, "utf8").split("\n").filter(Boolean)) {
|
||||
const result = JSON.parse(line).event?.tool_result;
|
||||
if (!result?.call_id) continue;
|
||||
const artifact = result.artifact_ref ? join(root, id, "tool-results", result.artifact_ref) : "";
|
||||
results.set(result.call_id, artifact && existsSync(artifact) ? readFileSync(artifact, "utf8") : String(result.preview ?? ""));
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
const LOOP_STOP = "Repeated shell validation failures stopped the tool loop";
|
||||
|
||||
export async function runTurn(ctx: RunContext, spec: TurnSpec): Promise<ObservedTurn> {
|
||||
const { helpers } = ctx;
|
||||
const slug = spec.name.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
|
||||
const root = mkdtempSync(join(ctx.artifacts, `${slug}-`));
|
||||
const home = join(root, "home"), cwd = join(root, "workspace");
|
||||
mkdirSync(join(home, ".fx"), { recursive: true, mode: 0o700 });
|
||||
mkdirSync(cwd);
|
||||
const needsSession = spec.calls.some((c) => SESSION_INTENTS.has(c.intent) || c.shape.startsWith("both_"));
|
||||
const offset = needsSession ? 1 : 0;
|
||||
const requests: string[] = [];
|
||||
const sent: string[] = [];
|
||||
let sid = "";
|
||||
const toolCall = (id: string, input: string) => helpers.fakeGatewaySse([
|
||||
{ type: "tool-call", toolCallId: id, toolName: "shell", input },
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool-calls" } },
|
||||
]);
|
||||
const gateway = helpers.startDynamicFakeGateway((body: string) => {
|
||||
requests.push(body);
|
||||
const index = requests.length - 1;
|
||||
if (needsSession && index === 0) {
|
||||
return toolCall("setup", JSON.stringify({ action: "run", command: "cat", tty: true, yield_time_ms: 500 }));
|
||||
}
|
||||
const j = index - offset;
|
||||
if (needsSession && j === 0) {
|
||||
const setup = parts(body).find((p) => p.type === "tool-result" && p.toolCallId === "setup");
|
||||
sid = /"session_id"\s*:\s*"([^"]+)"/.exec(textOf(setup?.output))?.[1] ?? "fxt-missing-session";
|
||||
}
|
||||
if (j < spec.calls.length) {
|
||||
sent[j] = encode(spec.calls[j]!, sid);
|
||||
return toolCall(`c${j + 1}`, sent[j]!);
|
||||
}
|
||||
return helpers.fakeGatewayFinalText("FIXTURE_DONE");
|
||||
});
|
||||
const env = {
|
||||
PATH: "/usr/bin:/bin:/usr/sbin:/sbin", HOME: home, TMPDIR: root, SHELL: "/bin/zsh", LANG: "en_US.UTF-8",
|
||||
AI_GATEWAY_API_KEY: "synthetic-shell-contract", FX_DISABLE_KEYCHAIN: "1", FX_E2E_DISABLE_DOTENV: "1",
|
||||
FX_SKIP_ONBOARDING: "1", FX_SOUND: "0", FX_AUTO_UPGRADE: "0", FX_MODEL: helpers.FAKE_GATEWAY_MODEL,
|
||||
FX_GATEWAY_BASE_URL: gateway.baseUrl, FX_GATEWAY_CHAT_URL: gateway.chatUrl,
|
||||
FX_E2E_GATEWAY_CHAT_URL: gateway.chatUrl, FX_E2E_GATEWAY_MODELS_URL: `${gateway.baseUrl}/coding-agent/v1/models`,
|
||||
FX_TRACE_LOG: join(root, "trace.log"), FX_TRACE_SCOPES: "tool,terminal",
|
||||
};
|
||||
const child = Bun.spawn([ctx.binary, "ask", "--json", "--full-access", "fixture"], {
|
||||
cwd, env, stdin: "ignore", stdout: "pipe", stderr: "pipe",
|
||||
});
|
||||
let expired = false;
|
||||
const timer = setTimeout(() => { expired = true; child.kill("SIGKILL"); }, 90_000);
|
||||
const [exitCode, stdout, stderr] = await Promise.all([
|
||||
child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
|
||||
]);
|
||||
clearTimeout(timer);
|
||||
gateway.stop();
|
||||
|
||||
const stored = sessionResults(home);
|
||||
const calls: ObservedCall[] = sent.map((text, j) => {
|
||||
const id = `c${j + 1}`;
|
||||
const next = requests[offset + j + 1];
|
||||
const fromRequest = next ? parts(next).find((p) => p.type === "tool-result" && p.toolCallId === id) : undefined;
|
||||
const resultBody = fromRequest ? textOf(fromRequest.output) : stored.get(id) ?? "";
|
||||
const replayed = next ? parts(next).find((p) => p.type === "tool-call" && p.toolCallId === id) : undefined;
|
||||
return {
|
||||
id, intent: spec.calls[j]!.intent, shape: spec.calls[j]!.shape, sent: text,
|
||||
...interpret(resultBody, needsSession ? sid : null),
|
||||
history: next ? { observed: true, input: replayed ? replayed.input : "fxt-missing-from-history" } : { observed: false },
|
||||
resultBody,
|
||||
} as ObservedCall;
|
||||
});
|
||||
// final: fx asked again after the last served call and got the final text.
|
||||
// guard: fx stopped after the last served call and reported its loop guard.
|
||||
const askedAfterLast = requests.length > offset + sent.length;
|
||||
// fx ask --json reports the stop as a stderr notice.
|
||||
const guardReported = stderr.includes(LOOP_STOP) || stdout.includes(LOOP_STOP) ||
|
||||
existsSync(join(home, ".fx", "sessions")) && readdirSync(join(home, ".fx", "sessions")).some((id) => {
|
||||
const events = join(home, ".fx", "sessions", id, "events.jsonl");
|
||||
return existsSync(events) && readFileSync(events, "utf8").includes(LOOP_STOP);
|
||||
});
|
||||
const ended: ObservedTurn["ended"] = expired || exitCode === null ? "unknown"
|
||||
: askedAfterLast ? "final" : guardReported ? "guard" : "unknown";
|
||||
const turn: ObservedTurn = { name: spec.name, calls, ended };
|
||||
writeFileSync(join(root, "observation.json"), JSON.stringify({
|
||||
spec, sid, exitCode, expired, requests: requests.length, ended, calls,
|
||||
stderrTail: stderr.slice(-2000), stdoutTail: stdout.slice(-2000),
|
||||
}, null, 2), { mode: 0o600 });
|
||||
if (ended === "unknown" || calls.some((c) => c.executed === null || c.decoded === "unknown")) {
|
||||
writeFileSync(join(root, "requests.jsonl"), requests.join("\n") + "\n", { mode: 0o600 });
|
||||
}
|
||||
return turn;
|
||||
}
|
||||
|
||||
/** One turn per modeled emission, plus multi-call turns for the loop guard. */
|
||||
export function turnSpecs(emits: string[]): TurnSpec[] {
|
||||
const single = emits.map((key) => {
|
||||
const [intent, shape] = key.split("/") as [string, string];
|
||||
return { name: `${intent} ${shape}`, calls: [{ intent, shape }] };
|
||||
});
|
||||
return [
|
||||
...single,
|
||||
// These shapes are rejected in both designs, so the guard is exercised either way.
|
||||
{ name: "guard same rejected call repeated", calls: Array(3).fill({ intent: "run", shape: "wrapped_broken" }) },
|
||||
{ name: "no guard after accepted repeats", calls: Array(2).fill({ intent: "run", shape: "flat_action" }) },
|
||||
{ name: "no guard for different rejected calls",
|
||||
calls: [{ intent: "run", shape: "wrapped_broken" }, { intent: "observe", shape: "both_bare" }] },
|
||||
];
|
||||
}
|
||||
|
||||
export async function collectObservations(ctx: RunContext, specs: TurnSpec[]): Promise<ObservedTurn[]> {
|
||||
mkdirSync(ctx.artifacts, { recursive: true, mode: 0o700 });
|
||||
const results: ObservedTurn[] = new Array(specs.length);
|
||||
let next = 0;
|
||||
const worker = async () => {
|
||||
while (next < specs.length) {
|
||||
const index = next++;
|
||||
results[index] = await runTurn(ctx, specs[index]!);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(ctx.concurrency ?? 6, specs.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,413 @@
|
||||
// Replays LeanUsageDrive behaviors from TLC against the real fx binary.
|
||||
//
|
||||
// bun tests/tla/drivers/usage/lean/drive.ts [--traces N] [--depth D] [--seed S] [file...]
|
||||
//
|
||||
// TLC simulates N random behaviors of tests/tla/usage/LeanUsageDrive.tla (or the given
|
||||
// trace files are replayed). Each behavior runs two fx TUIs (p1, p2) in tmux
|
||||
// that share one HOME, so one ~/.fx/usage-cache. Identity a and b are two
|
||||
// AI Gateway API keys; each process has its own fake gateway that holds every
|
||||
// /v1/report request until the behavior says the fetch completes or fails.
|
||||
// The usage clock seam (FX_E2E_USAGE_CLOCK_FILE) maps model time to real time:
|
||||
// tick n is 23:40 UTC + 5n minutes, so the 60 s floor is one tick, the
|
||||
// 15-minute timer three, and UTC midnight falls between ticks 3 and 4.
|
||||
//
|
||||
// After every step the driver checks the real system against the next model
|
||||
// state: which processes started a fetch and whether it read the finished
|
||||
// days, what each identity's cache file holds, and what "today" each open
|
||||
// dashboard shows. The first difference ends that behavior.
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync,
|
||||
realpathSync, rmSync, writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { REPO_ROOT, TLA_ROOT, outDir as newOutDir, tlcToolchain } from "../../lib/tlc";
|
||||
import { parseBehavior, type State, type Value } from "./tla.ts";
|
||||
|
||||
const WORKTREE = process.env.FX_WORKTREE ?? REPO_ROOT;
|
||||
const { startFakeGateway, TmuxSession } = await import(join(WORKTREE, "tests/e2e/tmux-helpers.ts"));
|
||||
const { usageEnv, utcDate, writeUsageClock } = await import(join(WORKTREE, "tests/e2e/usage-fakes.ts"));
|
||||
|
||||
const SPECS = join(TLA_ROOT, "usage");
|
||||
const PROCS = ["p1", "p2"] as const;
|
||||
type Proc = (typeof PROCS)[number];
|
||||
const MIN = 60_000;
|
||||
const DAY_MS = 24 * 60 * MIN;
|
||||
/** UTC midnight between ticks 3 and 4: Wed Oct 21 2026. */
|
||||
const MIDNIGHT = Date.UTC(2026, 9, 21);
|
||||
const tickMs = (n: number) => MIDNIGHT - 20 * MIN + n * 5 * MIN;
|
||||
const utcDay = (ms: number) => Math.floor(ms / DAY_MS);
|
||||
const DAY0 = utcDay(tickMs(0));
|
||||
const KEYS: Record<string, string> = { a: "drive-key-a", b: "drive-key-b" };
|
||||
/** Today's spend for identity i fetched at tick k is BASE[i] + k dollars. */
|
||||
const BASE: Record<string, number> = { a: 10, b: 20 };
|
||||
const SETTLE_MS = 300;
|
||||
const TIMEOUT_MS = 8_000;
|
||||
|
||||
type Fetch = { ident: string; at: number; fin: boolean };
|
||||
type Held = { url: URL; answer: (response: Response) => void };
|
||||
type Action =
|
||||
| { kind: "Tick" }
|
||||
| { kind: "Open" | "Close" | "Manual" | "Timer" | "Complete" | "Fail"; p: Proc };
|
||||
|
||||
class Divergence extends Error {
|
||||
constructor(message: string, readonly detail: Record<string, unknown> = {}) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms));
|
||||
async function until<T>(probe: () => T | undefined | false | Promise<T | undefined | false>, ms = TIMEOUT_MS): Promise<T | undefined> {
|
||||
const deadline = Date.now() + ms;
|
||||
for (;;) {
|
||||
const value = await probe();
|
||||
if (value) return value as T;
|
||||
if (Date.now() > deadline) return undefined;
|
||||
await sleep(25);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The model side
|
||||
|
||||
const rec = (v: Value) => v as Record<string, Value>;
|
||||
const fetchOf = (s: State, p: Proc): Fetch | null => {
|
||||
const f = rec(s.inflight)[p];
|
||||
return f === null ? null : (f as unknown as Fetch);
|
||||
};
|
||||
const since = (t: Value, n: number, now: number) => t === null || now - (t as number) >= n;
|
||||
|
||||
function timerDue(s: State, p: Proc): boolean {
|
||||
const now = s.now as number;
|
||||
const lastStart = rec(s.lastStart)[p];
|
||||
return rec(s.open)[p] === true && fetchOf(s, p) === null &&
|
||||
since(lastStart, 1, now) && since(rec(s.failedAt)[p], 1, now) &&
|
||||
lastStart !== null && since(lastStart, 3, now);
|
||||
}
|
||||
|
||||
/** The action that takes `s` to `t` (LeanUsageDrive's actions leave distinct marks). */
|
||||
function actionBetween(s: State, t: State): Action {
|
||||
if (t.now !== s.now) return { kind: "Tick" };
|
||||
for (const p of PROCS) {
|
||||
const wasOpen = rec(s.open)[p] === true;
|
||||
const isOpen = rec(t.open)[p] === true;
|
||||
if (!wasOpen && isOpen) return { kind: "Open", p };
|
||||
if (wasOpen && !isOpen) return { kind: "Close", p };
|
||||
const before = fetchOf(s, p);
|
||||
const after = fetchOf(t, p);
|
||||
if (before === null && after !== null) return { kind: timerDue(s, p) ? "Timer" : "Manual", p };
|
||||
if (before !== null && after === null) {
|
||||
return { kind: rec(t.failedAt)[p] === t.now ? "Fail" : "Complete", p };
|
||||
}
|
||||
}
|
||||
throw new Error("no LeanUsageDrive action explains this step");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The real side
|
||||
|
||||
function cachePath(home: string, ident: string): string {
|
||||
const hash = createHash("sha256").update(`fx-usage-cache-v1\0api_key\0${KEYS[ident]}\0`).digest("hex");
|
||||
return join(home, ".fx", "usage-cache", `${hash}.json`);
|
||||
}
|
||||
|
||||
type CacheRec = { todayDay: number | null; todayAt: number | null; finDay: number | null };
|
||||
|
||||
function readCache(home: string, ident: string): CacheRec {
|
||||
const path = cachePath(home, ident);
|
||||
if (!existsSync(path)) return { todayDay: null, todayAt: null, finDay: null };
|
||||
const wire = JSON.parse(readFileSync(path, "utf8"));
|
||||
const tick = (ms: number) => (ms - tickMs(0)) / (5 * MIN);
|
||||
return {
|
||||
todayDay: wire.today ? wire.today.day - DAY0 : null,
|
||||
todayAt: wire.today ? tick(wire.today.at_ms) : null,
|
||||
finDay: wire.finished ? wire.finished.day - DAY0 : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** The "today" spend on an open dashboard, decoded to identity and tick, or null. */
|
||||
function shownToday(pane: string): { ident: string; today: number } | null {
|
||||
const match = pane.match(/^\s*(?:❯\s*)?today\s+\$([\d,]+(?:\.\d+)?)\s/m);
|
||||
if (!match) return null;
|
||||
const dollars = Number(match[1].replace(/,/g, ""));
|
||||
for (const [ident, base] of Object.entries(BASE)) {
|
||||
const tick = dollars - base;
|
||||
if (Number.isInteger(tick) && tick >= 0 && tick < 10) return { ident, today: tick };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const rows = (cost: number) => ({
|
||||
results: [{ model: "drive/usage", request_count: 1, input_tokens: 1, output_tokens: 1, total_cost: cost }],
|
||||
});
|
||||
|
||||
class Process {
|
||||
held: Held[] = [];
|
||||
fetch: (Fetch & { today: Held }) | null = null;
|
||||
session: any = null;
|
||||
gateway: any;
|
||||
|
||||
constructor(readonly name: Proc, readonly ident: string) {
|
||||
this.gateway = startFakeGateway([], {
|
||||
usage: (url: URL) =>
|
||||
url.pathname !== "/v1/report"
|
||||
? null
|
||||
: new Promise<Response>((answer) => this.held.push({ url, answer })),
|
||||
});
|
||||
}
|
||||
|
||||
async start(home: string, clock: string): Promise<void> {
|
||||
this.session = await TmuxSession.create({
|
||||
width: 120,
|
||||
height: 40,
|
||||
env: {
|
||||
...usageEnv(this.gateway.baseUrl, clock),
|
||||
HOME: home,
|
||||
AI_GATEWAY_API_KEY: KEYS[this.ident],
|
||||
VERCEL_OIDC_TOKEN: undefined,
|
||||
FX_E2E_GATEWAY_CHAT_URL: this.gateway.chatUrl,
|
||||
FX_E2E_GATEWAY_MODELS_URL: `${this.gateway.baseUrl}/coding-agent/v1/models`,
|
||||
FX_AUTO_UPGRADE: "0",
|
||||
FX_SOUND: "0",
|
||||
FX_DEBUG_RECORD: undefined,
|
||||
FX_DEBUG_RECORD_SILENT_BANNER: undefined,
|
||||
},
|
||||
});
|
||||
await this.session.waitForComposer(TIMEOUT_MS * 2);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
for (const h of this.held) h.answer(new Response("driver stopped", { status: 503 }));
|
||||
await this.session?.kill();
|
||||
this.gateway.stop();
|
||||
}
|
||||
|
||||
/** Claims the first request of a fetch the model started. */
|
||||
async claimStart(f: Fetch): Promise<void> {
|
||||
const got = await until(() => this.held.length > 0);
|
||||
if (!got) throw new Divergence(`${this.name}: the model starts a fetch, fx sent no request`, { fetch: f });
|
||||
const today = this.held.shift()!;
|
||||
const day = utcDate(tickMs(f.at));
|
||||
const range = `${today.url.searchParams.get("start_date")}..${today.url.searchParams.get("end_date")}`;
|
||||
if (range !== `${day}..${day}`) {
|
||||
throw new Divergence(`${this.name}: first request reads ${range}, model expects today ${day}`, { fetch: f });
|
||||
}
|
||||
this.fetch = { ...f, today };
|
||||
}
|
||||
|
||||
async answerNext(label: string, start: string, end: string, cost: number): Promise<void> {
|
||||
const got = await until(() => this.held.length > 0);
|
||||
if (!got) throw new Divergence(`${this.name}: fetch reads finished days in the model, fx sent no ${label} request`);
|
||||
const h = this.held.shift()!;
|
||||
const range = `${h.url.searchParams.get("start_date")}..${h.url.searchParams.get("end_date")}`;
|
||||
if (range !== `${start}..${end}`) {
|
||||
throw new Divergence(`${this.name}: ${label} request reads ${range}, model expects ${start}..${end}`);
|
||||
}
|
||||
h.answer(Response.json(rows(cost)));
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// One behavior
|
||||
|
||||
async function replay(states: State[]): Promise<{ steps: number }> {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "fx-usage-drive-")));
|
||||
const home = join(root, "home");
|
||||
mkdirSync(home);
|
||||
const clock = join(root, "clock");
|
||||
writeUsageClock(clock, tickMs(0));
|
||||
const ident = rec(states[0].ident);
|
||||
const procs = Object.fromEntries(PROCS.map((p) => [p, new Process(p, ident[p] as string)])) as Record<Proc, Process>;
|
||||
let step = 0;
|
||||
try {
|
||||
await Promise.all(PROCS.map((p) => procs[p].start(home, clock)));
|
||||
for (step = 1; step < states.length; step += 1) {
|
||||
const s = states[step - 1];
|
||||
const t = states[step];
|
||||
const action = actionBetween(s, t);
|
||||
try {
|
||||
await perform(action, s, t, procs, clock, home);
|
||||
await settleAndCheck(t, procs, home);
|
||||
} catch (err) {
|
||||
if (err instanceof Divergence) {
|
||||
err.detail.step = step;
|
||||
err.detail.action = action;
|
||||
err.detail.model = { now: t.now, open: t.open, shown: t.shown, cache: t.cache, inflight: t.inflight };
|
||||
err.detail.real = Object.fromEntries(Object.keys(KEYS).map((i) => [i, readCache(home, i)]));
|
||||
err.detail.panes = Object.fromEntries(
|
||||
await Promise.all(PROCS.map(async (p) => [p, await procs[p].session.capturePane()])),
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return { steps: states.length - 1 };
|
||||
} finally {
|
||||
await Promise.all(PROCS.map((p) => procs[p].stop()));
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function perform(action: Action, s: State, t: State, procs: Record<Proc, Process>, clock: string, home: string): Promise<void> {
|
||||
if (action.kind === "Tick") {
|
||||
writeUsageClock(clock, tickMs(t.now as number));
|
||||
return;
|
||||
}
|
||||
const proc = procs[action.p];
|
||||
const session = proc.session;
|
||||
switch (action.kind) {
|
||||
case "Open": {
|
||||
await session.sendText("/usage");
|
||||
const shown = await until(async () => (await session.capturePane()).includes("usage: API key"));
|
||||
if (!shown) throw new Divergence(`${action.p}: /usage did not open`);
|
||||
// Reopening while a fetch is in flight starts nothing new.
|
||||
const f = fetchOf(s, action.p) === null ? fetchOf(t, action.p) : null;
|
||||
if (f) await proc.claimStart(f);
|
||||
return;
|
||||
}
|
||||
case "Close": {
|
||||
await session.sendKeys("Escape");
|
||||
const closed = await until(async () => !(await session.capturePane()).includes("usage: API key"));
|
||||
if (!closed) throw new Divergence(`${action.p}: /usage did not close`);
|
||||
return;
|
||||
}
|
||||
case "Manual":
|
||||
await session.sendKeys("r");
|
||||
await proc.claimStart(fetchOf(t, action.p)!);
|
||||
return;
|
||||
case "Timer":
|
||||
// fx starts it on its own once the clock moved; the request may already be held.
|
||||
await proc.claimStart(fetchOf(t, action.p)!);
|
||||
return;
|
||||
case "Complete": {
|
||||
const f = proc.fetch;
|
||||
if (!f) throw new Error("Complete without a claimed fetch");
|
||||
proc.fetch = null;
|
||||
f.today.answer(Response.json(rows(BASE[f.ident] + f.at)));
|
||||
if (f.fin) {
|
||||
const ms = tickMs(f.at);
|
||||
const today = new Date(ms);
|
||||
await proc.answerNext("7d", utcDate(ms - 6 * DAY_MS), utcDate(ms - DAY_MS), 1000);
|
||||
const first = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), 1);
|
||||
if (utcDay(first) !== utcDay(ms)) await proc.answerNext("month", utcDate(first), utcDate(ms - DAY_MS), 2000);
|
||||
}
|
||||
// The worker merges into the cache file and then frees the process for
|
||||
// its next fetch; wait for the merge the model predicts.
|
||||
const want = rec(rec(t.cache)[f.ident]) as unknown as CacheRec;
|
||||
await until(() => sameCache(readCache(home, f.ident), want));
|
||||
return;
|
||||
}
|
||||
case "Fail": {
|
||||
const f = proc.fetch;
|
||||
if (!f) throw new Error("Fail without a claimed fetch");
|
||||
proc.fetch = null;
|
||||
f.today.answer(new Response("drive: failed on purpose", { status: 500 }));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sameCache = (real: CacheRec, want: CacheRec) =>
|
||||
real.todayDay === want.todayDay && real.todayAt === want.todayAt && real.finDay === want.finDay;
|
||||
|
||||
async function settleAndCheck(t: State, procs: Record<Proc, Process>, home: string): Promise<void> {
|
||||
await sleep(SETTLE_MS);
|
||||
// No fetch the model does not start. fx fires every due timer at once,
|
||||
// while the model takes them one step at a time: a process whose timer is
|
||||
// due may already have sent its first request.
|
||||
for (const p of PROCS) {
|
||||
const allowed = timerDue(t, p) ? 1 : 0;
|
||||
if (procs[p].held.length > allowed) {
|
||||
const ranges = procs[p].held.map((h) => `${h.url.searchParams.get("start_date")}..${h.url.searchParams.get("end_date")}`);
|
||||
throw new Divergence(`${p}: fx sent a request the model does not make`, { ranges });
|
||||
}
|
||||
}
|
||||
// The shared cache file, per identity.
|
||||
for (const ident of Object.keys(KEYS)) {
|
||||
const want = rec(rec(t.cache)[ident]) as unknown as CacheRec;
|
||||
const real = readCache(home, ident);
|
||||
if (!sameCache(real, want)) {
|
||||
throw new Divergence(`cache for ${ident} differs from the model`, { want, real });
|
||||
}
|
||||
}
|
||||
// What each open dashboard shows as today (ShowsOwnIdentity, TodayIsToday).
|
||||
for (const p of PROCS) {
|
||||
if (rec(t.open)[p] !== true) continue;
|
||||
const shown = rec(rec(t.shown)[p]);
|
||||
const want = shown.today === null ? null : { ident: shown.ident as string, today: shown.today as number };
|
||||
let last: ReturnType<typeof shownToday> = null;
|
||||
const ok = await until(async () => {
|
||||
last = shownToday(await procs[p].session.capturePane());
|
||||
return JSON.stringify(last) === JSON.stringify(want);
|
||||
}, 3_000);
|
||||
if (!ok) throw new Divergence(`${p}: dashboard shows today ${JSON.stringify(last)}, model shows ${JSON.stringify(want)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
|
||||
function args() {
|
||||
const out = { traces: 20, depth: 30, seed: Date.now() % 100_000, files: [] as string[] };
|
||||
const argv = process.argv.slice(2);
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const flag = argv[i];
|
||||
if (flag === "--traces") out.traces = Number(argv[++i]);
|
||||
else if (flag === "--depth") out.depth = Number(argv[++i]);
|
||||
else if (flag === "--seed") out.seed = Number(argv[++i]);
|
||||
else out.files.push(resolve(flag));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function simulate(dir: string, traces: number, depth: number, seed: number): string[] {
|
||||
const { java, jar } = tlcToolchain();
|
||||
const run = Bun.spawnSync([
|
||||
java, "-XX:+UseParallelGC", `-Djava.io.tmpdir=${mkdtempSync(join(tmpdir(), "fx-tlc-"))}`, "-cp", jar, "tlc2.TLC",
|
||||
"-metadir", join(dir, ".tlc-states"),
|
||||
"-config", join(SPECS, "LeanUsageDrive.cfg"), join(SPECS, "LeanUsageDrive.tla"),
|
||||
"-simulate", `file=${join(dir, "t")},num=${traces}`, "-depth", String(depth), "-seed", String(seed),
|
||||
], { stdout: "pipe", stderr: "pipe" });
|
||||
if (run.exitCode !== 0) {
|
||||
throw new Error(`TLC simulation failed (${run.exitCode}):\n${run.stdout.toString()}\n${run.stderr.toString()}`);
|
||||
}
|
||||
return readdirSync(dir).filter((name) => name.startsWith("t_")).sort().map((name) => join(dir, name));
|
||||
}
|
||||
|
||||
const opts = args();
|
||||
const outDir = newOutDir("usage-lean");
|
||||
mkdirSync(join(outDir, "traces"), { recursive: true });
|
||||
const files = opts.files.length > 0 ? opts.files : simulate(join(outDir, "traces"), opts.traces, opts.depth, opts.seed);
|
||||
console.log(`replaying ${files.length} behavior(s)${opts.files.length ? "" : ` (TLC seed ${opts.seed}, depth ${opts.depth})`}`);
|
||||
|
||||
const results: Array<Record<string, unknown>> = [];
|
||||
for (const file of files) {
|
||||
const states = parseBehavior(readFileSync(file, "utf8"));
|
||||
const actions = states.slice(1).map((t, i) => actionBetween(states[i], t));
|
||||
const name = file.split("/").pop()!;
|
||||
if (!actions.some((a) => a.kind === "Open")) {
|
||||
results.push({ trace: name, status: "skipped", reason: "never opens /usage" });
|
||||
continue;
|
||||
}
|
||||
const summary = actions.map((a) => (a.kind === "Tick" ? "Tick" : `${a.kind}(${a.p})`)).join(" ");
|
||||
const started = Date.now();
|
||||
try {
|
||||
const run = await replay(states);
|
||||
results.push({ trace: name, status: "pass", steps: run.steps, ms: Date.now() - started, actions: summary });
|
||||
console.log(`pass ${name} ${run.steps} steps ${summary}`);
|
||||
} catch (err) {
|
||||
const detail = err instanceof Divergence ? err.detail : { stack: String((err as Error).stack) };
|
||||
results.push({ trace: name, status: err instanceof Divergence ? "diverged" : "error", message: (err as Error).message, actions: summary, ...detail });
|
||||
copyFileSync(file, join(outDir, name));
|
||||
console.log(`${err instanceof Divergence ? "DIVERGED" : "ERROR"} ${name}: ${(err as Error).message}`);
|
||||
console.log(` ${summary}`);
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(join(outDir, "report.json"), JSON.stringify({ seed: opts.seed, depth: opts.depth, results }, null, 2) + "\n");
|
||||
const count = (status: string) => results.filter((r) => r.status === status).length;
|
||||
console.log(`\n${count("pass")} pass, ${count("diverged")} diverged, ${count("error")} error, ${count("skipped")} skipped`);
|
||||
console.log(`report: ${join(outDir, "report.json")}`);
|
||||
process.exit(count("diverged") + count("error") > 0 ? 1 : 0);
|
||||
@@ -0,0 +1,94 @@
|
||||
// Reads the behaviors TLC writes with `-simulate file=...`: each file is a
|
||||
// TLA+ module of `STATE_n == /\ var = value ...` definitions. Values are the
|
||||
// few forms LeanUsage uses: numbers, TRUE/FALSE, model values (None reads as
|
||||
// null), records `[k |-> v]` and functions `(k :> v @@ ...)`.
|
||||
|
||||
export type Value = number | boolean | string | null | { [key: string]: Value };
|
||||
export type State = Record<string, Value>;
|
||||
|
||||
const TOKEN = /\s*(\|->|:>|@@|\(|\)|\[|\]|,|-?\d+|[A-Za-z_][A-Za-z0-9_]*)/y;
|
||||
|
||||
function tokenize(text: string): string[] {
|
||||
const tokens: string[] = [];
|
||||
TOKEN.lastIndex = 0;
|
||||
while (TOKEN.lastIndex < text.length) {
|
||||
if (/^\s*$/.test(text.slice(TOKEN.lastIndex))) break;
|
||||
const start = TOKEN.lastIndex;
|
||||
const match = TOKEN.exec(text);
|
||||
if (!match) throw new Error(`unexpected TLA+ text at ${start}: ${text.slice(start, start + 30)}`);
|
||||
tokens.push(match[1]);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function parseValue(tokens: string[]): Value {
|
||||
let at = 0;
|
||||
const next = () => {
|
||||
const token = tokens[at++];
|
||||
if (token === undefined) throw new Error("TLA+ value ended early");
|
||||
return token;
|
||||
};
|
||||
const expect = (want: string) => {
|
||||
const got = next();
|
||||
if (got !== want) throw new Error(`expected ${want}, found ${got}`);
|
||||
};
|
||||
const atom = (token: string): Value => {
|
||||
if (/^-?\d+$/.test(token)) return Number(token);
|
||||
if (token === "TRUE") return true;
|
||||
if (token === "FALSE") return false;
|
||||
if (token === "None") return null;
|
||||
if (/^[A-Za-z_]/.test(token)) return token;
|
||||
throw new Error(`unexpected token ${token}`);
|
||||
};
|
||||
const value = (): Value => {
|
||||
const token = next();
|
||||
if (token === "[") {
|
||||
const record: { [key: string]: Value } = {};
|
||||
for (;;) {
|
||||
const key = next();
|
||||
expect("|->");
|
||||
record[key] = value();
|
||||
const sep = next();
|
||||
if (sep === "]") return record;
|
||||
if (sep !== ",") throw new Error(`expected , or ], found ${sep}`);
|
||||
}
|
||||
}
|
||||
if (token === "(") {
|
||||
// `(k :> v @@ ...)` is a function; anything else is a parenthesized value.
|
||||
if (tokens[at + 1] !== ":>") {
|
||||
const inner = value();
|
||||
expect(")");
|
||||
return inner;
|
||||
}
|
||||
const fn: { [key: string]: Value } = {};
|
||||
for (;;) {
|
||||
const key = String(atom(next()));
|
||||
expect(":>");
|
||||
fn[key] = value();
|
||||
const sep = next();
|
||||
if (sep === ")") return fn;
|
||||
if (sep !== "@@") throw new Error(`expected @@ or ), found ${sep}`);
|
||||
}
|
||||
}
|
||||
return atom(token);
|
||||
};
|
||||
const result = value();
|
||||
if (at !== tokens.length) throw new Error(`trailing TLA+ tokens: ${tokens.slice(at).join(" ")}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** The states of one TLC behavior file, in order. */
|
||||
export function parseBehavior(text: string): State[] {
|
||||
const chunks = text.split(/^STATE_\d+ ==\s*$/m).slice(1);
|
||||
return chunks.map((chunk) => {
|
||||
const body = chunk.split(/^={4,}/m)[0];
|
||||
const state: State = {};
|
||||
const assignments = [...body.matchAll(/^\/\\ (\w+) = /gm)];
|
||||
assignments.forEach((match, index) => {
|
||||
const start = match.index! + match[0].length;
|
||||
const end = index + 1 < assignments.length ? assignments[index + 1].index! : body.length;
|
||||
state[match[1]] = parseValue(tokenize(body.slice(start, end)));
|
||||
});
|
||||
return state;
|
||||
});
|
||||
}
|
||||
Loaded 100 of 348 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user