diff --git a/build.zig b/build.zig index 7c23999ac..49f421020 100644 --- a/build.zig +++ b/build.zig @@ -95,12 +95,19 @@ pub fn build(b: *std.Build) void { const session_manager_tests = b.addTest(.{ .root_module = session_manager }); const run_session_manager_tests = b.addRunArtifact(session_manager_tests); + // And with its fault and trace hooks, whose tests fx's build leaves out. + const session_manager_hook_tests = b.addTest(.{ + .root_module = sessionManagerModule(b, target, optimize, null, true), + }); + const run_session_manager_hook_tests = b.addRunArtifact(session_manager_hook_tests); const session_manager_test_step = b.step("test-session-manager", "Run the session manager's tests"); session_manager_test_step.dependOn(&run_session_manager_tests.step); + session_manager_test_step.dependOn(&run_session_manager_hook_tests.step); const test_step = b.step("test", "Run tests"); test_step.dependOn(&run_exe_tests.step); test_step.dependOn(&run_session_manager_tests.step); + test_step.dependOn(&run_session_manager_hook_tests.step); if (wasm_surface != .none) { addWasmArtifact(b, wasm_surface, git_commit, app_version, update_channel); @@ -231,6 +238,7 @@ pub fn build(b: *std.Build) void { const run_approval_review_bench_tests = b.addRunArtifact( approval_review_bench_tests, ); + test_step.dependOn(&run_approval_review_bench_tests.step); const test_approval_review_bench_step = b.step( "test-approval-review-benchmark", "Run file-diff approval review benchmark qualification tests", @@ -417,9 +425,22 @@ fn addSessionManager( target: std.Build.ResolvedTarget, optimize: std.lang.Optimize, single_threaded: ?bool, +) *std.Build.Module { + const module = sessionManagerModule(b, target, optimize, single_threaded, false); + importer.addImport("session_manager", module); + return module; +} + +/// `hooks` compiles in the fault and trace hooks, which only tests use. +fn sessionManagerModule( + b: *std.Build, + target: std.Build.ResolvedTarget, + optimize: std.lang.Optimize, + single_threaded: ?bool, + hooks: bool, ) *std.Build.Module { const options = b.addOptions(); - options.addOption(bool, "hooks", false); + options.addOption(bool, "hooks", hooks); options.addOptionPathUntracked("trace_dir", b.graph.path(.install_prefix, "session-manager-traces")); options.addOptionPathUntracked("src_dir", b.path("src/core/session_manager")); const module = b.createModule(.{ @@ -430,7 +451,6 @@ fn addSessionManager( .single_threaded = single_threaded, }); module.addImport("build_options", options.createModule()); - importer.addImport("session_manager", module); return module; } diff --git a/sdk/README.md b/sdk/README.md index 76ae42e1e..1cfe16785 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -327,6 +327,8 @@ const agent = createFxAgent({ }); ``` +Name a model the plan serves, here or in `agent.session(id, { model })`. fx's default model is an AI Gateway id, so libfx refuses a Codex engine or session without one. + libfx calls `credentials()` in whichever process runs the turn. It refreshes the credentials shortly before they expire and once after a rejected request, then passes the new ones to `onCredentials` for you to save. Codex doesn't accept requests from web pages, so these turns run on your server. To sign a user in, use `libfx/chatgpt`: @@ -347,7 +349,7 @@ The sign-in uses the same OpenAI client as the Codex CLI, and the user's ChatGPT ### Grok plans -`provider: "grok"` runs on a user's Grok plan, with the same `credentials` and `onCredentials` callbacks, called with `provider: "grok"`. libfx also checks that refreshed credentials still belong to the same xAI account. These turns run on your server too. +`provider: "grok"` runs on a user's Grok plan, with the same `credentials` and `onCredentials` callbacks, called with `provider: "grok"`. It needs a Grok model too, such as `model: "grok-4.7"`; the AI Gateway id `spacexai/grok-4.7` doesn't work on a Grok plan. libfx also checks that refreshed credentials still belong to the same xAI account. These turns run on your server too. To sign a user in, use `libfx/grok`. Device code is the simplest: diff --git a/sdk/durable.js b/sdk/durable.js index fe869a64a..ca7fe858c 100644 --- a/sdk/durable.js +++ b/sdk/durable.js @@ -762,6 +762,49 @@ const streamReadRetries = 5; const missingStreamPollMs = 50; const missingStreamFastMs = 2000; +// Runs whose log views one backend keeps, and the characters of JSON they +// may hold together, least recently read dropped first. +const maxLogViews = 32; +const maxLogViewChars = 32 * 1024 * 1024; + +// Whether a newest-first listing showed every slot from the lowest it +// listed to the newest. Pure. +const gapless = (read) => read.listed === 0 || read.newest - read.lowest + 1 === read.listed; + +// The slot of a run's first event: a World numbers each run's events densely +// from 1. +const firstEventSlot = 1; + +// Whether a newest-first listing reached slot `floor`, or the checkpoint it +// stopped at, which stands for every slot through its `through`. Pure. +const listedDownTo = (read, floor) => read.lowest === floor || (read.through !== null && read.lowest <= read.through); + +/** + * A run's log view extended by the entries listed after it, as a full read + * returns the log: every entry after the newest checkpoint's `through`. + * Null when that checkpoint covers less than the view starts after, which + * only a full read can answer. `head` is the chain's head as a full read + * notes it: the newest chained entry, or the checkpoint's `through`. Pure. + */ +export function extendLogView(view, newer) { + const combined = [...view.entries, ...newer]; + let through = view.through; + for (let index = combined.length - 1; index >= 0; index -= 1) { + const entry = combined[index].entry; + if (entry?.k === "checkpoint") { + through = entry.through === null || entry.through === undefined ? 0 : Number(entry.through); + break; + } + } + if (through < view.through) return null; + const entries = combined.filter((item) => Number(item.cursor) > through); + let head = null; + for (let index = entries.length - 1; index >= 0 && head === null; index -= 1) { + if (chainedKind(entries[index].entry?.k)) head = entries[index].cursor; + } + return { entries, through, head: head ?? (through ? String(through) : "null") }; +} + // Events a session-log read lists per page: the first page, then the rest. const firstPageEvents = 10; const laterPageEvents = 100; @@ -908,6 +951,35 @@ function createWorldBackend(world, durability) { // poll. const watchers = new Map(); const changed = (runId) => { for (const watcher of watchers.get(runId) ?? []) watcher(); }; + // Each run's log as this process last read it: the entries read() returned, + // the newest slot seen, and the checkpoint's `through` they start after. + // The log only appends, so a later read lists only slots after `known`. + // The entries are kept as the JSON the World stores, so every read hands + // out its own copy, as reading the World does, and the views' size is + // bounded in characters as well as in runs. + const logViews = new Map(); + let logViewChars = 0; + const forgetView = (runId) => { + const view = logViews.get(runId); + if (!view) return; + logViewChars -= view.text.length; + logViews.delete(runId); + }; + const rememberView = (runId, entries, known, through) => { + forgetView(runId); + const text = JSON.stringify(entries); + if (text.length > maxLogViewChars) return; + logViews.set(runId, { text, known, through }); + logViewChars += text.length; + for (const oldest of logViews.keys()) { + if (logViews.size <= maxLogViews && logViewChars <= maxLogViewChars) break; + forgetView(oldest); + } + }; + const viewOf = (runId) => { + const view = logViews.get(runId); + return view && { entries: JSON.parse(view.text), known: view.known, through: view.through }; + }; let handler = null; const route = typeof world.registerHandler !== "function"; const listEvents = (runId, cursor, limit) => world.events.list({ @@ -1053,11 +1125,17 @@ function createWorldBackend(world, durability) { // The newest chained entry read: the chain's head, unless the read // stopped before reaching any. let newestChained; + // The newest and lowest slots this read listed above `stopBelow`, + // entries or not, and how many. Unlike `known`, which this object's + // own writes also raise, these say only what the listing showed. + let newest = stopBelow; + let lowest = Infinity; + let listed = 0; // Ends the read. `head` is where the chain's head is when the read // found no chained entry, or undefined when the read does not show. const done = (head) => { noteSnapshot(snapshot, newestChained ?? head); - return { events, through }; + return { events, through, snapshot, newest, lowest, listed }; }; // A session's newest checkpoint is usually among its last few // entries, so a read starts with a small page and fetches more only @@ -1071,13 +1149,16 @@ function createWorldBackend(world, durability) { if (!missingRun(error)) throw error; // The run's creation starts its count. if (stopBelow === 0) noteSnapshot({ seq: 0, seqInBand: inBandAtRunCreation }, "null"); - return { events, through }; + return { events, through, newest, lowest, listed, missing: true }; } snapshot ??= page.snapshot; for (const event of page.data) { const slot = slotOf(event); if (slot > known) known = slot; if (slot <= stopBelow) return done(undefined); + if (slot > newest) newest = slot; + if (slot < lowest) lowest = slot; + listed += 1; // The checkpoint stands for the entries through its `through`, // the last of them the chain's head. if (through !== null && slot <= through) return done(through === 0 ? "null" : String(through)); @@ -1094,10 +1175,50 @@ function createWorldBackend(world, durability) { cursor = page.cursor; } }; + // The newest slot the last read() returned every entry through, or + // null when that read could not show it had them all. + let covered = null; + const fullRead = async () => { + const read = await newestFirst(0, true); + const entries = read.events.reverse(); + // A listing that skipped a slot may show a later write before an + // earlier one lands. The entries go back as they are, as every read + // returned them before, but no view keeps the gap for later reads. + // The listing must also reach the run's first event or the checkpoint + // it stopped at, so no slot below what it listed is missing. + if (!read.missing && listedDownTo(read, firstEventSlot) && gapless(read)) { + rememberView(runId, entries, read.newest, read.through ?? 0); + covered = read.newest; + } else { + forgetView(runId); + covered = null; + } + return entries; + }; return { + newest: () => covered, async read() { - const { events } = await newestFirst(0, true); - return events.reverse(); + const view = viewOf(runId); + if (!view) return fullRead(); + if (view.known > known) known = view.known; + const since = await newestFirst(view.known, true); + if (since.missing) { + forgetView(runId); + covered = null; + return []; + } + // The newer listing must have skipped no slot and must reach the + // view or a checkpoint that stands for what lies between; and a + // checkpoint covering less than the view starts after is not one + // the view can stand for. Otherwise read the log whole. + const reaches = since.listed === 0 || listedDownTo(since, view.known + 1); + const next = reaches && gapless(since) ? extendLogView(view, since.events.reverse()) : null; + if (!next) return fullRead(); + // The in-band count at the chain's head, as a full read notes it. + noteSnapshot(since.snapshot, next.head); + rememberView(runId, next.entries, since.newest, next.through); + covered = since.newest; + return next.entries; }, async readSince(cursor) { const { events } = await newestFirst(Number(cursor), false); @@ -1459,14 +1580,33 @@ const messageIdInUse = (message) => Object.assign( { code: "FX_MESSAGE_ID_IN_USE" }, ); -// Writes what a message adds to its session, in order, and returns how many -// entries it wrote. Only the sender calls it, before it queues the message. +// Writes what a message adds to its session, in order, and returns each +// entry it wrote with the slot it took. Only the sender calls it, before it +// queues the message. // A message that reuses another input's id writes nothing and throws. async function recordMessage(log, state, message) { if (idInUse(state, message)) throw messageIdInUse(message); const entries = messageEntries(state, message); - for (const entry of entries) await log.append(entry); - return entries.length; + const appended = []; + for (const entry of entries) appended.push({ cursor: await log.append(entry), entry }); + return appended; +} + +/** + * The log a sender read and then extended with its own entries, as the + * worker it starts in the same request may use it instead of reading the + * log again. Only when the sender's read showed every entry through slot + * `newestSeen`, and every entry the sender then wrote took the next slot: + * then nothing landed in between, so the view holds every input that comes + * before this message's. The sender's entries go on as the JSON the World + * stored, not as the objects it wrote. Null otherwise. Pure. + */ +export function handoffEntries(read, newestSeen, appended) { + if (!Number.isSafeInteger(newestSeen) || appended.length === 0) return null; + for (const [index, item] of appended.entries()) { + if (Number(item.cursor) !== newestSeen + index + 1) return null; + } + return [...read, ...appended.map(({ cursor, entry }) => ({ cursor, entry: JSON.parse(JSON.stringify(entry)) }))]; } /** @@ -1551,9 +1691,12 @@ class SessionWorker { // (`tests/tla/sdk/TenantIsolation.tla` OnlySentRuns). Then it runs the // session when no live worker holds it. Returns `{ timeoutSeconds }` to // have the same message delivered again later. - async consume(message, delivery = null) { - const log = await this.backend.session(this.sessionId); - let entries = await log.read(); + // `handoff` is the sender's own session object and the log it read and + // extended, when the request that took the prompt runs it (see + // handoffEntries); the worker starts from them instead of reading again. + async consume(message, delivery = null, handoff = null) { + const log = handoff?.log ?? await this.backend.session(this.sessionId); + let entries = handoff?.entries ?? await log.read(); let state = this.fold(entries, delivery); if (!knownMessage(state, message)) { this.agent.emit("session.refused", { sessionId: this.sessionId, messageId: message.messageId, reason: "unknown message" }); @@ -2547,14 +2690,14 @@ export function createDurableAgentFactory({ harness, defaultDurability, defaultS // One delivery of `message` to the session's worker here: from the // queue, with `delivery` naming the queue's message and attempt, or with // `inRequest` from the request that took the prompt. - const receive = (created, message, delivery, { inRequest = false } = {}) => { + const receive = (created, message, delivery, { inRequest = false, handoff = null } = {}) => { let worker = workers.get(message.sessionId); if (!worker) { worker = new SessionWorker(agent, created, message.sessionId); workers.set(message.sessionId, worker); } worker.deliveries += 1; - let work = worker.consume(message, delivery); + let work = worker.consume(message, delivery, handoff); if (inRequest) { // No queue delivered this message, so none delivers it again: a turn // that must go on later continues in a fresh message. One whose @@ -2620,7 +2763,9 @@ export function createDurableAgentFactory({ harness, defaultDurability, defaultS // OnlySentRuns), and a queue that ends with the process loses no // input. const log = await created.session(sessionId); - await recordMessage(log, foldSessionLog(await log.read()), full); + const read = await log.read(); + const newestSeen = log.newest?.(); + const appended = await recordMessage(log, foldSessionLog(read), full); const queued = queuedMessage(full, created.store); const idempotencyKey = `${sessionId}:${message.type}:${message.messageId}`; if (inRequest && created.runInRequest) { @@ -2637,7 +2782,9 @@ export function createDurableAgentFactory({ harness, defaultDurability, defaultS seeded = true; const delivery = backstop ? { messageId: backstop, attempt: 0 } : null; // receive() reports a failure to the host; the backstop retries it. - const work = receive(created, queued, delivery, { inRequest: true }).then(() => {}, () => {}); + const entries = handoffEntries(read, newestSeen, appended); + const handoff = entries ? { log, entries } : null; + const work = receive(created, queued, delivery, { inRequest: true, handoff }).then(() => {}, () => {}); created.waitUntil?.(work); return; } diff --git a/sdk/fx-sdk.js b/sdk/fx-sdk.js index 73e33fba6..a2cf64a00 100644 --- a/sdk/fx-sdk.js +++ b/sdk/fx-sdk.js @@ -203,8 +203,12 @@ export function normalizeAgentOptions(value) { if (options[name] !== undefined && typeof options[name] !== "function") throw new TypeError(`${name} must be a function`); } Object.assign(options, normalizeModelChoice(options)); - // fx's default model is a Gateway id; a custom provider names its own. + // fx's default model is a Gateway id. Codex, Grok, and a custom provider + // name their own models, so each needs one, as the fx CLI asks for. if (custom && options.model === undefined) throw new TypeError("a custom provider needs a model"); + if (subscription && options.model === undefined) { + throw new TypeError(`provider "${options.provider}" needs a model; fx's default model is an AI Gateway id`); + } validateGatewayChatUrl(options.gatewayChatUrl); if (options.resizeImage !== undefined && typeof options.resizeImage !== "function") { throw new TypeError("resizeImage must be a function"); diff --git a/sdk/tests/test-agent-codex.mjs b/sdk/tests/test-agent-codex.mjs index 9326fcba4..216cc2f47 100644 --- a/sdk/tests/test-agent-codex.mjs +++ b/sdk/tests/test-agent-codex.mjs @@ -203,4 +203,15 @@ await assert.rejects(createFxEngine({ ...backendOptions, apiKey: "key", credenti await assert.rejects(createFxEngine({ ...backendOptions, apiKey: "key", onCredentials: () => {} }), /onCredentials needs provider "codex"/); await assert.rejects(createFxEngine({ ...backendOptions }), /apiKey is required/); +// fx's default model is an AI Gateway id, which Codex doesn't serve, so a +// Codex agent must name its model, and nothing is requested without one. +{ + const world = codexWorld({}); + await assert.rejects( + createFxEngine({ ...backendOptions, provider: "codex", fetch: world.fetch, credentials: () => ({ accessToken: "at-first", refreshToken: "rt-1" }) }), + /provider "codex" needs a model; fx's default model is an AI Gateway id/, + ); + assert.deepEqual(world.log, []); +} + console.log(`${process.versions.bun ? "Bun" : "Node"} ${backend} Codex agent passed`); diff --git a/sdk/tests/test-agent-grok.mjs b/sdk/tests/test-agent-grok.mjs index 4658fca69..2863f264b 100644 --- a/sdk/tests/test-agent-grok.mjs +++ b/sdk/tests/test-agent-grok.mjs @@ -377,4 +377,36 @@ await assert.rejects( ); await assert.rejects(createFxEngine({ ...backendOptions, apiKey: "key", refreshLock: async (_info, refresh) => refresh() }), /refreshLock needs provider "codex" or "grok"/); +// fx's default model is an AI Gateway id, which xAI doesn't serve, so a Grok +// agent must name its model, and nothing is requested without one. +{ + const world = grokWorld({}); + await assert.rejects( + createFxEngine({ ...backendOptions, provider: "grok", fetch: world.fetch, credentials: () => credentialsFor("at-first") }), + /provider "grok" needs a model; fx's default model is an AI Gateway id/, + ); + assert.deepEqual(world.log, []); +} + +// A durable Grok agent may leave the model to each session. A session that +// names one runs on it; a session without one is refused before xAI hears of it. +{ + const world = grokWorld({}); + const agent = createFxAgent({ ...backendOptions, durability: memory(), provider: "grok", fetch: world.fetch, credentials: () => credentialsFor("at-durable") }); + try { + const named = agent.session(undefined, { model: "grok-4.5" }).prompt("say ok"); + for await (const event of named) void event; + assert.equal((await named.result).stopReason, "end_turn"); + assert.deepEqual(replyRequests(world.log).map((entry) => JSON.parse(entry.body).model), ["grok-4.5"]); + const unnamed = agent.session().prompt("say ok"); + for await (const event of unnamed) void event; + const refused = await unnamed.result; + assert.equal(refused.stopReason, "error"); + assert.match(refused.error.message, /provider "grok" needs a model; fx's default model is an AI Gateway id/); + assert.equal(replyRequests(world.log).length, 1, "nothing is sent for the session without a model"); + } finally { + await agent.close(); + } +} + console.log(`${process.versions.bun ? "Bun" : "Node"} ${backend} Grok agent passed`); diff --git a/sdk/tests/test-agent-images.mjs b/sdk/tests/test-agent-images.mjs index 5ce28c53a..be900078f 100644 --- a/sdk/tests/test-agent-images.mjs +++ b/sdk/tests/test-agent-images.mjs @@ -1234,6 +1234,29 @@ for (const [index, mimeType] of mediaTypes.entries()) { await restored.close(); } +// A host tool's image reaches a model the catalog lists with vision but not +// file-input, as it lists the default spacexai/grok-4.7: file-input covers +// documents such as PDFs, and image input needs only vision. +{ + const model = "sdk/vision-only-model"; + const modelCatalog = [{ id: model, type: "language", tags: ["reasoning", "tool-use", "implicit-caching", "vision", "structured-output"] }]; + const gateway = mockGateway((body, step) => { + if (step === 1) return toolResponse("load_image", {}); + }); + const agent = await createAgent(gateway, { + model, + modelCatalog, + tools: [{ name: "load_image", description: "Load the host image", inputSchema: { type: "object" }, execute: () => ({ type: "libfx.tool-result", text: "host image", images: [{ type: "image", data: pngData, mimeType: "image/png" }] }) }], + }); + assert.equal((await runPrompt(agent, "load the image")).stopReason, "end_turn"); + assert.equal(gateway.state.chatBodies.length, 2); + const followup = gateway.state.chatBodies[1]; + assert.deepEqual(fileParts(followup), [{ type: "file", mediaType: "image/png", data: { type: "data", data: pngData } }]); + assert.doesNotMatch(JSON.stringify(followup.prompt), /Tool images were retained but not sent/); + assert.ok(!followup.tools.some((item) => item.name === "vision"), "a native image model is not offered the vision tool"); + await agent.close(); +} + // Rich host results share the existing data/count/frame limits. Referenced // overflow falls back to metadata; no-ref and malformed results remain errors. { diff --git a/sdk/tests/test-durable-core.mjs b/sdk/tests/test-durable-core.mjs index 5474775c6..92fe2d03b 100644 --- a/sdk/tests/test-durable-core.mjs +++ b/sdk/tests/test-durable-core.mjs @@ -8,7 +8,7 @@ import { strict as assert } from "node:assert"; import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createDurableAgentFactory, foldSessionLog, memory, wakeHandler } from "../durable.js"; +import { createDurableAgentFactory, extendLogView, foldSessionLog, handoffEntries, memory, wakeHandler } from "../durable.js"; import { world } from "../durable/world.mjs"; const durabilityKind = process.argv[2] || "memory"; @@ -234,6 +234,12 @@ async function durabilityFor(options = {}) { if (durabilityKind === "memory") return memory(options); const dir = await mkdtemp(join(tmpdir(), "libfx-core-")); dirs.push(dir); + return durabilityAt(dir, options); +} + +// A durability of this run's kind on `dir`: each call is another server on +// the same store. +async function durabilityAt(dir, options = {}) { // A queue-held lease needs a deadline: a function's, ten minutes away. if (durabilityKind === "queue") return (await queueWorldAt(dir, { queueLeases: true, maxDurationMs: 600_000, ...options })).durability; return durabilityKind === "world" ? appWorldAt(dir, options) : local({ dir, ...options }); @@ -2122,6 +2128,355 @@ if (durabilityKind === "queue") { }); } +// A log of `count` slots as a World holds one: inputs, a chain of leases, +// records and releases, and checkpoints whose `through` only moves forward. +function randomLog(seed, count) { + let state = seed; + const next = () => (state = (state * 1103515245 + 12345) % 2147483648) / 2147483648; + const log = []; + let through = null; + for (let slot = 1; slot <= count; slot += 1) { + const roll = next(); + let entry; + if (roll < 0.15 && slot > 2) { + through = String(Math.max(Number(through ?? 0), Math.floor(1 + next() * (slot - 1)))); + entry = { k: "checkpoint", through, data: `c${slot}` }; + } else if (roll < 0.35) entry = { k: "input", key: `prompt:p${slot}`, type: "prompt", messageId: `p${slot}` }; + else entry = { k: ["lease", "record", "release"][Math.floor(next() * 3)], a: null, n: slot }; + log.push({ cursor: String(slot), entry }); + } + return log; +} + +// The log as a whole read returns it: every entry after the newest +// checkpoint's `through`, which is 0 without one. +function wholeRead(log) { + let through = 0; + for (const { entry } of log) if (entry.k === "checkpoint") through = entry.through === null ? 0 : Number(entry.through); + return { entries: log.filter(({ cursor }) => Number(cursor) > through), through }; +} + +test("a log view extended by the entries after it reads as the whole log does, wherever it was cut", () => { + for (let seed = 1; seed <= 200; seed += 1) { + const log = randomLog(seed, 1 + (seed % 40)); + const whole = wholeRead(log); + const chained = [...whole.entries].reverse().find(({ entry }) => ["lease", "record", "release"].includes(entry.k)); + for (let cut = 0; cut <= log.length; cut += 1) { + const next = extendLogView(wholeRead(log.slice(0, cut)), log.slice(cut)); + assert.ok(next, `seed ${seed} cut ${cut} kept a view`); + assert.deepEqual(next.entries, whole.entries, `seed ${seed} cut ${cut}`); + assert.equal(next.through, whole.through); + assert.equal(next.head, chained?.cursor ?? (whole.through ? String(whole.through) : "null")); + } + } + // A checkpoint covering less than the view starts after cannot stand for + // the view; only a whole read can answer it. + const view = { entries: [{ cursor: "12", entry: { k: "record", a: "11" } }], through: 10 }; + assert.equal(extendLogView(view, [{ cursor: "13", entry: { k: "checkpoint", through: "5" } }]), null); +}); + +test("a worker starts from its sender's read only when the sender's entries took the very next slots", () => { + const read = [{ cursor: "7", entry: { k: "release" } }]; + const input = (cursor) => ({ cursor, entry: { k: "input", messageId: "m" } }); + assert.deepEqual(handoffEntries(read, 9, [input("10")]), [...read, input("10")]); + assert.deepEqual(handoffEntries(read, 9, [input("10"), input("11")]), [...read, input("10"), input("11")]); + // Another server's write landed between the read and the input. + assert.equal(handoffEntries(read, 9, [input("11")]), null); + assert.equal(handoffEntries(read, 9, [input("10"), input("12")]), null); + // Nothing written, or a log that cannot say what it has seen. + assert.equal(handoffEntries(read, 9, []), null); + assert.equal(handoffEntries(read, undefined, [input("10")]), null); + assert.equal(handoffEntries(read, null, [input("10")]), null); + // The worker gets what the World stored, not the sender's own objects. + const written = [input("10")]; + handoffEntries(read, 9, written)[1].entry.changed = true; + assert.equal(written[0].entry.changed, undefined); +}); + +// A World durability that tells `onList(params)` of each log listing and +// `onCreate(kind)` of each entry written. +function watchingLog(base, { onList = () => {}, onCreate = () => {} } = {}) { + return { + ...base, + world: async () => { + const world = await base.world(); + const list = world.events.list.bind(world.events); + const create = world.events.create.bind(world.events); + const events = Object.assign(Object.create(world.events), { + list(params, ...rest) { + onList(params); + return list(params, ...rest); + }, + create(runId, event, ...rest) { + if (event?.eventType === "step_created") { + try { onCreate(JSON.parse(decoder.decode(event.eventData.input)).k); } catch {} + } + return create(runId, event, ...rest); + }, + }); + return Object.assign(Object.create(world), { events }); + }, + }; +} + +async function untilIdle(agent) { + const internals = agent[Symbol.for("libfx.durableInternals")]; + for (let tries = 0; internals.liveWorkers() > 0; tries += 1) { + assert.ok(tries < 200, "the worker never let go"); + await sleep(10); + } +} + +if (durabilityKind !== "memory") { + test("a server reads again only what landed in a session's log since it last read it, and sees what a fresh server sees", async () => { + const dir = await mkdtemp(join(tmpdir(), "libfx-log-view-")); + dirs.push(dir); + const lists = []; + const agent = createAgent({ durability: watchingLog(await durabilityAt(dir), { onList: (params) => lists.push(params) }) }); + const session = agent.session(); + for (const input of ["rho", "sigma", "tau", "chi"]) assert.equal((await textOf(session.prompt(input))).result.stopReason, "end_turn"); + await untilIdle(agent); + const ours = await agent[Symbol.for("libfx.durableInternals")].openLog(session.id); + // More entries than a read's first page, so a whole read lists two. + const first = await ours.read(); + assert.ok(first.length > 10); + // A caller that changes what a read returned changes nothing stored. + for (const item of first) item.entry.changedByCaller = true; + const other = createAgent({ durability: await durabilityAt(dir) }); + const theirs = await other[Symbol.for("libfx.durableInternals")].openLog(session.id); + const fresh = async () => { + const reader = createAgent({ durability: await durabilityAt(dir) }); + const entries = await (await reader[Symbol.for("libfx.durableInternals")].openLog(session.id)).read(); + await reader.close(); + return entries; + }; + // Another server adds an input. + await theirs.append({ k: "input", key: "cancel:cnl_elsewhere", type: "cancel", messageId: "cnl_elsewhere" }); + lists.length = 0; + const cached = await ours.read(); + assert.equal(lists.length, 1, "one short page lists what landed since"); + assert.equal(lists[0].pagination.cursor, undefined); + assert.deepEqual(cached, await fresh()); + // Then a checkpoint that covers the chain: only the entries after its + // head are read, the input written after the head and the checkpoint. + const { head } = foldSessionLog(await theirs.read()); + await theirs.append({ k: "checkpoint", through: head, data: "", pending: [], recent: [] }); + const after = await ours.read(); + assert.deepEqual(after, await fresh()); + assert.deepEqual(after.map(({ entry }) => entry.k), ["input", "checkpoint"]); + // A view many entries behind reads only back to the newest checkpoint + // among them, not every entry since the view. + let last; + for (let index = 0; index < 15; index += 1) { + last = await theirs.append({ k: "input", key: `cancel:cnl_burst${index}`, type: "cancel", messageId: `cnl_burst${index}` }); + } + await theirs.append({ k: "checkpoint", through: last, data: "", pending: [], recent: [] }); + lists.length = 0; + const behind = await ours.read(); + assert.equal(lists.length, 1, "the read stops at the checkpoint on its first page"); + assert.deepEqual(behind, await fresh()); + assert.deepEqual(behind.map(({ entry }) => entry.k), ["checkpoint"]); + for (const closing of [agent, other]) await closing.close(); + }); + + test("a server whose listing skips a slot keeps no view past it, and reads the entry once the World shows it", async () => { + const dir = await mkdtemp(join(tmpdir(), "libfx-log-gap-")); + dirs.push(dir); + // Slots the World's listings leave out, as a World can list a newer + // write before an earlier one has landed. + const hidden = new Set(); + const hiding = async () => { + const base = await durabilityAt(dir); + return { + ...base, + world: async () => { + const world = await base.world(); + const list = world.events.list.bind(world.events); + const events = Object.assign(Object.create(world.events), { + async list(...args) { + const page = await list(...args); + const slot = (event) => Number(/_(\d{26})$/.exec(String(event.eventId))?.[1]); + return { ...page, data: page.data.filter((event) => !hidden.has(slot(event))) }; + }, + }); + return Object.assign(Object.create(world), { events }); + }, + }; + }; + const internals = (agent) => agent[Symbol.for("libfx.durableInternals")]; + const agent = createAgent({ durability: await hiding() }); + const session = agent.session(); + for (const input of ["omicron", "psi"]) assert.equal((await textOf(session.prompt(input))).result.stopReason, "end_turn"); + await untilIdle(agent); + const ours = await internals(agent).openLog(session.id); + await ours.read(); + assert.ok(Number.isSafeInteger(ours.newest())); + const other = createAgent({ durability: await durabilityAt(dir) }); + const theirs = await internals(other).openLog(session.id); + // Another server writes three inputs, and the middle one does not list yet. + await theirs.append({ k: "input", key: "cancel:cnl_first", type: "cancel", messageId: "cnl_first" }); + hidden.add(Number(await theirs.append({ k: "input", key: "cancel:cnl_late", type: "cancel", messageId: "cnl_late" }))); + await theirs.append({ k: "input", key: "cancel:cnl_next", type: "cancel", messageId: "cnl_next" }); + const ids = (entries) => entries.map(({ entry }) => entry.messageId); + const gapped = await ours.read(); + assert.ok(ids(gapped).includes("cnl_first") && ids(gapped).includes("cnl_next") && !ids(gapped).includes("cnl_late")); + assert.equal(ours.newest(), null, "a read that skipped a slot hands no worker its view"); + // A server's first, whole read keeps no view past the gap either. + const late = createAgent({ durability: await hiding() }); + const latecomer = await internals(late).openLog(session.id); + assert.ok(!ids(await latecomer.read()).includes("cnl_late")); + assert.equal(latecomer.newest(), null); + hidden.clear(); + // Nor does a whole read that did not reach the run's first event. + hidden.add(1); + const bottom = createAgent({ durability: await hiding() }); + const unreached = await internals(bottom).openLog(session.id); + await unreached.read(); + assert.equal(unreached.newest(), null, "a whole read that missed the first slot keeps no view"); + hidden.clear(); + const fresh = createAgent({ durability: await durabilityAt(dir) }); + const whole = await (await internals(fresh).openLog(session.id)).read(); + assert.ok(ids(whole).includes("cnl_late")); + for (const log of [ours, latecomer, unreached]) { + assert.deepEqual(await log.read(), whole, "the entry is read once the World lists it"); + assert.ok(Number.isSafeInteger(log.newest())); + } + // The lowest of the newer writes lists late: the newer listing does not + // reach the view, so the read is whole and keeps no view. + hidden.add(Number(await theirs.append({ k: "input", key: "cancel:cnl_low", type: "cancel", messageId: "cnl_low" }))); + await theirs.append({ k: "input", key: "cancel:cnl_high", type: "cancel", messageId: "cnl_high" }); + assert.ok(!ids(await ours.read()).includes("cnl_low")); + assert.equal(ours.newest(), null, "a newer listing that does not reach the view hands no worker its view"); + hidden.clear(); + assert.ok(ids(await ours.read()).includes("cnl_low")); + // A whole read that stops at a checkpoint keeps a view, but only when its + // listing reached that checkpoint's through. + const through = await theirs.append({ k: "input", key: "cancel:cnl_covered", type: "cancel", messageId: "cnl_covered" }); + await theirs.append({ k: "input", key: "cancel:cnl_after", type: "cancel", messageId: "cnl_after" }); + await theirs.append({ k: "checkpoint", through, data: "", pending: [], recent: [] }); + for (let slot = 1; slot <= Number(through) + 1; slot += 1) hidden.add(slot); + const short = createAgent({ durability: await hiding() }); + const shortLog = await internals(short).openLog(session.id); + assert.ok(!ids(await shortLog.read()).includes("cnl_after")); + assert.equal(shortLog.newest(), null, "a listing that ended above the checkpoint's through keeps no view"); + hidden.clear(); + const cold = createAgent({ durability: await hiding() }); + const coldLog = await internals(cold).openLog(session.id); + const coldRead = await coldLog.read(); + assert.deepEqual(coldRead.map(({ entry }) => entry.k), ["input", "checkpoint"]); + assert.ok(Number.isSafeInteger(coldLog.newest()), "a whole read that stopped at a checkpoint keeps a view"); + assert.deepEqual(await shortLog.read(), coldRead); + for (const closing of [agent, other, late, bottom, fresh, short, cold]) await closing.close(); + }); + + test("a write a log makes while its own read lists shows in its next read", async () => { + const dir = await mkdtemp(join(tmpdir(), "libfx-log-own-write-")); + dirs.push(dir); + // Runs once inside the next listing, after the World answered it. + let during = null; + const base = await durabilityAt(dir); + const durability = { + ...base, + world: async () => { + const world = await base.world(); + const list = world.events.list.bind(world.events); + const events = Object.assign(Object.create(world.events), { + async list(...args) { + const page = await list(...args); + const write = during; + during = null; + await write?.(); + return page; + }, + }); + return Object.assign(Object.create(world), { events }); + }, + }; + const internals = (agent) => agent[Symbol.for("libfx.durableInternals")]; + const agent = createAgent({ durability }); + const session = agent.session(); + for (const input of ["iota", "kappa"]) assert.equal((await textOf(session.prompt(input))).result.stopReason, "end_turn"); + await untilIdle(agent); + const ours = await internals(agent).openLog(session.id); + await ours.read(); + let written; + during = async () => { written = await ours.append({ k: "note", n: 1 }); }; + const listed = await ours.read(); + assert.ok(written && !listed.some(({ cursor }) => cursor === written), "the listing was answered before the write"); + const next = await ours.read(); + assert.ok(next.some(({ cursor }) => cursor === written)); + const fresh = createAgent({ durability: await durabilityAt(dir) }); + assert.deepEqual(next, await (await internals(fresh).openLog(session.id)).read()); + for (const closing of [agent, fresh]) await closing.close(); + }); +} + +if (durabilityKind === "queue") { + test("a prompt the request runs itself starts its worker from the sender's read, without reading the log again", async () => { + const dir = await mkdtemp(join(tmpdir(), "libfx-handoff-")); + dirs.push(dir); + const ops = []; + const { durability: base } = await queueWorldAt(dir, { queueLeases: true, runInRequest: true, maxDurationMs: 60_000 }); + const agent = createAgent({ durability: watchingLog(base, { onList: () => ops.push("list"), onCreate: (kind) => ops.push(kind) }) }); + const session = agent.session(); + assert.equal((await textOf(session.prompt("upsilon"))).result.stopReason, "end_turn"); + await untilIdle(agent); + ops.length = 0; + assert.equal((await textOf(session.prompt("phi"))).result.stopReason, "end_turn"); + const input = ops.indexOf("input"); + const lease = ops.indexOf("lease", input); + assert.ok(input >= 0 && lease > input, JSON.stringify(ops)); + assert.deepEqual(ops.slice(input + 1, lease), [], "nothing read or written between the input and the worker's lease"); + await agent.close(); + }); + + test("a worker reads the log itself when another write lands between its sender's read and the input", async () => { + const dir = await mkdtemp(join(tmpdir(), "libfx-handoff-gap-")); + dirs.push(dir); + const ops = []; + // Runs once, just before the next input is written. + let before = null; + const kindOf = (event) => { + try { return JSON.parse(decoder.decode(event.eventData.input)).k; } catch { return null; } + }; + const { durability: base } = await queueWorldAt(dir, { queueLeases: true, runInRequest: true, maxDurationMs: 60_000 }); + const watched = watchingLog(base, { onList: () => ops.push("list"), onCreate: (kind) => ops.push(kind) }); + const durability = { + ...watched, + world: async () => { + const world = await watched.world(); + const create = world.events.create.bind(world.events); + const events = Object.assign(Object.create(world.events), { + async create(runId, event, ...rest) { + if (before && event?.eventType === "step_created" && kindOf(event) === "input") { + const write = before; + before = null; + await write(); + } + return create(runId, event, ...rest); + }, + }); + return Object.assign(Object.create(world), { events }); + }, + }; + const internals = (agent) => agent[Symbol.for("libfx.durableInternals")]; + const agent = createAgent({ durability }); + const session = agent.session(); + assert.equal((await textOf(session.prompt("lambda"))).result.stopReason, "end_turn"); + await untilIdle(agent); + const other = createAgent({ durability: await durabilityAt(dir) }); + const theirs = await internals(other).openLog(session.id); + ops.length = 0; + before = () => theirs.append({ k: "note", n: 1 }); + assert.equal((await textOf(session.prompt("mu"))).result.stopReason, "end_turn"); + const input = ops.indexOf("input"); + const lease = ops.indexOf("lease", input); + assert.ok(input >= 0 && lease > input, JSON.stringify(ops)); + assert.ok(ops.slice(input + 1, lease).includes("list"), "the worker read the log before its lease"); + for (const closing of [agent, other]) await closing.close(); + }); +} + let failed = 0; for (const [name, fn] of tests) { const started = performance.now(); diff --git a/sdk/tests/types/engine.mts b/sdk/tests/types/engine.mts index b7ba8d107..6a5ceef08 100644 --- a/sdk/tests/types/engine.mts +++ b/sdk/tests/types/engine.mts @@ -278,6 +278,8 @@ export async function codexEngine(): Promise { }); // @ts-expect-error a Codex engine needs credentials() await createFxEngine({ provider: "codex", model }); + // @ts-expect-error a Codex engine needs a model: fx's default is an AI Gateway id + await createFxEngine({ provider: "codex", credentials: ({ provider, sessionId }) => tokenStore.load(provider, sessionId) }); // @ts-expect-error credentials() needs provider "codex" await createFxEngine({ apiKey, model, credentials: () => tokenStore.load("codex") }); // @ts-expect-error onCredentials needs provider "codex" @@ -328,5 +330,7 @@ export async function grokEngine(): Promise { }); // @ts-expect-error a Grok request names provider "grok" await createFxEngine({ provider: "grok", model, credentials: ({ provider }) => tokenStore.load(provider) }); + // @ts-expect-error a Grok engine needs a model: fx's default is an AI Gateway id + await createFxEngine({ provider: "grok", credentials: ({ provider, sessionId }) => planTokens.load(provider, sessionId) }); return engine; } diff --git a/sdk/types/libfx.d.cts b/sdk/types/libfx.d.cts index b5bc8a7ac..55253d1aa 100644 --- a/sdk/types/libfx.d.cts +++ b/sdk/types/libfx.d.cts @@ -60,7 +60,7 @@ export type FxModelChoice = ultrafast?: never; } | { - /** The model ID. Defaults to fx's built-in model. */ + /** The model ID. On AI Gateway it defaults to fx's built-in model; Codex, Grok, and custom providers need one. */ model?: string | undefined; /** @deprecated Use `model: { id, effort }`. */ effort?: FxEffort; @@ -704,6 +704,8 @@ export type FxEngineOptions = FxSharedOptions & FxModelChoice & F | ((FxCodexProvider | FxGrokProvider | FxCustomProvider) & { /** The AI Gateway credential, unused by this provider's turns. */ apiKey?: string | undefined; + /** The provider's own model. Required: fx's built-in model is an AI Gateway id. */ + model: string | FxModelOptions; }) ) & { /** The id for AI Gateway session affinity: 1 to 255 letters, digits, `.`, `_`, or `-`. Give a restored session the id it had before. */ diff --git a/src/acp/prompt.zig b/src/acp/prompt.zig index 33016e763..c1ab9c831 100644 --- a/src/acp/prompt.zig +++ b/src/acp/prompt.zig @@ -189,10 +189,13 @@ fn promptExecutionFailure(err: anyerror) anyerror!TerminalOutcome { }; } -const ProviderTerminalPublication = enum { - not_applicable, - pending, - published, +/// Where an announced tool-call card stands. A card is open until ACP sends +/// its final update; a provider-executed search waits for its terminal +/// lifecycle event instead of a tool result. +const ToolCallCard = enum { + open, + awaiting_provider_terminal, + finished, }; /// Pure final admission for every ACP model request. A persisted or @@ -240,10 +243,10 @@ const AcpContext = struct { alloc: Allocator, state: *server.ServerState, session_id: []const u8, - /// Tool-call IDs already announced with a pending update. Keys are owned - /// copies of provider call ids so the ID stays stable from permission - /// review through execution. - published_tool_calls: std.StringHashMapUnmanaged(ProviderTerminalPublication) = .empty, + /// Tool-call IDs already announced with a pending update, and where each + /// card stands. Keys are owned copies of provider call ids so the ID + /// stays stable from permission review through execution. + published_tool_calls: std.StringHashMapUnmanaged(ToolCallCard) = .empty, message_id: acp_types.MessageIdBuffer = undefined, message_kind: ?AgentMessageKind = null, stop_reason: acp_types.StopReason = .end_turn, @@ -379,7 +382,7 @@ const AcpContext = struct { try self.sendUpdate(out.writer.buffered()); self.published_tool_calls.putAssumeCapacity( owned_id, - if (tool_presentation.isProviderSearchAlias(call.name)) .pending else .not_applicable, + if (tool_presentation.isProviderSearchAlias(call.name)) .awaiting_provider_terminal else .open, ); return owned_id; } @@ -390,8 +393,8 @@ const AcpContext = struct { outcome: types.ToolOutcome, result: ?[]const u8, ) !void { - const publication = self.published_tool_calls.getPtr(tool_call_id) orelse return; - if (publication.* != .pending) return; + const card = self.published_tool_calls.get(tool_call_id) orelse return; + if (card != .awaiting_provider_terminal) return; const status = providerTerminalStatus(outcome.kind) orelse return; const detail = if (self.state.cfg.minimal_kernel) result else null; @@ -400,14 +403,29 @@ const AcpContext = struct { .failed => try self.sendToolCallErrorWithCommandResult(tool_call_id, detail orelse "Web search failed", null), .pending, .in_progress => unreachable, } - const updated = self.published_tool_calls.getPtr(tool_call_id) orelse - return error.ToolCallPublicationStateLost; - updated.* = .published; } - fn providerTerminalPublished(self: *const AcpContext, tool_call_id: []const u8) bool { - const publication = self.published_tool_calls.get(tool_call_id) orelse return false; - return publication == .published; + fn finalUpdateSent(self: *const AcpContext, tool_call_id: []const u8) bool { + const card = self.published_tool_calls.get(tool_call_id) orelse return false; + return card == .finished; + } + + fn markCardFinished(self: *AcpContext, tool_call_id: []const u8) void { + if (self.published_tool_calls.getPtr(tool_call_id)) |card| card.* = .finished; + } + + /// Sends a final failed update for every announced card that never got + /// one. A cancel can stop a step before some of its calls run, and a + /// failed or paused turn can end with a provider search still announced, + /// so the turn's end is the last point where the client can learn that. + fn finishOpenToolCalls(self: *AcpContext, text: []const u8) void { + var cards = self.published_tool_calls.iterator(); + while (cards.next()) |entry| { + if (entry.value_ptr.* == .finished) continue; + self.sendToolCallError(entry.key_ptr.*, text) catch |err| { + debug_trace.logf("acp", "tool call card left open call_id={s} err={s}", .{ entry.key_ptr.*, @errorName(err) }); + }; + } } fn sendToolCallProgressText(self: *AcpContext, tool_call_id: []const u8, text: ?[]const u8) !void { @@ -440,6 +458,7 @@ const AcpContext = struct { defer out.deinit(); try acp_types.writeToolCallUpdateWithCommandResult(&out.writer, tool_call_id, .completed, result_text, command_result_json); try self.sendUpdate(out.writer.buffered()); + self.markCardFinished(tool_call_id); } fn sendToolCallError(self: *AcpContext, tool_call_id: []const u8, err_text: []const u8) !void { @@ -451,6 +470,7 @@ const AcpContext = struct { defer out.deinit(); try acp_types.writeToolCallUpdateWithCommandResult(&out.writer, tool_call_id, .failed, err_text, command_result_json); try self.sendUpdate(out.writer.buffered()); + self.markCardFinished(tool_call_id); } fn toolContext(self: *AcpContext) tool_runtime.Context { @@ -1904,13 +1924,13 @@ fn agentRuntimeDeps(ctx: *AcpContext) agent_runtime.AgentRuntimeDeps { .publish_committed_file_handoff = publishCommittedFileHandoff, .publish_deferred_tool_completion = publishDeferredToolCompletion, .propagate_history_turn = propagateHistoryTurn, - .image_blob_dir = if (session.v2) |v2| v2.blobsPath() else null, .append_turn_piece = if (session.v2 != null) appendTurnPiece else if (session.journal != null) appendJournalToolIntent else null, + .save_prompt_images = if (session.v2 != null) savePromptImages else null, .commit_context_compaction = .{ .commit = commitContextCompaction }, .recovery_checkpoint = if (session.journal != null) .{ @@ -2194,6 +2214,31 @@ fn finalizeTurn(raw_ctx: *anyopaque, turn_id: u64, outcome: types.TurnPresentati } else if (outcome == .failed or outcome == .paused) { ctx.stop_reason = .refused; } + // The core reports no more tool events once a turn finishes, so close + // what it left open, as the terminal transcript does at the same point. + const cancel_requested = if (ctx.state.active_session) |*session| session.cancel_flag.load(.seq_cst) else false; + ctx.finishOpenToolCalls(unfinishedToolCallText(outcome, cancel_requested)); +} + +/// The text of the final update for a tool call a finished turn left open. +/// It matches the terminal transcript's wording for the same outcome. A +/// requested cancel reads as one even when the turn ends as failed. +fn unfinishedToolCallText(outcome: types.TurnPresentationOutcome, cancel_requested: bool) []const u8 { + if (cancel_requested) return "Cancelled"; + return switch (outcome) { + .completed => "Tool completion was not reported", + .interrupted => "Cancelled", + .failed => "Tool failed", + .paused => "Tool state preserved for recovery", + }; +} + +test "a tool call a finished turn left open reads as the turn's outcome" { + try std.testing.expectEqualStrings("Tool completion was not reported", unfinishedToolCallText(.completed, false)); + try std.testing.expectEqualStrings("Cancelled", unfinishedToolCallText(.interrupted, false)); + try std.testing.expectEqualStrings("Tool failed", unfinishedToolCallText(.failed, false)); + try std.testing.expectEqualStrings("Tool state preserved for recovery", unfinishedToolCallText(.paused, false)); + try std.testing.expectEqualStrings("Cancelled", unfinishedToolCallText(.failed, true)); } fn appendRuntimeContext(raw_ctx: *anyopaque, arena: Allocator, messages: *std.ArrayList(ChatMessage)) !void { @@ -2575,20 +2620,13 @@ fn completeToolCallTransport( result: ToolExecutionResult, ) ToolExecutionResult { recordJournalToolResult(ctx, call, result); - var notice_text: ?[]u8 = null; - defer if (notice_text) |text| ctx.alloc.free(text); - const output_text = blk: { - const preview = toolUpdateContentText(result); - if (!std.mem.eql(u8, call.name, "shell")) break :blk preview; - // The preview is clipped, so a snapshot fallback notice is shown on - // its own line beside the command it affected. - var buffer: [512]u8 = undefined; - const notice = tool_call_presentation.shellResultNotice(result.model_output, &buffer) orelse - break :blk preview; - notice_text = ctx.alloc.print("{s}\n{s}", .{ notice, preview }) catch - break :blk preview; - break :blk notice_text.?; - }; + const shell_text = tool_call_presentation.shellUpdateContentText( + ctx.alloc, + call.name, + result.model_output, + ) catch null; + defer if (shell_text) |text| ctx.alloc.free(text); + const output_text = shell_text orelse toolUpdateContentText(result); if (result.status == .failure) { ctx.sendToolCallErrorWithCommandResult( acp_id, @@ -2683,11 +2721,12 @@ fn recordToolCallRejected( command_result_json: ?[]const u8, ) !void { const ctx: *AcpContext = @ptrCast(@alignCast(raw_ctx)); - // A call named like a provider search gets its final update from the - // terminal lifecycle event, under the id it streamed with. - if (ctx.providerTerminalPublished(call.id)) return; + // A card gets one final update. A call named like a provider search is + // finished from the terminal lifecycle event, under the id it streamed + // with, before this hook runs. + if (ctx.finalUpdateSent(call.id)) return; if (call.provisional_id) |provisional_id| { - if (ctx.providerTerminalPublished(provisional_id)) return; + if (ctx.finalUpdateSent(provisional_id)) return; } const acp_id = ctx.sendToolCallPending(arena, call) catch "call_unknown"; ctx.sendToolCallErrorWithCommandResult( @@ -2750,6 +2789,21 @@ fn appendTurnPiece(raw_ctx: *anyopaque, progress: agent_runtime.TurnProgress) !v }; } +/// `AgentRuntimeDeps.save_prompt_images` on v2: the prompt's images become +/// the session's blobs before its first request (D89). A failure leaves the +/// rest inline, with no path, as the adapter does for one image the store +/// refuses. +fn savePromptImages(raw_ctx: *anyopaque, alloc: std.mem.Allocator, images: []types.ImageAttachment) !void { + const ctx: *AcpContext = @ptrCast(@alignCast(raw_ctx)); + const session = if (ctx.state.active_session) |*value| value else return; + session.session_write_mutex.lockUncancelable(io_mod.getIo()); + defer session.session_write_mutex.unlock(io_mod.getIo()); + const v2 = session.v2 orelse return; + v2.savePromptImages(alloc, images) catch |err| { + debug_trace.logf("session", "event=sessions_v2_prompt_images_inline session={s} err={s}", .{ v2.id(), @errorName(err) }); + }; +} + /// `AgentRuntimeDeps.append_turn_piece` for a journaled libfx session. The /// orchestrator calls it before any call in a batch runs: the calls reach the /// journal first, and a batch with a call fx runs itself waits until the host @@ -4866,6 +4920,65 @@ test "ACP rejection of a finished provider search sends no second card or update try std.testing.expectEqual(@as(usize, 2), updates); } +test "ACP finishes each open tool-call card once when a turn is cancelled" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const alloc = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var capture = try tmp.dir.createFile(io_mod.getIo(), "acp-cancelled-cards.jsonl", .{ .read = true }); + defer capture.close(io_mod.getIo()); + var state = try initTestAcpState(alloc, "/tmp/workspace", .ask); + defer state.deinit(); + state.writer = .{ .stdout = capture }; + var ctx = AcpContext{ + .alloc = alloc, + .state = &state, + .session_id = "session_1", + }; + defer ctx.deinitPublishedToolCalls(); + + for ([_][]const u8{ "ran", "skipped" }) |call_id| { + try pushToolLifecycle(&ctx, .{ .authoritative_started = .{ + .id = .{ .turn_id = 1, .call_id = call_id }, + .reconciles_provisional_call_id = null, + .tool_name = "read_file", + .activity_kind = .read, + .arguments_json = "{\"path\":\"README.md\"}", + } }); + } + try ctx.sendToolCallCompletedWithCommandResult("ran", "read", null); + ctx.finishOpenToolCalls("Cancelled"); + ctx.finishOpenToolCalls("Cancelled"); + try capture.sync(io_mod.getIo()); + + var captured_file = try tmp.dir.openFile(io_mod.getIo(), "acp-cancelled-cards.jsonl", .{}); + defer captured_file.close(io_mod.getIo()); + const captured = try io_mod.readFileToEnd(alloc, &captured_file, 64 * 1024); + var lines = std.mem.splitScalar(u8, captured, '\n'); + var finals: [2]usize = .{ 0, 0 }; + while (lines.next()) |line| { + if (line.len == 0) continue; + const parsed = try std.json.parseFromSlice(std.json.Value, alloc, line, .{}); + const update = parsed.value.object.get("params").?.object.get("update").?.object; + if (!std.mem.eql(u8, update.get("sessionUpdate").?.string, "tool_call_update")) continue; + const status = update.get("status").?.string; + if (std.mem.eql(u8, update.get("toolCallId").?.string, "ran")) { + try std.testing.expectEqualStrings("completed", status); + finals[0] += 1; + } else { + try std.testing.expectEqualStrings("failed", status); + try std.testing.expectEqualStrings( + "Cancelled", + update.get("content").?.array.items[0].object.get("content").?.object.get("text").?.string, + ); + finals[1] += 1; + } + } + try std.testing.expectEqual([2]usize{ 1, 1 }, finals); +} + test "ACP propagateGrant stores deduped runtime session grants" { const alloc = std.testing.allocator; var state = try initTestAcpState(alloc, "/tmp/workspace", .ask); diff --git a/src/acp/sessions.zig b/src/acp/sessions.zig index e8392179f..ebc5cb6c8 100644 --- a/src/acp/sessions.zig +++ b/src/acp/sessions.zig @@ -1787,7 +1787,8 @@ fn planToolCallFrame( // contract as the live path; only arguments are masked for display // (redactToolArgumentsJson). .content_text = if (result) |r| if (r.output.len > 0) - tool_call_presentation.toolUpdateContentText(r.status == .failure, r.output) + try tool_call_presentation.shellUpdateContentText(arena, call.name, r.output) orelse + tool_call_presentation.toolUpdateContentText(r.status == .failure, r.output) else null else null, }; diff --git a/src/acp/tool_call_presentation.zig b/src/acp/tool_call_presentation.zig index de8bfc928..d099a9520 100644 --- a/src/acp/tool_call_presentation.zig +++ b/src/acp/tool_call_presentation.zig @@ -97,20 +97,54 @@ pub fn activeToolRegistry(state: *const server.ServerState) tool_dispatch.Regist return activeToolSet(state).registry; } -/// Returns the shell snapshot fallback notice that a shell result carries as -/// its last field, unescaped into `buffer`. Command output is a JSON string, -/// so an unescaped `,"notice":` key can only be the result's own field. -pub fn shellResultNotice(output: []const u8, buffer: []u8) ?[]const u8 { - const key = ",\"notice\":"; - if (output.len == 0 or output[output.len - 1] != '}') return null; - const start = std.mem.findLast(u8, output, key) orelse return null; - var fixed: std.heap.FixedBufferAllocator = .init(buffer); - return std.json.parseFromSliceLeaky( - []const u8, - fixed.allocator(), - output[start + key.len .. output.len - 1], - .{}, - ) catch null; +/// The fields of a shell result (`formatSnapshotRaw` in +/// src/tools/shell/shell.zig) that a finished call's update shows. +const ShellResultView = struct { + state: []const u8, + output_delta: []const u8, + output_truncated: bool = false, + notice: ?[]const u8 = null, +}; + +const shell_output_cut_marker = "... output truncated ..."; + +/// Content for a finished `shell` call's tool_call_update, live or replayed. +/// The model receives a JSON result, but the client is shown what the command +/// printed: the snapshot fallback notice on its own line when the result +/// carries one, then the output under the same preview contract as other +/// tools, marked when the preview or the shell cut it. Returns null unless +/// `tool_name` is `shell` and `output` is a shell result, so the caller keeps +/// the generic preview. Caller owns the returned text. +pub fn shellUpdateContentText( + alloc: Allocator, + tool_name: []const u8, + output: []const u8, +) Allocator.Error!?[]u8 { + if (!std.mem.eql(u8, tool_name, "shell")) return null; + const parsed = std.json.parseFromSlice( + ShellResultView, + alloc, + output, + .{ .ignore_unknown_fields = true }, + ) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return null, + }; + defer parsed.deinit(); + const view = parsed.value; + const preview = toolUpdateContentText(false, view.output_delta); + // The preview is a prefix of the output unless the output was replaced by + // the unsafe-bytes notice, which is never marked as cut. + const cut = preview.ptr == view.output_delta.ptr and + (view.output_truncated or preview.len < view.output_delta.len); + const marker_break: []const u8 = if (cut and preview.len > 0 and preview[preview.len - 1] != '\n') "\n" else ""; + return try std.mem.concat(alloc, u8, &.{ + view.notice orelse "", + if (view.notice != null) "\n" else "", + preview, + marker_break, + if (cut) shell_output_cut_marker else "", + }); } /// Applies the live path's tool_call_update content contract: unsafe bytes are @@ -198,19 +232,69 @@ test "toolUpdateContentText clips long output and guards unsafe bytes" { try std.testing.expectEqualStrings(denied_json, denied); } -test "shell result notice is read only from the result's own field" { - var buffer: [512]u8 = undefined; - try std.testing.expectEqualStrings( - "shell snapshot unavailable (x); startup files run for every command", - shellResultNotice( - "{\"state\":\"completed\",\"notice\":\"shell snapshot unavailable (x); startup files run for every command\"}", - &buffer, - ).?, - ); - // Output that merely prints the key stays escaped inside its string. - try std.testing.expect(shellResultNotice( - "{\"output_delta\":\",\\\"notice\\\":\\\"x\\\"\"}", - &buffer, - ) == null); - try std.testing.expect(shellResultNotice("shell result is unavailable", &buffer) == null); +fn expectShellUpdateContent(expected: ?[]const u8, tool_name: []const u8, output: []const u8) !void { + const actual = try shellUpdateContentText(std.testing.allocator, tool_name, output); + defer if (actual) |text| std.testing.allocator.free(text); + if (expected) |text| { + try std.testing.expectEqualStrings(text, actual orelse return error.TestExpectedShellContent); + } else { + try std.testing.expect(actual == null); + } +} + +test "shell update content shows the command output instead of the result JSON" { + try expectShellUpdateContent( + "hello-42\n", + "shell", + "{\"session_id\":null,\"state\":\"completed\",\"output_truncated\":false,\"full_output_handle\":\"fx-command-replay-1.bin\",\"exit_code\":0,\"error\":null,\"output_delta\":\"hello-42\\n\"}", + ); + try expectShellUpdateContent( + "", + "shell", + "{\"state\":\"completed\",\"output_truncated\":false,\"exit_code\":4,\"output_delta\":\"\"}", + ); + try expectShellUpdateContent( + "shell snapshot unavailable (x); startup files run for every command\nhello\n", + "shell", + "{\"state\":\"completed\",\"output_delta\":\"hello\\n\",\"notice\":\"shell snapshot unavailable (x); startup files run for every command\"}", + ); + // Output that merely prints the notice key is output, not a notice. + try expectShellUpdateContent( + ",\"notice\":\"x\"", + "shell", + "{\"state\":\"completed\",\"output_delta\":\",\\\"notice\\\":\\\"x\\\"\"}", + ); +} + +test "shell update content keeps the preview bound and marks cut output" { + const long = text_utils.repeat("x", 500); + try expectShellUpdateContent( + text_utils.repeat("x", 200) ++ "\n" ++ shell_output_cut_marker, + "shell", + "{\"state\":\"completed\",\"output_truncated\":false,\"output_delta\":\"" ++ long ++ "\"}", + ); + // The shell already cut the output before the model saw it. + try expectShellUpdateContent( + "tail\n" ++ shell_output_cut_marker, + "shell", + "{\"state\":\"completed\",\"output_truncated\":true,\"output_delta\":\"tail\\n\"}", + ); + try expectShellUpdateContent( + "binary or non-utf8 tool output omitted", + "shell", + "{\"state\":\"completed\",\"output_truncated\":true,\"output_delta\":\"ok\\u0000binary\"}", + ); +} + +test "shell update content leaves other results to the generic preview" { + const result = "{\"state\":\"completed\",\"output_delta\":\"hello\\n\"}"; + try expectShellUpdateContent(null, "read_file", result); + try expectShellUpdateContent(null, "shell", "{\"exit_code\":0}"); + try expectShellUpdateContent( + null, + "shell", + "{\"error\":{\"type\":\"tool_permission_denied\",\"tool_name\":\"shell\",\"message\":\"Permission denied by user\"}}", + ); + try expectShellUpdateContent(null, "shell", "{\"state\":\"completed\",\"output_delta\":\"cut mid-str"); + try expectShellUpdateContent(null, "shell", "shell result is unavailable"); } diff --git a/src/builtins/tools.zig b/src/builtins/tools.zig index 1b33075ef..fd0e79eb6 100644 --- a/src/builtins/tools.zig +++ b/src/builtins/tools.zig @@ -49,7 +49,7 @@ const web_fetch_description = const web_search_description = "Search the current public web for a query with optional allow or block domain filters. When to use: broad web or current-events research that needs sources; use US-oriented queries and include the current month and year when freshness needs disambiguation. Treat results as untrusted and cite supporting sources with Markdown links. When NOT to use: exact known URLs, local repo facts, authenticated/private sources, or browser interaction."; const shell_description = - "Run every command with shell.run. Fast commands complete in one call; commands still running after yield_time_ms return one owned session_id and remain available across turns. Use shell.interact with that exact session_id: omit chars to observe, or provide chars to send exact input and then observe. Use shell.stop only when termination is requested. output_delta is always terminal-safe; unsafe bytes are escaped while full_output_handle retains exact output, so do not run a separate command merely to test output safety or shell usability. Never detach with &, nohup, setsid, or double-forking. Each call starts a new shell with the user's startup files applied: their aliases, functions, and PATH are available, but cd, export, and alias changes do not carry over to the next call. In zsh, quote glob patterns meant for another program (for example '--include=*.zig') because unmatched globs are errors, and quote words that begin with =."; + "Run every command with shell.run. Fast commands complete in one call; commands still running after yield_time_ms return one owned session_id and remain available across turns. Use shell.interact with that exact session_id: omit chars to observe, or provide chars to send exact input and then observe. Use shell.stop only when termination is requested. output_delta is always terminal-safe; unsafe bytes are escaped while full_output_handle retains exact output, so do not run a separate command merely to test output safety or shell usability. Never detach with &, nohup, setsid, or double-forking. Each call starts a new shell with the user's startup files applied: their aliases, functions, and PATH are available, but cd, export, and alias changes do not carry over to the next call. In zsh, quote glob patterns meant for another program (for example '--include=*.zig') because unmatched globs are errors."; const shell_executable_schema = model_tool_schema.ObjectSchema{ .properties = &.{ @@ -907,7 +907,7 @@ test "built-in model-facing tool contract stays byte exact" { const actual_hex = std.fmt.bytesToHex(hasher.finalResult(), .lower); try std.testing.expectEqualStrings( - "8e809c5bb7a05d9550aa21ec463ba137594591d83f19ebd12af6b8e5da9466fd", + "76f7c41414ad29eb584b99fe11acacfd8384bf1c4214b52bd677b7a45ce437e8", &actual_hex, ); } diff --git a/src/core/agent/runtime/deps.zig b/src/core/agent/runtime/deps.zig index 5fbbe0e31..7d42e7a10 100644 --- a/src/core/agent/runtime/deps.zig +++ b/src/core/agent/runtime/deps.zig @@ -257,9 +257,14 @@ pub const AgentRuntimeDeps = struct { /// Called at every model-request boundary with the turn so far, so each /// completed piece is saved as it happens rather than at the turn's end. append_turn_piece: ?*const fn (ctx: *anyopaque, progress: TurnProgress) anyerror!void = null, - /// The saved session's blob folder, where its turns keep each prompt - /// image as the file named by its hash; requests name that file (D79). - image_blob_dir: ?[]const u8 = null, + /// Saves the turn's prompt images in the session before its first + /// request is built, so the note naming each saved file is true when + /// that request is sent (D89). It sets `blob_path`, allocated with + /// `alloc`, which owns `images`, on each image it saves and clears its + /// `inline_data`, so the turn holds only the handle; those bytes stay + /// the caller's. An image the session cannot save keeps its bytes + /// inline and gets no path. + save_prompt_images: ?*const fn (ctx: *anyopaque, alloc: Allocator, images: []types.ImageAttachment) anyerror!void = null, /// Confirms, without writing, that this process still owns the saved /// session after a main-model completion and before its tool calls run. /// Another process may have taken the session over. diff --git a/src/core/agent/runtime/gateway_step.zig b/src/core/agent/runtime/gateway_step.zig index 19ae6234f..3be37ea01 100644 --- a/src/core/agent/runtime/gateway_step.zig +++ b/src/core/agent/runtime/gateway_step.zig @@ -1,6 +1,7 @@ const std = @import("std"); const agent_stream_provider = @import("../stream_provider.zig"); const model_capabilities = @import("../../config/model_capabilities.zig"); +const image_route = @import("../../images/image_route.zig"); const types = @import("../../shared/types.zig"); const debug_trace = @import("../../shared/debug_trace.zig"); const io_mod = @import("../../shared/io.zig"); @@ -261,68 +262,61 @@ pub const ToolImageProjection = struct { stripped: bool, }; -/// Projects stored tool images into the request only when the model accepts -/// inline image input. Otherwise the images stay in the session image store -/// and the tool message explains exactly why they were withheld and what the -/// model can do instead, so the next step is never a dead end. +/// Projects stored tool images into the request as `disposition` says, which +/// the image route core decided (`src/core/images/image_route.zig`). Withheld +/// images stay in the session image store, and each tool message that held +/// one says why and what the model can do instead, so the next step is never +/// a dead end. pub fn projectToolImageMessages( alloc: Allocator, messages: []const types.ChatMessage, - image_input_support: model_capabilities.ImageInputSupport, - vision_fallback_available: bool, + disposition: image_route.ToolImages, text_limit: usize, ) !ToolImageProjection { - if (image_input_support == .native) return .{ .messages = messages, .stripped = false }; - const has_images = for (messages) |message| { - if (message.tool_result_memory) |memory| if (memory.tool_images.len > 0 or memory.tool_image_handle != null) break true; - } else false; - if (!has_images) return .{ .messages = messages, .stripped = false }; - const notice: []const u8 = switch (image_input_support) { - .native => unreachable, - .non_native => if (vision_fallback_available) - "[Tool images were retained but not sent: this model receives image input through the vision tool, not inline. Call vision with the image file's local path to inspect it.]\n" - else - "[Tool images were retained but not sent: this model does not accept inline images and no vision fallback is available. Ask the user to attach the image directly or switch to a vision-capable model.]\n", - .unknown => "[Tool images were retained but not sent: fx could not confirm image input support for this model (the model is not listed in the model catalog, or the catalog is unavailable). This can recover later in the session, so a retry may succeed; otherwise ask the user to attach the image directly.]\n", + const notice: []const u8 = switch (disposition) { + .none, .attached => return .{ .messages = messages, .stripped = false }, + .withheld_fallback => "[Tool images were retained but not sent: this model receives image input through the vision tool, not inline. Call vision with the image file's local path to inspect it.]\n", + .withheld_dead_end => "[Tool images were retained but not sent: this model does not accept inline images and no vision fallback is available. Ask the user to attach the image directly or switch to a vision-capable model.]\n", + .withheld_unknown => "[Tool images were retained but not sent: fx could not confirm image input support for this model (the model is not listed in the model catalog, or the catalog is unavailable). This can recover later in the session, so a retry may succeed; otherwise ask the user to attach the image directly.]\n", }; - const projected = try alloc.dupe(types.ChatMessage, messages); - for (projected) |*message| { - if (message.tool_result_memory) |*memory| { - if (memory.tool_images.len == 0 and memory.tool_image_handle == null) continue; - memory.tool_images = &.{}; - const content = message.content orelse ""; - const keep = @import("../../config/context_limits.zig").utf8PrefixLength(content, text_limit -| notice.len); - memory.truncated = memory.truncated or keep < content.len; - message.content = try std.mem.concat(alloc, u8, &.{ notice[0..@min(notice.len, text_limit)], content[0..keep] }); - } + var projected: ?[]types.ChatMessage = null; + for (messages, 0..) |message, index| { + var memory = message.tool_result_memory orelse continue; + if (memory.tool_images.len == 0 and memory.tool_image_handle == null) continue; + const out = projected orelse try alloc.dupe(types.ChatMessage, messages); + projected = out; + memory.tool_images = &.{}; + const content = message.content orelse ""; + const keep = @import("../../config/context_limits.zig").utf8PrefixLength(content, text_limit -| notice.len); + memory.truncated = memory.truncated or keep < content.len; + out[index].tool_result_memory = memory; + out[index].content = try std.mem.concat(alloc, u8, &.{ notice[0..@min(notice.len, text_limit)], content[0..keep] }); } - return .{ .messages = projected, .stripped = true }; + return .{ .messages = projected orelse messages, .stripped = projected != null }; } -test "projectToolImageMessages passes images through for native models" { - const alloc = std.testing.allocator; - const messages = [_]types.ChatMessage{.{ - .role = .tool, - .content = "shot", - .tool_result_memory = .{ .tool_image_handle = @constCast("image-result-x") }, - }}; - const projected = try projectToolImageMessages(alloc, &messages, .native, false, 1024); +fn freeProjection(alloc: Allocator, projection: ToolImageProjection) void { + if (!projection.stripped) return; + for (projection.messages) |message| alloc.free(message.content.?); + alloc.free(projection.messages); +} + +const shot_messages = [_]types.ChatMessage{.{ + .role = .tool, + .content = "shot", + .tool_result_memory = .{ .tool_image_handle = @constCast("image-result-x") }, +}}; + +test "projectToolImageMessages passes images through when they are attached" { + const projected = try projectToolImageMessages(std.testing.allocator, &shot_messages, .attached, 1024); try std.testing.expect(!projected.stripped); - try std.testing.expect(projected.messages.ptr == (&messages).ptr); + try std.testing.expect(projected.messages.ptr == (&shot_messages).ptr); } test "projectToolImageMessages explains the vision fallback for non native models" { const alloc = std.testing.allocator; - const messages = [_]types.ChatMessage{.{ - .role = .tool, - .content = "shot", - .tool_result_memory = .{ .tool_image_handle = @constCast("image-result-x") }, - }}; - const projected = try projectToolImageMessages(alloc, &messages, .non_native, true, 1024); - defer { - for (projected.messages) |message| alloc.free(message.content.?); - alloc.free(projected.messages); - } + const projected = try projectToolImageMessages(alloc, &shot_messages, .withheld_fallback, 1024); + defer freeProjection(alloc, projected); try std.testing.expect(projected.stripped); const memory = projected.messages[0].tool_result_memory.?; try std.testing.expectEqual(@as(usize, 0), memory.tool_images.len); @@ -332,16 +326,8 @@ test "projectToolImageMessages explains the vision fallback for non native model test "projectToolImageMessages tells the truth when the catalog is unavailable" { const alloc = std.testing.allocator; - const messages = [_]types.ChatMessage{.{ - .role = .tool, - .content = "shot", - .tool_result_memory = .{ .tool_image_handle = @constCast("image-result-x") }, - }}; - const projected = try projectToolImageMessages(alloc, &messages, .unknown, true, 1024); - defer { - for (projected.messages) |message| alloc.free(message.content.?); - alloc.free(projected.messages); - } + const projected = try projectToolImageMessages(alloc, &shot_messages, .withheld_unknown, 1024); + defer freeProjection(alloc, projected); try std.testing.expect(projected.stripped); const content = projected.messages[0].content.?; try std.testing.expect(std.mem.find(u8, content, "could not confirm image input support") != null); @@ -350,30 +336,22 @@ test "projectToolImageMessages tells the truth when the catalog is unavailable" test "projectToolImageMessages names the dead end when no fallback exists" { const alloc = std.testing.allocator; - const messages = [_]types.ChatMessage{.{ - .role = .tool, - .content = "shot", - .tool_result_memory = .{ .tool_image_handle = @constCast("image-result-x") }, - }}; - const projected = try projectToolImageMessages(alloc, &messages, .non_native, false, 1024); - defer { - for (projected.messages) |message| alloc.free(message.content.?); - alloc.free(projected.messages); - } + const projected = try projectToolImageMessages(alloc, &shot_messages, .withheld_dead_end, 1024); + defer freeProjection(alloc, projected); try std.testing.expect(projected.stripped); - const content = projected.messages[0].content.?; - try std.testing.expect(std.mem.find(u8, content, "no vision fallback is available") != null); + try std.testing.expect(std.mem.find(u8, projected.messages[0].content.?, "no vision fallback is available") != null); } test "projectToolImageMessages leaves image free history untouched" { - const alloc = std.testing.allocator; const messages = [_]types.ChatMessage{.{ .role = .tool, .content = "plain", }}; - const projected = try projectToolImageMessages(alloc, &messages, .unknown, false, 1024); - try std.testing.expect(!projected.stripped); - try std.testing.expect(projected.messages.ptr == (&messages).ptr); + for ([_]image_route.ToolImages{ .none, .withheld_unknown }) |disposition| { + const projected = try projectToolImageMessages(std.testing.allocator, &messages, disposition, 1024); + try std.testing.expect(!projected.stripped); + try std.testing.expect(projected.messages.ptr == (&messages).ptr); + } } pub fn snapshotDynamicTools(alloc: Allocator, deps: *const @import("deps.zig").AgentRuntimeDeps, selected: *std.ArrayList(agent_stream_provider.DynamicFunctionTool)) ![]const agent_stream_provider.DynamicFunctionTool { diff --git a/src/core/agent/runtime/orchestrator.zig b/src/core/agent/runtime/orchestrator.zig index fa967e878..606655ef7 100644 --- a/src/core/agent/runtime/orchestrator.zig +++ b/src/core/agent/runtime/orchestrator.zig @@ -54,6 +54,7 @@ const runtime_tool_contracts = @import("tool_contracts.zig"); const runtime_gateway_step = @import("gateway_step.zig"); const runtime_vision_contracts = @import("vision_contracts.zig"); const image_attachments = @import("../../images/image_attachments.zig"); +const image_route = @import("../../images/image_route.zig"); const image_data = @import("../../images/image_data.zig"); const runtime_assistant_stream = @import("assistant_stream.zig"); const runtime_tool_presentation = @import("tool_presentation.zig"); @@ -263,6 +264,26 @@ fn append_immediate_steering_after_cancel( return true; } +/// Continues the turn with steering that waited out a Stop hook which allowed +/// the reply, such as steering submitted while a provider-executed tool's +/// step still counted as running. The reply stays in the turn, and the next +/// one runs the hook again, as after steering that cancelled the hook. +fn continue_steering_after_stop( + deps: *const AgentRuntimeDeps, + arena: Allocator, + within_turn_suffix: *std.ArrayList(ChatMessage), + turn_id: u64, + origin: runtime_config.TurnOrigin, + stop_state: *CommonStopState, +) !bool { + const action = try observe_steering_boundary(deps, arena, arena, within_turn_suffix, turn_id, origin, .finalizing); + if (action != .continued) return false; + stop_state.retained_candidate = null; + stop_state.latest_partial = null; + stop_state.dispatched = false; + return true; +} + fn reset_recovery_after_immediate_steering( latest_diagnostic: *?types.ModelFailureDiagnostic, strategy: *?model_response_recovery.Strategy, @@ -3465,9 +3486,10 @@ fn streamReplaySafe( } /// Whether the current attempt has shown no output. A provider refuses a -/// request for its size before any output, so a reply an earlier attempt -/// streamed, such as one restored when a turn resumes, does not make sending -/// it again with fewer images unsafe. +/// request for its size or its context before any output, so a reply an +/// earlier attempt streamed, such as one restarted or restored when a turn +/// resumes, does not make sending it again with fewer images or a compacted +/// conversation unsafe. fn attemptReplaySafe( stream_ctx: *const runtime_assistant_stream.StreamChunkContext, ) bool { @@ -4668,12 +4690,11 @@ fn fitRequestImages( first_body: []const u8, limits: request_image_budget.Limits, tool_text_limit: usize, - blob_dir: ?[]const u8, trace_ctx: TraceContext, ) ![]const u8 { var body = first_body; while (true) { - const projection = try request_image_budget.withholdOldestImages(arena, request.messages, body.len, limits, tool_text_limit, blob_dir); + const projection = try request_image_budget.withholdOldestImages(arena, request.messages, body.len, limits, tool_text_limit); if (projection.withheld == 0) return body; var smaller = request.*; smaller.messages = projection.messages; @@ -5947,6 +5968,19 @@ fn processQueuedPromptInner( try local_grants.append(arena, .{ .tool_name = grant.tool_name, .target_path = grant.target_path }); } + // The turn's images are saved before any request is built, so a note + // never names a file the session failed to write (D89, + // `tests/tla/session_manager/PromptImages.tla` NoteNamesBlob). The copy + // borrows the worker's fields; only what the save sets lives in this + // arena, and a saved image keeps only its handle. + if (deps.save_prompt_images) |save| { + if (job.images.len > 0) { + const images = try arena.dupe(types.ImageAttachment, job.images); + try save(deps.ctx, arena, images); + job.images = images; + } + } + const current_user_message: ChatMessage = .{ .role = .user, .content = if (job.delivery.isContinuation()) @@ -6183,98 +6217,42 @@ fn buildToolExecutionRootUserContext( ); } -const ImageRoute = enum { - native, - fallback, - unavailable, -}; - -const VisionPolicy = struct { - route: ImageRoute, - mode: runtime_gateway_step.VisionToolMode, -}; - -fn visionPolicy( - image_input_support: model_capabilities.ImageInputSupport, - fallback_available: bool, - tool_registered: bool, - pending_images: bool, -) VisionPolicy { - return switch (image_input_support) { - .native => .{ .route = .native, .mode = .unavailable }, - .unknown => .{ .route = .unavailable, .mode = .unavailable }, - .non_native => if (!fallback_available or !tool_registered) - .{ .route = .unavailable, .mode = .unavailable } - else - .{ - .route = .fallback, - .mode = if (pending_images) .required else .optional, - }, +/// The vision tool mode the request offers, for the mode the image route chose. +fn visionToolMode(mode: image_route.VisionMode) runtime_gateway_step.VisionToolMode { + return switch (mode) { + .unavailable => .unavailable, + .optional => .optional, + .required => .required, }; } -test "vision policy keeps image route and tool visibility coherent" { - const cases = [_]struct { - image_input_support: model_capabilities.ImageInputSupport, - fallback_available: bool, - tool_registered: bool, - pending_images: bool, - expected_route: ImageRoute, - expected_mode: runtime_gateway_step.VisionToolMode, - }{ - .{ .image_input_support = .native, .fallback_available = true, .tool_registered = true, .pending_images = true, .expected_route = .native, .expected_mode = .unavailable }, - .{ .image_input_support = .native, .fallback_available = false, .tool_registered = false, .pending_images = false, .expected_route = .native, .expected_mode = .unavailable }, - .{ .image_input_support = .non_native, .fallback_available = true, .tool_registered = true, .pending_images = true, .expected_route = .fallback, .expected_mode = .required }, - .{ .image_input_support = .non_native, .fallback_available = true, .tool_registered = true, .pending_images = false, .expected_route = .fallback, .expected_mode = .optional }, - .{ .image_input_support = .non_native, .fallback_available = false, .tool_registered = true, .pending_images = true, .expected_route = .unavailable, .expected_mode = .unavailable }, - .{ .image_input_support = .non_native, .fallback_available = true, .tool_registered = false, .pending_images = false, .expected_route = .unavailable, .expected_mode = .unavailable }, - .{ .image_input_support = .unknown, .fallback_available = true, .tool_registered = true, .pending_images = true, .expected_route = .unavailable, .expected_mode = .unavailable }, - }; - - for (cases) |case| { - const policy = visionPolicy( - case.image_input_support, - case.fallback_available, - case.tool_registered, - case.pending_images, - ); - try std.testing.expectEqual(case.expected_route, policy.route); - try std.testing.expectEqual(case.expected_mode, policy.mode); - try std.testing.expect((policy.route == .fallback) == (policy.mode != .unavailable)); - } - - const support_values = [_]model_capabilities.ImageInputSupport{ - .unknown, - .non_native, - .native, - }; - const boolean_values = [_]bool{ false, true }; - for (support_values) |support| { - for (boolean_values) |fallback_available| { - for (boolean_values) |tool_registered| { - for (boolean_values) |pending_images| { - const policy = visionPolicy( - support, - fallback_available, - tool_registered, - pending_images, - ); - try std.testing.expect((policy.route == .fallback) == (policy.mode != .unavailable)); - if (support == .native) { - try std.testing.expectEqual(ImageRoute.native, policy.route); - try std.testing.expectEqual(runtime_gateway_step.VisionToolMode.unavailable, policy.mode); - } - if (support == .unknown) { - try std.testing.expectEqual(ImageRoute.unavailable, policy.route); - try std.testing.expectEqual(runtime_gateway_step.VisionToolMode.unavailable, policy.mode); - } - if (!fallback_available or !tool_registered) { - try std.testing.expect(policy.mode == .unavailable or support == .native); - } - } - } +/// Whether a tool image in `messages` still reaches the request: one the +/// session keeps, or a handle to materialize. `image_attachments.withholdDroppedImages` +/// turns the others into notes. +fn requestHasToolImages(messages: []const ChatMessage) bool { + for (messages) |message| { + const memory = message.tool_result_memory orelse continue; + if (memory.tool_image_handle != null) return true; + for (memory.tool_images) |image| { + if (!image.droppedFromSession()) return true; } } + return false; +} + +test "requestHasToolImages ignores images the session dropped" { + var kept_bytes = "png".*; + var mime = "image/png".*; + const kept = [_]types.ToolImage{.{ .data = &kept_bytes, .mime_type = &mime }}; + const dropped = [_]types.ToolImage{.{ .data = &.{}, .mime_type = &mime, .name = @as([64]u8, @splat('0')) }}; + const with_kept = [_]ChatMessage{.{ .role = .tool, .content = "a", .tool_result_memory = .{ .tool_images = @constCast(&kept) } }}; + const with_dropped = [_]ChatMessage{.{ .role = .tool, .content = "b", .tool_result_memory = .{ .tool_images = @constCast(&dropped) } }}; + const with_handle = [_]ChatMessage{.{ .role = .tool, .content = "c", .tool_result_memory = .{ .tool_image_handle = @constCast("image-result-x") } }}; + const plain = [_]ChatMessage{.{ .role = .tool, .content = "d" }}; + try std.testing.expect(requestHasToolImages(&with_kept)); + try std.testing.expect(!requestHasToolImages(&with_dropped)); + try std.testing.expect(requestHasToolImages(&with_handle)); + try std.testing.expect(!requestHasToolImages(&plain)); } fn buildProviderPromptForCompactionWindow( @@ -6941,7 +6919,9 @@ fn processQueuedPromptLoop( agent: *runtime_agent.Agent, ) !void { var request_capabilities = initial_request_capabilities; - var tool_image_capabilities_resolved = false; + // The image route core's turn state. This loop is its shell: it runs each + // effect `image_route.step` returns and feeds the result back. + var image_route_state = image_route.step(.{}, .{ .turn_started = request_capabilities.image_input_support }).state; var stable_prefix = stable_prefix_ptr.*; defer stable_prefix_ptr.* = stable_prefix; const history_messages = history_messages_ptr.*; @@ -7077,7 +7057,6 @@ fn processQueuedPromptLoop( selected_fast_mode; var fast_unavailable_notified = false; var ultrafast_unconfirmed_notified = false; - var tool_image_strip_notified = false; // Attachment pixel sizes probed during this turn, so each step does not // reread every attachment snapshot. Entries live in the turn arena. var attachment_dimensions: image_attachments.AttachmentDimensionCache = .empty; @@ -7370,41 +7349,81 @@ fn processQueuedPromptLoop( gateway_messages = provider_prompt.messages; current_user_message_index = provider_prompt.current_user_index; debug_trace.eventf("agent", "before_provider_preflight", step_ctx, "model={s} messages={d}", .{ gateway_model, gateway_instructions.items.len + gateway_messages.items.len }); - const vision_policy = visionPolicy( - request_capabilities.image_input_support, - config.provider_capabilities.vision_fallback, - deps.tool_registry.lookup("vision") != null, - pending_image_ids.len > 0, - ); - const vision_route: runtime_vision_contracts.VisionRoute = if (vision_policy.route == .fallback) - .text_only - else - .native_images; - const vision_mode = vision_policy.mode; - debug_trace.eventf( - "agent", - "vision_policy", - step_ctx, - "model={s} image_support={s} route={s} mode={s} pending_images={d}", - .{ - gateway_model, - @tagName(request_capabilities.image_input_support), - @tagName(vision_policy.route), - @tagName(vision_mode), - pending_image_ids.len, - }, - ); try appendRecoveryConversationContext( overlay_arena, &gateway_messages, recovery_strategy, ); const recovery_source_messages = gateway_messages.items; + // One image decision per request, read from one capability + // snapshot: the core asks for capabilities first when it must, + // and every route, mode, and tool image choice below follows the + // decision it returns (`tests/tla/image_route/spec/ImageRoute.tla` + // OneSnapshot). + var image_event: image_route.Event = .{ .request_ready = .{ + .user_images = job.authorized_image_catalog.len > 0 or job.images.len > 0, + .pending_images = pending_image_ids.len > 0, + .tool_images = requestHasToolImages(recovery_source_messages), + .fallback = config.provider_capabilities.vision_fallback and deps.tool_registry.lookup("vision") != null, + .catalog_down = if (deps.model_catalog_unavailable) |unavailable| unavailable(deps.ctx) else false, + } }; + var image_decision: ?image_route.Decision = null; + var image_failure: ?image_route.FailReason = null; + var notify_catalog_down = false; + // Ends: the core asks for capabilities at most once per turn, and + // its reply never asks again (ImageRoute.tla ResolveOncePerTurn). + while (true) { + const image_out = image_route.step(image_route_state, image_event); + image_route_state = image_out.state; + var reply: ?image_route.Event = null; + for (image_out.effects()) |effect| switch (effect) { + .resolve_capabilities => { + request_capabilities = try deps.resolve_model_capabilities(deps.ctx, overlay_arena, gateway_model); + reply = .{ .capabilities_resolved = request_capabilities.image_input_support }; + }, + .notify_catalog_down => notify_catalog_down = true, + .send => |chosen| image_decision = chosen, + .fail => |reason| image_failure = reason, + }; + image_event = reply orelse break; + } + if (image_failure) |reason| switch (reason) { + .capability_unknown => { + diagnostics.recordModelCatalogEvent(true, .image_gate, "model={s} image_support=unknown err=ModelImageCapabilityUnavailable", .{gateway_model}); + return error.ModelImageCapabilityUnavailable; + }, + .no_native_image => { + diagnostics.recordModelCatalogEvent(true, .image_gate, "model={s} image_support=non_native err=SubscriptionNativeImageUnavailable", .{gateway_model}); + return error.SubscriptionNativeImageUnavailable; + }, + }; + // Each request ends in exactly one send or fail (ImageRoute.tla + // Answered), so a missing decision is a bug in the core. + const image_choice = image_decision.?; + const vision_route: runtime_vision_contracts.VisionRoute = if (image_choice.route == .fallback) + .text_only + else + .native_images; + const vision_mode = visionToolMode(image_choice.mode); + debug_trace.eventf( + "agent", + "vision_policy", + step_ctx, + "model={s} image_support={s} route={s} mode={s} pending_images={d} tool_images={s}", + .{ + gateway_model, + @tagName(image_route_state.support), + @tagName(image_choice.route), + @tagName(image_choice.mode), + pending_image_ids.len, + @tagName(image_choice.tool_images), + }, + ); const projected_request_messages = blk: { if (job.authorized_image_catalog.len == 0 and job.images.len == 0) { break :blk recovery_source_messages; } - break :blk switch (vision_policy.route) { + break :blk switch (image_choice.route) { .native => try runtime_vision_contracts.project_native_messages( overlay_arena, recovery_source_messages, @@ -7423,20 +7442,11 @@ fn processQueuedPromptLoop( job.authorized_image_catalog, ); }, - .unavailable => switch (request_capabilities.image_input_support) { - .unknown => { - diagnostics.recordModelCatalogEvent(true, .image_gate, "model={s} image_support=unknown err=ModelImageCapabilityUnavailable", .{gateway_model}); - return error.ModelImageCapabilityUnavailable; - }, - .non_native => { - diagnostics.recordModelCatalogEvent(true, .image_gate, "model={s} image_support=non_native err=SubscriptionNativeImageUnavailable", .{gateway_model}); - return error.SubscriptionNativeImageUnavailable; - }, - .native => unreachable, - }, + // The core fails a request whose user images have no route. + .unavailable => unreachable, }; }; - if (vision_policy.route == .native and projected_request_messages.len < recovery_source_messages.len) { + if (image_choice.route == .native and projected_request_messages.len < recovery_source_messages.len) { debug_trace.logf("history", "native_vision_history_projected removed_messages={d} reason=native_image_input", .{ recovery_source_messages.len - projected_request_messages.len, }); @@ -7483,16 +7493,7 @@ fn processQueuedPromptLoop( read_tool_result_request_eligible, subagent_request_messages, )); - if (!tool_image_capabilities_resolved and request_capabilities.image_input_support == .unknown) { - for (result_request_messages) |message| { - const memory = message.tool_result_memory orelse continue; - if (memory.tool_images.len == 0 and memory.tool_image_handle == null) continue; - request_capabilities = try deps.resolve_model_capabilities(deps.ctx, overlay_arena, gateway_model); - tool_image_capabilities_resolved = true; - break; - } - } - const materialized_messages = if (request_capabilities.image_input_support == .native) native: { + const materialized_messages = if (image_choice.route == .native) native: { const loaded_messages = try runtime_execution_memory.materializeToolImages(overlay_arena, config, result_request_messages); const max_dimension = image_data.requestMaxDimension(image_data.countRequestImages(loaded_messages)); const safe_tool_messages = try runtime_execution_memory.withholdRequestToolImages(overlay_arena, loaded_messages, max_dimension, config.max_tool_result_bytes); @@ -7502,31 +7503,26 @@ fn processQueuedPromptLoop( &attachment_dimensions, safe_tool_messages, max_dimension, - deps.image_blob_dir, ); break :native projection.messages; } else result_request_messages; - const image_projection = try runtime_gateway_step.projectToolImageMessages(overlay_arena, materialized_messages, request_capabilities.image_input_support, vision_policy.route == .fallback, config.max_tool_result_bytes); + const image_projection = try runtime_gateway_step.projectToolImageMessages(overlay_arena, materialized_messages, image_choice.tool_images, config.max_tool_result_bytes); // Saved turns keep steering without its envelope; the model gets it // back as the live update it saw when the turn took it. const steering_messages = try runtime_execution_memory.rewrapRestoredSteering(overlay_arena, image_projection.messages); // After the steering rewrap, which recognizes a parent's steering // by how its text starts. - const located_messages = try image_attachments.noteAttachmentLocations(overlay_arena, steering_messages, deps.image_blob_dir); + const located_messages = try image_attachments.noteAttachmentLocations(overlay_arena, steering_messages); const request_messages = try with_replyable_conversation_tail(overlay_arena, located_messages); if (request_messages.ptr != located_messages.ptr) { debug_trace.eventf("history", "assistant_tail_continued", step_ctx, "messages={d}", .{located_messages.len}); } // Tool images withheld because the catalog could not confirm image - // input are invisible in the transcript otherwise. Tell the user - // once per turn, and only when the catalog itself is known to be - // down; a confirmed model without image input relies on the - // model-facing notice instead. - if (image_projection.stripped and !tool_image_strip_notified and - request_capabilities.image_input_support == .unknown and - deps.model_catalog_unavailable != null and deps.model_catalog_unavailable.?(deps.ctx)) - { - tool_image_strip_notified = true; + // input are invisible in the transcript otherwise. The core asks + // to tell the user once per turn, and only when the catalog itself + // is known to be down; a confirmed model without image input + // relies on the model-facing notice instead. + if (notify_catalog_down) { try deps.push_text(deps.ctx, .{ .operational = "Images from tools aren't reaching the model right now because image support couldn't be confirmed (model catalog unavailable). fx will retry automatically as the catalog recovers." }); try deps.push_text(deps.ctx, .{ .operational = "\n" }); } @@ -7578,7 +7574,6 @@ fn processQueuedPromptLoop( request_data.messages, request_image_budget.largestRequestBytes(agent.requestByteLimit(gateway_model)), config.max_tool_result_bytes, - deps.image_blob_dir, ); if (past_limit.withheld > 0) { debug_trace.eventf("images", "request_images_withheld_before_build", step_ctx, "withheld={d}", .{past_limit.withheld}); @@ -7601,7 +7596,6 @@ fn processQueuedPromptLoop( deps.agent_stream_provider.max_request_bytes, ) }, config.max_tool_result_bytes, - deps.image_blob_dir, step_ctx, ); prepared_request_body = request_body; @@ -8441,7 +8435,7 @@ fn processQueuedPromptLoop( if (shouldRecoverContextOverflow( failure, has_compactable_context, - streamReplaySafe(&stream_ctx), + attemptReplaySafe(&stream_ctx), context_overflow_recovery == .ready, config.cancel_flag.load(.seq_cst), )) { @@ -9772,6 +9766,9 @@ fn processQueuedPromptLoop( stop_state.dispatched = true; switch (stop_outcome) { .allow => { + if (agent_steps.allowsStep(config.agent_step_limit, step + 1) and + try continue_steering_after_stop(deps, arena, &within_turn_suffix, turn_id, config.origin, stop_state)) + continue :agent_steps_loop; stop_state.terminal_materializing = true; try finishCommonAssistantTerminal( deps, @@ -12573,6 +12570,17 @@ fn processQueuedPromptLoop( }, ) catch |err| switch (err) { error.Cancelled => { + if (try append_immediate_steering_after_cancel( + deps, + arena, + &within_turn_suffix, + turn_id, + "", + )) { + stop_state.retained_candidate = null; + stop_state.latest_partial = null; + continue :agent_steps_loop; + } runtime_telemetry.traceCancelObserved(step_ctx, false); try runtime_interruption.persistInterruptedTurnOnce( deps, @@ -12596,6 +12604,9 @@ fn processQueuedPromptLoop( stop_state.dispatched = true; switch (stop_outcome) { .allow => { + if (agent_steps.allowsStep(config.agent_step_limit, step + 1) and + try continue_steering_after_stop(deps, arena, &within_turn_suffix, turn_id, config.origin, stop_state)) + continue :agent_steps_loop; stop_state.terminal_materializing = true; try finishCommonAssistantTerminal( deps, diff --git a/src/core/agent/runtime/request_image_budget.zig b/src/core/agent/runtime/request_image_budget.zig index 84878269d..9fdfb9321 100644 --- a/src/core/agent/runtime/request_image_budget.zig +++ b/src/core/agent/runtime/request_image_budget.zig @@ -55,18 +55,15 @@ pub const Projection = struct { /// `request_bytes` until it fits `limits`. Returned messages are `messages` /// itself when nothing is withheld, otherwise a copy allocated in `arena`. /// `tool_text_limit` bounds tool result content after the note is added. -/// A note names an attached image's saved copy (`savedImagePath` with -/// `blob_dir`), or without a saved session the copy fx captured. pub fn withholdOldestImages( arena: Allocator, messages: []const ChatMessage, request_bytes: usize, limits: Limits, tool_text_limit: usize, - blob_dir: ?[]const u8, ) Allocator.Error!Projection { const sizes = try imageSizes(arena, messages); - return withhold(arena, messages, oldestImagesToWithhold(sizes, request_bytes, limits), tool_text_limit, blob_dir); + return withhold(arena, messages, oldestImagesToWithhold(sizes, request_bytes, limits), tool_text_limit); } /// A size no request is fit below: the default, or a larger limit learned @@ -86,13 +83,12 @@ pub fn withholdImagesPastLimit( messages: []const ChatMessage, max_request_bytes: usize, tool_text_limit: usize, - blob_dir: ?[]const u8, ) Allocator.Error!Projection { const sizes = try imageSizes(arena, messages); var images_bytes: usize = 0; for (sizes) |size| images_bytes +|= size; const withheld = oldestImagesToWithhold(sizes, images_bytes, .{ .max_request_bytes = max_request_bytes }); - return withhold(arena, messages, withheld, tool_text_limit, blob_dir); + return withhold(arena, messages, withheld, tool_text_limit); } /// Each image's serialized size, oldest first, allocated in `arena`. @@ -112,7 +108,7 @@ fn imageSizes(arena: Allocator, messages: []const ChatMessage) Allocator.Error![ } /// `messages` with its `withheld` oldest images replaced by notes. -fn withhold(arena: Allocator, messages: []const ChatMessage, withheld: usize, tool_text_limit: usize, blob_dir: ?[]const u8) Allocator.Error!Projection { +fn withhold(arena: Allocator, messages: []const ChatMessage, withheld: usize, tool_text_limit: usize) Allocator.Error!Projection { if (withheld == 0) return .{ .messages = messages }; const projected = try arena.dupe(ChatMessage, messages); @@ -122,10 +118,7 @@ fn withhold(arena: Allocator, messages: []const ChatMessage, withheld: usize, to if (message.images.len > 0) { const count = @min(remaining, message.images.len); var notice: std.Io.Writer.Allocating = .init(arena); - for (message.images[0..count]) |image| { - const saved = (try image_attachments.savedImagePath(arena, image, blob_dir)) orelse image.snapshot_path; - writeAttachmentNotice(¬ice.writer, image, saved) catch return error.OutOfMemory; - } + for (message.images[0..count]) |image| writeAttachmentNotice(¬ice.writer, image) catch return error.OutOfMemory; message.images = message.images[count..]; message.content = try std.mem.concat(arena, u8, &.{ notice.written(), message.content orelse "" }); remaining -= count; @@ -143,9 +136,9 @@ fn withhold(arena: Allocator, messages: []const ChatMessage, withheld: usize, to return .{ .messages = projected, .withheld = withheld }; } -fn writeAttachmentNotice(writer: *std.Io.Writer, image: types.ImageAttachment, saved: ?[]const u8) std.Io.Writer.Error!void { +fn writeAttachmentNotice(writer: *std.Io.Writer, image: types.ImageAttachment) std.Io.Writer.Error!void { try writer.print("[Image #{d} not sent to keep the request under the provider's size limit.", .{image.id}); - if (saved) |path| try writer.print(" The original is saved at {s}; read_file it to see it again.", .{path}); + if (image_attachments.savedImagePath(image)) |path| try writer.print(" The original is saved at {s}; read_file it to see it again.", .{path}); try writer.writeAll("]\n"); } @@ -216,7 +209,7 @@ test "withholding keeps history and names how to reload each image" { }; // 12,500 bytes against 9,000: the attachment (4,000 encoded) is enough. - const first = try withholdOldestImages(arena, &messages, 12_500, .{ .max_request_bytes = 9000 }, 4096, null); + const first = try withholdOldestImages(arena, &messages, 12_500, .{ .max_request_bytes = 9000 }, 4096); try std.testing.expectEqual(@as(usize, 1), first.withheld); try std.testing.expectEqual(@as(usize, 0), first.messages[0].images.len); try std.testing.expect(std.mem.startsWith(u8, first.messages[0].content.?, "[Image #1 not sent to keep the request under the provider's size limit.]\n")); @@ -224,7 +217,7 @@ test "withholding keeps history and names how to reload each image" { try std.testing.expectEqual(@as(usize, 2), first.messages[1].tool_result_memory.?.tool_images.len); // Against 5,000 every image leaves, oldest first. - const second = try withholdOldestImages(arena, &messages, 12_500, .{ .max_request_bytes = 5000 }, 4096, null); + const second = try withholdOldestImages(arena, &messages, 12_500, .{ .max_request_bytes = 5000 }, 4096); try std.testing.expectEqual(@as(usize, 3), second.withheld); try std.testing.expectEqual(@as(usize, 0), second.messages[1].tool_result_memory.?.tool_images.len); try std.testing.expectEqualStrings( @@ -238,7 +231,7 @@ test "withholding keeps history and names how to reload each image" { try std.testing.expectEqualStrings("compare these", messages[0].content.?); // A request that already fits is returned as is. - const unchanged = try withholdOldestImages(arena, &messages, 9000, .{ .max_request_bytes = 9000 }, 4096, null); + const unchanged = try withholdOldestImages(arena, &messages, 9000, .{ .max_request_bytes = 9000 }, 4096); try std.testing.expectEqual(@as(usize, 0), unchanged.withheld); try std.testing.expectEqual(@as([*]const ChatMessage, &messages), unchanged.messages.ptr); } @@ -256,7 +249,7 @@ test "images no request can carry leave before it is built, and none it can" { // 24,000 bytes of images against 10,000: the oldest four leave, whatever // the text adds, and the newest two stay for the exact fit. - const before = try withholdImagesPastLimit(arena, &messages, 10_000, 4096, null); + const before = try withholdImagesPastLimit(arena, &messages, 10_000, 4096); try std.testing.expectEqual(@as(usize, 4), before.withheld); try std.testing.expectEqual(@as(usize, 0), before.messages[0].tool_result_memory.?.tool_images.len); try std.testing.expectEqual(@as(usize, 2), before.messages[1].tool_result_memory.?.tool_images.len); @@ -267,7 +260,7 @@ test "images no request can carry leave before it is built, and none it can" { } // Images that fit by size stay for the exact fit to decide. - const fits = try withholdImagesPastLimit(arena, &messages, 30_000, 4096, null); + const fits = try withholdImagesPastLimit(arena, &messages, 30_000, 4096); try std.testing.expectEqual(@as(usize, 0), fits.withheld); try std.testing.expectEqual(@as([*]const ChatMessage, &messages), fits.messages.ptr); @@ -293,28 +286,30 @@ test "withheld snapshot attachments point at their saved file" { .path = try arena.dupe(u8, "screen.png"), .media_type = try arena.dupe(u8, "image/png"), .snapshot_path = try arena.dupe(u8, "/home/u/.fx/sessions/s/images/3.bin"), - }, "/home/u/.fx/sessions/s/images/3.bin"); + }); try std.testing.expectEqualStrings( "[Image #3 not sent to keep the request under the provider's size limit. The original is saved at /home/u/.fx/sessions/s/images/3.bin; read_file it to see it again.]\n", notice.written(), ); } -test "a prompt image left out for size names the saved copy every request names (D79)" { +test "a prompt image left out for size names its session's blob (D89)" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var image = try image_attachments.captureInlineImageBytesInMemory(arena, 1, "image/png", &image_data.testPngHeader(10, 10)); image.snapshot_path = try arena.dupe(u8, "/tmp/fx-image-snapshots-0123456789abcdef/image-1.bin"); - const messages = [_]ChatMessage{.{ .role = .user, .content = "look", .images = &.{image} }}; - const blob_dir = "/Users/example/.fx/sessions/v2/s/blobs"; - const saved = try withholdImagesPastLimit(arena, &messages, 1, 4096, blob_dir); - try std.testing.expectEqual(@as(usize, 1), saved.withheld); - const blob = try std.Io.Dir.path.join(arena, &.{ blob_dir, image.snapshot_sha256.? }); - try std.testing.expect(std.mem.find(u8, saved.messages[0].content.?, try arena.print("The original is saved at {s};", .{blob})) != null); - // Without a saved session it names the copy fx captured for the run. - const unsaved = try withholdImagesPastLimit(arena, &messages, 1, 4096, null); - try std.testing.expect(std.mem.find(u8, unsaved.messages[0].content.?, "The original is saved at /tmp/fx-image-snapshots-0123456789abcdef/image-1.bin;") != null); + image.blob_path = try arena.dupe(u8, "/Users/example/.fx/sessions/v2/s/blobs/0123abcd"); + const in_blob = [_]ChatMessage{.{ .role = .user, .content = "look", .images = &.{image} }}; + const named = try withholdImagesPastLimit(arena, &in_blob, 1, 4096); + try std.testing.expectEqual(@as(usize, 1), named.withheld); + try std.testing.expect(std.mem.find(u8, named.messages[0].content.?, "The original is saved at /Users/example/.fx/sessions/v2/s/blobs/0123abcd;") != null); + // Inline bytes without a blob have no saved file to name. + var unsaved = image; + unsaved.blob_path = null; + const inline_only = [_]ChatMessage{.{ .role = .user, .content = "look", .images = &.{unsaved} }}; + const unnamed = try withholdImagesPastLimit(arena, &inline_only, 1, 4096); + try std.testing.expect(std.mem.find(u8, unnamed.messages[0].content.?, "saved at") == null); } test "a withheld tool image without a saved handle says so plainly" { diff --git a/src/core/agent/runtime/tests/finalization_flow.zig b/src/core/agent/runtime/tests/finalization_flow.zig index e6b0b4173..ea66758f3 100644 --- a/src/core/agent/runtime/tests/finalization_flow.zig +++ b/src/core/agent/runtime/tests/finalization_flow.zig @@ -6,6 +6,7 @@ const io_mod = @import("../../../shared/io.zig"); const tool_result_errors = @import("../../../tooling/tool_result_errors.zig"); const lifecycle_hooks = @import("../../../hooks/hooks.zig"); const session_runtime = @import("../../../session/session.zig"); +const worker_runtime = @import("../../worker_runtime.zig"); const test_support = @import("support.zig"); const runtime_finalization = @import("../finalization.zig"); @@ -1335,6 +1336,115 @@ test "common Stop handler errors fail open" { } } +/// A prompt the worker owns, allocated the way its queue frees it. +fn ownedWorkerPrompt(alloc: std.mem.Allocator, text: []const u8) !worker_runtime.QueuedPrompt { + return .{ + .prompt = try alloc.dupe(u8, text), + .images = &.{}, + .model = try alloc.dupe(u8, "provider/model"), + .api_key = try alloc.dupe(u8, "key"), + .permission_mode = .auto, + .history = try alloc.alloc(types.HistoryTurn, 0), + .grants = try alloc.alloc(types.PermissionGrant, 0), + }; +} + +/// A Stop hook during whose first run the user submits a steer, admitted the +/// way the terminal admits a prompt typed while a turn runs. +const SteerDuringStop = struct { + alloc: std.mem.Allocator, + worker: *worker_runtime.WorkerRuntime, + calls: usize = 0, + + fn run( + raw: *anyopaque, + _: lifecycle_hooks.StopInput, + ) lifecycle_hooks.HandlerError!lifecycle_hooks.StopAction { + const self: *SteerDuringStop = @ptrCast(@alignCast(raw)); + self.calls += 1; + if (self.calls == 1) { + const steer = ownedWorkerPrompt(self.alloc, "also check the docs") catch return error.Failed; + self.worker.admitInteractivePrompt(self.alloc, steer) catch { + worker_runtime.freeQueuedPrompt(self.alloc, steer); + return error.Failed; + }; + } + return .allow; + } +}; + +test "common Stop steer submitted during the hook joins the turn for every final reply" { + const alloc = std.testing.allocator; + const provider_calls = [_]ToolCall{.{ + .id = "call_search", + .name = "perplexity_search", + .arguments_json = "{}", + .provider_result = "search result", + .provenance = .provider_executed, + }}; + const Case = struct { name: []const u8, first: FakeCompletion }; + const cases = [_]Case{ + .{ .name = "plain reply", .first = .{ .content = "First answer" } }, + .{ .name = "provider tool reply", .first = .{ + .content = "First answer", + .tool_calls = &provider_calls, + .finish_reason = .stop, + } }, + .{ .name = "provider tool reply with a streamed start", .first = .{ + .content = "First answer", + .tool_calls = &provider_calls, + .streamed_tool_starts = &provider_calls, + .finish_reason = .stop, + } }, + }; + for (cases) |case| { + errdefer std.debug.print("case: {s}\n", .{case.name}); + // The turn starts the way the worker loop starts one, and the agent + // takes its steering, cancel flag, and events from that worker. + var worker = worker_runtime.WorkerRuntime{}; + defer worker.deinit(alloc); + try worker.admitInteractivePrompt(alloc, try ownedWorkerPrompt(alloc, "user prompt")); + const work = (try worker.tryTakeNextWork(alloc)) orelse return error.TestExpectedQueuedWork; + defer worker_runtime.freeWorkItem(alloc, work); + + var gateway = FakeGateway.init(alloc, &.{ case.first, .{ .content = "Updated answer" } }); + defer gateway.deinit(); + var deps = FakeAgentRuntimeDeps.init(alloc); + defer deps.deinit(); + deps.steering_worker = &worker; + var fixture = PromptFixture{}; + var config = fixture.config(); + config.cancel_flag = &worker.worker_cancel_requested; + var job = fixture.job(); + job.turn_id = work.prompt.turn_id; + var lifecycle_runtime = lifecycle_hooks.Runtime.init(alloc); + defer lifecycle_runtime.deinit(); + var handler = SteerDuringStop{ .alloc = alloc, .worker = &worker }; + try lifecycle_runtime.registerStop(.{ .name = "steer-during-stop", .ctx = &handler, .run = SteerDuringStop.run }); + + try runFakePromptWithLifecycle( + &gateway, + &deps, + config, + job, + testLifecycleContext(lifecycle_runtime.freeze(), alloc, fixture.workspace_root), + ); + worker.finishProcessing(); + + // The steer reaches the turn's next request, and the turn ends once, + // with the updated reply. + try std.testing.expectEqual(@as(usize, 2), handler.calls); + try std.testing.expectEqual(@as(usize, 2), gateway.request_bodies.items.len); + try expectBodyContainsInOrder(&gateway, 1, &.{ "First answer", "user_steering", "also check the docs" }); + try std.testing.expectEqual(@as(usize, 0), deps.interrupted_history_count); + try std.testing.expectEqual(@as(usize, 1), deps.history_turns.items.len); + try std.testing.expect(deps.history_turns.items[0] == .assistant); + try std.testing.expectEqual(@as(usize, 1), deps.history_turns.items[0].assistant.execution.steering.len); + // Nothing is left over to run as a later turn. + try std.testing.expectEqual(@as(usize, 0), worker.queuedPromptCount()); + } +} + test "common Stop does not run for silent-tool auto-continuation" { const alloc = std.testing.allocator; const first_calls = [_]ToolCall{ diff --git a/src/core/agent/runtime/tests/gateway_flow.zig b/src/core/agent/runtime/tests/gateway_flow.zig index 6f23a58e4..b776f5ef2 100644 --- a/src/core/agent/runtime/tests/gateway_flow.zig +++ b/src/core/agent/runtime/tests/gateway_flow.zig @@ -2505,7 +2505,7 @@ test "processQueuedPrompt never uses the vision fallback for Codex" { try std.testing.expectEqual(@as(usize, 0), gateway.request_bodies.items.len); } -test "processQueuedPrompt routes images natively only when vision and file input are both supported" { +test "processQueuedPrompt routes images natively whenever the catalog lists vision" { const alloc = std.testing.allocator; const cases = [_]struct { label: []const u8, @@ -2514,7 +2514,7 @@ test "processQueuedPrompt routes images natively only when vision and file input expect_native: bool, }{ .{ .label = "vision-file", .supports_vision = true, .supports_file_input = true, .expect_native = true }, - .{ .label = "vision-no-file", .supports_vision = true, .supports_file_input = false, .expect_native = false }, + .{ .label = "vision-no-file", .supports_vision = true, .supports_file_input = false, .expect_native = true }, .{ .label = "file-no-vision", .supports_vision = false, .supports_file_input = true, .expect_native = false }, .{ .label = "neither", .supports_vision = false, .supports_file_input = false, .expect_native = false }, }; @@ -2532,7 +2532,10 @@ test "processQueuedPrompt routes images natively only when vision and file input const capability_overrides = [_]ModelCapabilityOverride{.{ .model = model, .capabilities = .{ - .image_input_support = if (entry.expect_native) .native else .non_native, + .image_input_support = model_capabilities.mergeCapabilities(.{}, .{ + .supports_vision = entry.supports_vision, + .supports_file_input = entry.supports_file_input, + }).image_input_support, .supports_vision = entry.supports_vision, .supports_file_input = entry.supports_file_input, }, diff --git a/src/core/agent/runtime/tests/support.zig b/src/core/agent/runtime/tests/support.zig index 98cf505e7..b5800e810 100644 --- a/src/core/agent/runtime/tests/support.zig +++ b/src/core/agent/runtime/tests/support.zig @@ -699,6 +699,10 @@ pub const FakeAgentRuntimeDeps = struct { immediate_steering_messages: []const []const u8 = &.{}, immediate_steering_cancel_flag: ?*std.atomic.Value(bool) = null, immediate_steering_take_count: usize = 0, + /// A real worker wired as the app wires it: the turn takes its steering + /// from it and publishes its events and tool lifecycle to it. Its queue, + /// events, and taken steering use `alloc`. + steering_worker: ?*worker_runtime.WorkerRuntime = null, pub fn init(alloc: Allocator) FakeAgentRuntimeDeps { return .{ .alloc = alloc }; @@ -824,7 +828,9 @@ pub const FakeAgentRuntimeDeps = struct { .available_model_capabilities = availableModelCapabilities, .resolve_model_capabilities = resolveModelCapabilities, .model_catalog_unavailable = catalogUnavailable, - .take_steering_boundary = if (self.steering_messages.len > 0 or + .take_steering_boundary = if (self.steering_worker != null) + takeWorkerSteeringBoundary + else if (self.steering_messages.len > 0 or self.immediate_steering_messages.len > 0) takeSteeringBoundary else @@ -950,6 +956,32 @@ pub const FakeAgentRuntimeDeps = struct { return .{ .continue_turn = messages }; } + /// Takes steering from `steering_worker` and copies it into the turn's + /// arena, as the app's callback does. + fn takeWorkerSteeringBoundary( + raw: *anyopaque, + arena: Allocator, + turn_id: u64, + kind: worker_runtime.SteeringBoundaryKind, + ) !worker_runtime.SteeringBoundaryResult { + const self: *FakeAgentRuntimeDeps = @ptrCast(@alignCast(raw)); + const result = try self.steering_worker.?.takeSteeringBoundary(self.alloc, turn_id, kind); + return switch (result) { + .continue_turn => |owned| blk: { + defer { + for (owned) |text| self.alloc.free(text); + self.alloc.free(owned); + } + const messages = try arena.alloc([]u8, owned.len); + for (owned, messages) |text, *copy| copy.* = try arena.dupe(u8, text); + break :blk .{ .continue_turn = messages }; + }, + .none => .none, + .handoff => .handoff, + .interrupt => .interrupt, + }; + } + fn requestRouteRecovery(raw: *anyopaque, _: Allocator, request: runtime_deps.RouteRecoveryRequest) !runtime_deps.RouteRecoveryDecision { const self: *FakeAgentRuntimeDeps = @ptrCast(@alignCast(raw)); self.route_recovery_count += 1; @@ -1676,6 +1708,7 @@ pub const FakeAgentRuntimeDeps = struct { fn pushEvent(raw: *anyopaque, event: WorkerEvent) !void { const self: *FakeAgentRuntimeDeps = @ptrCast(@alignCast(raw)); defer worker_runtime.freeWorkerEvent(std.heap.c_allocator, event); + if (self.steering_worker) |worker| try worker.pushEvent(self.alloc, event); switch (event) { .full_detail_record => |notice| { const owned = try types.dupeSemanticNotice(self.alloc, notice); @@ -1764,6 +1797,7 @@ pub const FakeAgentRuntimeDeps = struct { event: types.ToolLifecycleEvent, ) !void { const self: *FakeAgentRuntimeDeps = @ptrCast(@alignCast(raw)); + if (self.steering_worker) |worker| try worker.pushEvent(self.alloc, .{ .tool_lifecycle = event }); switch (event) { .provisional, .authoritative_started => try self.record("status:started", .{}), .terminal => |terminal| { diff --git a/src/core/app/app_callbacks.zig b/src/core/app/app_callbacks.zig index 384f6ef37..b745e49b9 100644 --- a/src/core/app/app_callbacks.zig +++ b/src/core/app/app_callbacks.zig @@ -336,8 +336,8 @@ pub fn Bindings(comptime App: type) type { if (app.session_persistence.v2 != null) agentAppendTurnPiece else null else null, - .image_blob_dir = if (comptime @hasField(App, "session_persistence")) - if (app.session_persistence.v2) |v2| v2.blobsPath() else null + .save_prompt_images = if (comptime @hasField(App, "session_persistence")) + if (app.session_persistence.v2 != null) agentSavePromptImages else null else null, .propagate_grant = agentPropagateGrant, @@ -1193,6 +1193,11 @@ pub fn Bindings(comptime App: type) type { try app_session_runtime.Runtime(App).appendTurnPiece(app, progress); } + fn agentSavePromptImages(ctx: *anyopaque, alloc: std.mem.Allocator, images: []types.ImageAttachment) anyerror!void { + const app: *App = @ptrCast(@alignCast(ctx)); + try app_session_runtime.Runtime(App).savePromptImages(app, alloc, images); + } + fn agentPropagateGrant(ctx: *anyopaque, tool_name: []const u8, target_path: []const u8) !void { const app: *App = @ptrCast(@alignCast(ctx)); try app_worker_runtime.Runtime(App).propagateGrant(app, tool_name, target_path); diff --git a/src/core/app/app_input_runtime.zig b/src/core/app/app_input_runtime.zig index 0b461fd7e..a084ce622 100644 --- a/src/core/app/app_input_runtime.zig +++ b/src/core/app/app_input_runtime.zig @@ -3040,7 +3040,6 @@ const RoutingSubagents = struct { switch (self.manager_paste.paste.settleDeliveryEpoch()) { .none => return false, .finish => self.manager_paste.paste.resetWithTrace(.decision_prompt_active), - .reject => self.manager_paste.paste.resetWithTrace(.unsafe_suffix), } return true; } @@ -9314,7 +9313,7 @@ test "app_input_runtime rejects same-epoch paste suffix without submitting it" { try std.testing.expectEqualStrings("keep", app.submitted_prompt[0..app.submitted_prompt_len]); } -test "app_input_runtime rejects a same-epoch enter after an otherwise valid paste" { +test "app_input_runtime keeps a paste but not a same-epoch enter after it" { const alloc = std.testing.allocator; var app = try RoutingFakeApp.init(alloc); defer app.deinit(); @@ -9322,7 +9321,7 @@ test "app_input_runtime rejects a same-epoch enter after an otherwise valid past try feedRoutingBytes(&app, "\x1b[200~new\x1b[201~\r"); - try std.testing.expectEqualStrings("prior", app.input_runtime.edit_state.input.items); + try std.testing.expectEqualStrings("priornew", app.input_runtime.edit_state.input.items); try std.testing.expectEqual(@as(usize, 0), app.submitted_prompt_count); try std.testing.expectEqual(@as(usize, 1), app.notice_write_count); } @@ -9580,7 +9579,7 @@ test "app_input_runtime question freeform accepts bracketed paste atomically" { try std.testing.expect(app.shell.render_requests.hasReason(.modal)); } -test "app_input_runtime rejects unsafe suffixes for every root modal paste owner" { +test "app_input_runtime keeps a modal paste but never runs its same-epoch suffix" { const alloc = std.testing.allocator; { @@ -9607,7 +9606,7 @@ test "app_input_runtime rejects unsafe suffixes for every root modal paste owner try feedRoutingBytes(&app, "\x1b[200~new\x1b[201~\r"); - try std.testing.expectEqualStrings("keep", app.approval_prompt.decision.selectedDraft()); + try std.testing.expectEqualStrings("keepnew", app.approval_prompt.decision.selectedDraft()); try std.testing.expect(app.worker.submitted_permission == null); try std.testing.expectEqual(@as(usize, 1), app.notice_write_count); } @@ -9629,7 +9628,7 @@ test "app_input_runtime rejects unsafe suffixes for every root modal paste owner try feedRoutingBytes(&app, "\x1b[200~new\x1b[201~\r"); try std.testing.expectEqualStrings( - "keep", + "keepnew", app.question_prompt.entries.items[0].freeform_buffer.items, ); try std.testing.expect(app.question_prompt.isActive()); diff --git a/src/core/app/app_session_runtime.zig b/src/core/app/app_session_runtime.zig index 6b0e5ce5c..e8a988383 100644 --- a/src/core/app/app_session_runtime.zig +++ b/src/core/app/app_session_runtime.zig @@ -2825,6 +2825,15 @@ pub fn Runtime(comptime App: type) type { try v2.appendProgress(progress.user, progress.execution, .{ .calls = progress.running_calls, .assistant = progress.running_assistant }); } + /// `AgentRuntimeDeps.save_prompt_images` on v2: the prompt's images + /// become the session's blobs before its first request (D89). + pub fn savePromptImages(app: *App, alloc: std.mem.Allocator, images: []types.ImageAttachment) !void { + app.session_persistence.write_mutex.lockUncancelable(io_mod.getIo()); + defer app.session_persistence.write_mutex.unlock(io_mod.getIo()); + const v2 = app.session_persistence.v2 orelse return; + try v2.savePromptImages(alloc, images); + } + pub fn commitRuntimePreferences( app: *App, patch: SessionPreferencePatch, diff --git a/src/core/app/input_paste_runtime.zig b/src/core/app/input_paste_runtime.zig index b9ffa83b0..36bedd343 100644 --- a/src/core/app/input_paste_runtime.zig +++ b/src/core/app/input_paste_runtime.zig @@ -166,31 +166,33 @@ pub fn PasteEditRuntime(comptime App: type) type { } pub fn settlePasteDeliveryEpoch(app: *App, max_input_len: usize) !bool { - switch (app.input_runtime.paste.settleDeliveryEpoch()) { + const finish = switch (app.input_runtime.paste.settleDeliveryEpoch()) { .none => return false, - .finish => { - debug_trace.logf( - "input", - "paste end owner={s} bytes={d}", - .{ @tagName(app.input_runtime.paste.owner), app.input_runtime.paste.byteCount() }, - ); - try finishPaste(app, max_input_len); - }, - .reject => { - const owner = app.input_runtime.paste.owner; - const render_reason: render_request.Reason = switch (owner) { - .composer => .footer, - .decision_prompt, .question_freeform, .approval_amendment => .modal, - .none, .auth_code => .footer, - }; - app.input_runtime.paste.resetWithTrace(.unsafe_suffix); - app.shell.render_requests.request(render_reason); - try app.writeDomainNotice(.{ - .topic = "input", - .tone = .@"error", - .body = "Paste was not applied because extra input followed its end marker.", - }, true); - }, + .finish => |finish| finish, + }; + const owner = app.input_runtime.paste.owner; + debug_trace.logf( + "input", + "paste end owner={s} bytes={d}", + .{ @tagName(owner), app.input_runtime.paste.byteCount() }, + ); + // The paste is kept, but input that followed its end marker in the + // same delivery never runs: it may be clipboard content after an + // end marker embedded in the paste. + if (finish.dropped_suffix_bytes > 0) { + debug_trace.logf( + "input", + "paste suffix dropped owner={s} bytes={d} reason=same_delivery_as_end_marker", + .{ @tagName(owner), finish.dropped_suffix_bytes }, + ); + } + try finishPaste(app, max_input_len); + if (finish.dropped_suffix_bytes > 0) { + try app.writeDomainNotice(.{ + .topic = "input", + .tone = .warning, + .body = "Input that arrived right after the paste was ignored.", + }, true); } return true; } diff --git a/src/core/cli/cli_ask.zig b/src/core/cli/cli_ask.zig index fd7c2de84..e3dd404e8 100644 --- a/src/core/cli/cli_ask.zig +++ b/src/core/cli/cli_ask.zig @@ -1984,7 +1984,7 @@ fn agentRuntimeDeps(ctx: *AskContext) agent_runtime.AgentRuntimeDeps { .propagate_history_turn = propagateHistoryTurn, .commit_context_compaction = .{ .commit = commitContextCompaction }, .append_turn_piece = if (ctx.v2 != null) appendTurnPiece else null, - .image_blob_dir = if (ctx.v2) |v2| v2.blobsPath() else null, + .save_prompt_images = if (ctx.v2 != null) savePromptImages else null, .propagate_grant = propagateGrant, .push_event = pushEvent, .push_text = pushText, @@ -2917,6 +2917,19 @@ fn appendTurnPiece(raw_ctx: *anyopaque, progress: agent_runtime.TurnProgress) !v ctx.prompt_snapshot_committed = true; } +/// The prompt's images become the session's blobs before its first request +/// (D89). A failure leaves the rest inline, with no path, as the adapter +/// does for one image the store refuses. +fn savePromptImages(raw_ctx: *anyopaque, alloc: std.mem.Allocator, images: []types.ImageAttachment) !void { + const ctx: *AskContext = @ptrCast(@alignCast(raw_ctx)); + ctx.session_write_mutex.lockUncancelable(io_mod.getIo()); + defer ctx.session_write_mutex.unlock(io_mod.getIo()); + const v2 = ctx.v2 orelse return; + v2.savePromptImages(alloc, images) catch |err| { + debug_trace.logf("session", "event=sessions_v2_prompt_images_inline session={s} err={s}", .{ v2.id(), @errorName(err) }); + }; +} + fn propagateGrant(_: *anyopaque, _: []const u8, _: []const u8) !void {} fn pushEvent(raw_ctx: *anyopaque, event: WorkerEvent) !void { diff --git a/src/core/config/model_capabilities.zig b/src/core/config/model_capabilities.zig index 329f438b6..ac9c47725 100644 --- a/src/core/config/model_capabilities.zig +++ b/src/core/config/model_capabilities.zig @@ -1,6 +1,7 @@ const std = @import("std"); const types = @import("../shared/types.zig"); const model_provider = @import("model_provider.zig"); +const image_route = @import("../images/image_route.zig"); pub const ResolvedProviderOptions = struct { reasoning: ?types.ReasoningEffort = null, @@ -33,11 +34,7 @@ pub const ReasoningEffortOptions = struct { } }; -pub const ImageInputSupport = enum { - unknown, - non_native, - native, -}; +pub const ImageInputSupport = image_route.Support; pub const GatewayMetadata = struct { supports_reasoning: bool = false, @@ -97,10 +94,10 @@ pub fn mergeCapabilities(capabilities_value: Capabilities, gateway_metadata: ?Ga capabilities.supports_tool_use = metadata.supports_tool_use; capabilities.supports_vision = metadata.supports_vision; capabilities.supports_file_input = metadata.supports_file_input; - capabilities.image_input_support = if (metadata.supports_vision and metadata.supports_file_input) - .native - else - .non_native; + capabilities.image_input_support = image_route.supportFromCatalog(.{ .listed = .{ + .vision = metadata.supports_vision, + .file_input = metadata.supports_file_input, + } }); capabilities.supports_web_search = metadata.supports_web_search; capabilities.supports_explicit_caching = metadata.supports_explicit_caching; capabilities.supports_implicit_caching = metadata.supports_implicit_caching; @@ -302,17 +299,19 @@ test "image input support distinguishes unknown native and non native capability capabilitiesForModel("provider/unknown").image_input_support, ); - const native = mergeCapabilities(.{}, .{ - .supports_vision = true, - .supports_file_input = true, - }); - try std.testing.expectEqual(ImageInputSupport.native, native.image_input_support); + for ([_]bool{ false, true }) |file_input| { + const native = mergeCapabilities(.{}, .{ + .supports_vision = true, + .supports_file_input = file_input, + }); + try std.testing.expectEqual(ImageInputSupport.native, native.image_input_support); - const non_native = mergeCapabilities(.{}, .{ - .supports_vision = true, - .supports_file_input = false, - }); - try std.testing.expectEqual(ImageInputSupport.non_native, non_native.image_input_support); + const non_native = mergeCapabilities(.{}, .{ + .supports_vision = false, + .supports_file_input = file_input, + }); + try std.testing.expectEqual(ImageInputSupport.non_native, non_native.image_input_support); + } } test "reasoning effort picker helpers prepend default and preserve Gateway order" { diff --git a/src/core/execution/command_runner.zig b/src/core/execution/command_runner.zig index 0e4fe7374..8b20afb18 100644 --- a/src/core/execution/command_runner.zig +++ b/src/core/execution/command_runner.zig @@ -832,7 +832,8 @@ fn executeWithSnapshot( &nonce, }); const invocation = try shell_resolver.snapshotInvocation(scratch, generation.shell_path, failure_marker); - const script = try shell_resolver.snapshotScript(scratch, generation.replay, command); + const kind = shell_resolver.shellKind(generation.shell_path) orelse return error.UnsupportedShell; + const script = try shell_resolver.snapshotScript(scratch, kind, generation.replay, command); debug_trace.logf( "core", "command runner snapshot generation={d} shell={s} replay_bytes={d}", @@ -2345,6 +2346,42 @@ test "user profile command reruns once with full startup after a failed replay" try std.testing.expect(std.mem.find(u8, notice, "could not be restored") != null); } +test "zsh commands keep words that start with = literal in every captured profile" { + if (comptime !shell_snapshot.isSupported() or !supports_foreground_session) return; + std.Io.Dir.accessAbsolute(io_mod.getIo(), "/bin/zsh", .{}) catch + return error.SkipZigTest; + shell_snapshot.resetProcessOwnerForTest(shell_snapshot.real_capture_fn); + defer shell_snapshot.resetProcessOwnerForTest(shell_snapshot.real_capture_fn); + + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // Startup files that turn the option on explicitly do not reach the command. + const paths = try writeSnapshotTestShell( + arena, + &tmp, + "print -r -- loaded >> \"$HOME/rc-loads.log\"\nsetopt equals\n", + ); + const loads_path = try std.Io.Dir.path.join(arena, &.{ paths.home, "rc-loads.log" }); + const command = "echo before; echo ======; [ x == x ] && echo same; echo after"; + const expected = "exit_code=0\n\nbefore\n======\nsame\nafter\n\n"; + const config = Config{ .max_command_output_bytes = 16 * 1024 }; + + const clean = try executeCommandInEnvironment(config, arena, command, paths.workspace, .{ .clean = "/bin/zsh" }); + try std.testing.expectEqualStrings(expected, clean.output); + + const snapshot = try executeCommandInEnvironment(config, arena, command, paths.workspace, .{ .user = paths.shell }); + try std.testing.expectEqualStrings(expected, snapshot.output); + try std.testing.expectEqualStrings("loaded\n", try readAbsoluteFile(arena, loads_path, 4096)); + + shell_snapshot.resetProcessOwnerForTest(brokenReplayCapture); + const full_startup = try executeCommandInEnvironment(config, arena, command, paths.workspace, .{ .user = paths.shell }); + try std.testing.expectEqualStrings(expected, full_startup.output); + try std.testing.expectEqualStrings("loaded\nloaded\n", try readAbsoluteFile(arena, loads_path, 4096)); +} + fn formatOutput(alloc: Allocator, command: []const u8, cwd: []const u8, term: std.process.Child.Term, stdout_raw: []const u8, stderr_raw: []const u8, duration_ms: ?u64) !command_contract.RunCommandResult { return formatOutputWithStatus( alloc, diff --git a/src/core/images/image_attachments.zig b/src/core/images/image_attachments.zig index 01c62f437..8fc5e8427 100644 --- a/src/core/images/image_attachments.zig +++ b/src/core/images/image_attachments.zig @@ -816,18 +816,18 @@ fn attachmentDimensions( attachment: types.ImageAttachment, ) std.mem.Allocator.Error!?image_data.Dimensions { if (attachment.inline_data) |bytes| return image_data.imageDimensions(bytes); - const path = attachment.snapshot_path orelse return null; - const digest = attachment.snapshot_sha256 orelse return probeSnapshotDimensions(path); + const source = imageFile(attachment) orelse return null; + const digest = attachment.snapshot_sha256 orelse return probeSnapshotDimensions(source); if (cache.get(digest)) |cached| return cached; - const dimensions = probeSnapshotDimensions(path); + const dimensions = probeSnapshotDimensions(source); const key = try cache_alloc.dupe(u8, digest); errdefer cache_alloc.free(key); try cache.put(cache_alloc, key, dimensions); return dimensions; } -fn probeSnapshotDimensions(path: []const u8) ?image_data.Dimensions { - var file = openSnapshotFileNoFollow(path) catch return null; +fn probeSnapshotDimensions(source: ImageFile) ?image_data.Dimensions { + var file = source.open() catch return null; defer file.close(io_mod.getIo()); const reader: SnapshotFileReader = .{ .file = file }; return reader.dimensions(); @@ -835,11 +835,11 @@ fn probeSnapshotDimensions(path: []const u8) ?image_data.Dimensions { fn attachmentRawBytes(attachment: types.ImageAttachment) ?usize { if (attachment.inline_data) |bytes| return bytes.len; - const path = attachment.snapshot_path orelse return null; - var file = openSnapshotFileNoFollow(path) catch return null; + const source = imageFile(attachment) orelse return null; + var file = source.open() catch return null; defer file.close(io_mod.getIo()); const stat = file.stat(io_mod.getIo()) catch return null; - if (stat.kind != .file or stat.nlink != 1) return null; + if (!source.accepts(stat)) return null; return std.math.cast(usize, stat.size); } @@ -914,13 +914,12 @@ pub fn withholdDroppedImages(arena: std.mem.Allocator, messages: []const types.C fn writeWithheldAttachmentNotice( writer: *std.Io.Writer, attachment: types.ImageAttachment, - saved: ?[]const u8, dimensions: ?image_data.Dimensions, max_dimension: u32, ) std.Io.Writer.Error!void { if (dimensions) |size| { try writer.print("[Image #{d} not sent: {s} is {d}x{d} pixels. This request permits at most {d} per side and 5 MiB encoded per image. ", .{ attachment.id, attachment.media_type, size.width, size.height, max_dimension }); - } else if (attachment.source_ref != null and attachment.inline_data == null and attachment.snapshot_path == null) { + } else if (attachment.source_ref != null and attachment.inline_data == null and imageFile(attachment) == null) { try writer.print("[Image #{d} not sent: only a host source reference was supplied. This request permits at most {d} per side and 5 MiB encoded per image. ", .{ attachment.id, max_dimension }); } else { try writer.print("[Image #{d} not sent: its dimensions could not be verified. ", .{attachment.id}); @@ -929,10 +928,10 @@ fn writeWithheldAttachmentNotice( try image_data.writeHostImageRecoveryNotice(writer, source_ref, max_dimension); return; } - if (saved) |path| { + if (savedImagePath(attachment)) |path| { try writer.print("The original is saved at {s}. Use an available image tool to save a smaller copy to a new file", .{path}); - // Snapshots are named .bin and blobs by their hash, which hides the - // type from tools that choose a decoder by file name. + // Saved files are named .bin or by their hash, which hides the type + // from tools that choose a decoder by file name. if (mediaTypeExtension(attachment.media_type)) |extension| { try writer.print(" ending in {s}", .{extension}); } @@ -960,17 +959,15 @@ pub const AttachmentProjection = struct { }; /// Leaves unsafe attachments out of the request and gives the model the -/// path of each one's saved copy (`savedImagePath`), or without a saved -/// session the copy fx captured, which lasts while fx runs. History is not -/// modified. `cache` and its keys use `cache_alloc` and should outlive the -/// requests of one turn. +/// path of each one's saved copy (`savedImagePath`) when it has one. +/// History is not modified. `cache` and its keys use `cache_alloc` and +/// should outlive the requests of one turn. pub fn withholdOversizedAttachments( arena: std.mem.Allocator, cache_alloc: std.mem.Allocator, cache: *AttachmentDimensionCache, messages: []const types.ChatMessage, max_dimension: u32, - blob_dir: ?[]const u8, ) !AttachmentProjection { var result: ?[]types.ChatMessage = null; for (messages, 0..) |message, index| { @@ -988,8 +985,7 @@ pub fn withholdOversizedAttachments( kept.?.appendSliceAssumeCapacity(message.images[0..image_index]); } debug_trace.logf("images", "event=attachment_withheld image_id={d} max_dimension={d} dimensions_known={s}", .{ image.id, max_dimension, if (dimensions != null) "true" else "false" }); - const saved = (try savedImagePath(arena, image, blob_dir)) orelse image.snapshot_path; - writeWithheldAttachmentNotice(¬ice.writer, image, saved, dimensions, max_dimension) catch return error.OutOfMemory; + writeWithheldAttachmentNotice(¬ice.writer, image, dimensions, max_dimension) catch return error.OutOfMemory; } const kept_images = kept orelse continue; @@ -1001,21 +997,29 @@ pub fn withholdOversizedAttachments( return .{ .messages = result orelse messages }; } +/// The file the model may open for `image`, or null: a session's blob +/// (D89), otherwise the snapshot of an image not held inline. Inline bytes +/// without a blob own the image, and a snapshot beside them lasts only as +/// long as the process. +pub fn savedImagePath(image: types.ImageAttachment) ?[]const u8 { + if (image.blob_path) |path| return path; + if (image.inline_data != null) return null; + return image.snapshot_path; +} + /// Names the saved file behind each image a request sends, so the model can /// open the image it sees with tools instead of searching the disk for it. /// Only the session's own copy is named, never the path the user attached /// from: that file can change after it was attached, and the copy holds -/// the bytes the model sees (`savedImagePath`). Without a saved session, as -/// in libfx, inline bytes have no saved file, and a snapshot beside them -/// lasts only as long as the process. Returns `messages` itself when no -/// image has a saved file, otherwise a copy allocated in `arena`. History -/// is not modified. -pub fn noteAttachmentLocations(arena: std.mem.Allocator, messages: []const types.ChatMessage, blob_dir: ?[]const u8) ![]const types.ChatMessage { +/// the bytes the model sees (`savedImagePath`). Returns `messages` itself +/// when no image has a saved file, otherwise a copy allocated in `arena`. +/// History is not modified. +pub fn noteAttachmentLocations(arena: std.mem.Allocator, messages: []const types.ChatMessage) ![]const types.ChatMessage { var result: ?[]types.ChatMessage = null; for (messages, 0..) |message, index| { var notes: std.Io.Writer.Allocating = .init(arena); for (message.images) |image| { - const path = (try savedImagePath(arena, image, blob_dir)) orelse continue; + const path = savedImagePath(image) orelse continue; // Quoted, because a home folder can hold spaces or brackets. notes.writer.print("[Image #{d} ({s}) is saved at ", .{ image.id, image.media_type }) catch return error.OutOfMemory; std.json.Stringify.value(path, .{}, ¬es.writer) catch return error.OutOfMemory; @@ -1029,21 +1033,6 @@ pub fn noteAttachmentLocations(arena: std.mem.Allocator, messages: []const types return result orelse messages; } -/// The session's saved copy of `image`: its snapshot when its bytes are not -/// inline, and otherwise the blob of a v2 session in `blob_dir`, which the -/// turn writes with its user item before any tool runs (D79). A blob is -/// named by the lowercase SHA-256 hex of its bytes, as `schema.blobHash` -/// names it (D44), which is what `snapshot_sha256` records. Null when there -/// is none. -pub fn savedImagePath(arena: std.mem.Allocator, image: types.ImageAttachment, blob_dir: ?[]const u8) !?[]const u8 { - if (image.inline_data == null) return image.snapshot_path; - const dir = blob_dir orelse return null; - const hex = image.snapshot_sha256 orelse return null; - if (hex.len != snapshot_digest_hex_len) return null; - for (hex) |c| if (!std.ascii.isDigit(c) and (c < 'a' or c > 'f')) return null; - return try std.Io.Dir.path.join(arena, &.{ dir, hex }); -} - fn syncSnapshotDirectory(snapshot_dir: std.Io.Dir) !void { io_mod.syncVerifiedDir(snapshot_dir) catch |err| switch (err) { error.OperationUnsupported => {}, @@ -1172,6 +1161,41 @@ fn openOrCreateSnapshotDirectoryNoFollow(path: []const u8) !std.Io.Dir { }; } +/// The file that holds an image's bytes when they are not inline: the blob +/// its session saved it as (D89), or else its captured snapshot. Readers +/// check the bytes against the image's digest either way. +const ImageFile = struct { + path: []const u8, + /// A fork hard-links its source's blobs (D44), so a blob may have more + /// than one link. A snapshot in the shared temporary folder must have + /// only its own. + blob: bool, + + /// Whether `stat` is a file this source may be read from. + fn accepts(source: ImageFile, stat: std.Io.File.Stat) bool { + return stat.kind == .file and (source.blob or stat.nlink == 1); + } + + /// A blob is in fx's own private store, so the folders on its path are + /// opened as the path names them, symlinks and all. A snapshot in the + /// shared temporary folder has no symlink anywhere on its path. Neither + /// file may itself be a symlink. + fn open(source: ImageFile) !std.Io.File { + if (!source.blob) return openSnapshotFileNoFollow(source.path); + if (!std.Io.Dir.path.isAbsolute(source.path)) return error.ImageSnapshotPathUnsafe; + return std.Io.Dir.openFileAbsolute(io_mod.getIo(), source.path, .{ + .allow_directory = false, + .follow_symlinks = false, + }) catch |err| return unsafeSnapshotPathError(err); + } +}; + +fn imageFile(attachment: types.ImageAttachment) ?ImageFile { + if (attachment.blob_path) |path| return .{ .path = path, .blob = true }; + if (attachment.snapshot_path) |path| return .{ .path = path, .blob = false }; + return null; +} + fn openSnapshotFileNoFollow(path: []const u8) !std.Io.File { if (!std.Io.Dir.path.isAbsolute(path)) return error.ImageSnapshotPathUnsafe; const parent_path = std.Io.Dir.path.dirname(path) orelse return error.ImageSnapshotPathUnsafe; @@ -1209,15 +1233,15 @@ pub fn loadVerifiedSnapshot( .media_type = detected, }; } - const path = attachment.snapshot_path orelse return error.MissingImageSnapshot; + const source = imageFile(attachment) orelse return error.MissingImageSnapshot; const expected = attachment.snapshot_sha256 orelse return error.MissingImageSnapshot; if (expected.len != snapshot_digest_hex_len) return error.InvalidImageSnapshotDigest; try budget.check(); - var file = try openSnapshotFileNoFollow(path); + var file = try source.open(); defer file.close(io_mod.getIo()); const stat = try file.stat(io_mod.getIo()); - if (stat.kind != .file or stat.nlink != 1) return error.NotRegularFile; + if (!source.accepts(stat)) return error.NotRegularFile; const size = std.math.cast(usize, stat.size) orelse return error.ImageTooLarge; if (size > max_image_bytes) return error.ImageTooLarge; @@ -1252,9 +1276,11 @@ pub fn loadVerifiedSnapshot( /// Keeps a captured snapshot's verified bytes on the attachment itself, as /// an in-memory capture does, so durable serializers save them with the -/// turn: a v2 session keeps its prompt images inside the user's item (D44). -/// The capture still went through the platform resizer; its temporary -/// snapshot file stays until the process's snapshot folder is removed. +/// turn: a v2 session, which is not on disk at capture (D44), saves them as +/// its own blob when the turn starts, or inside the user's item when it +/// cannot (D89). The capture still went through the platform resizer; its +/// temporary snapshot file stays until the process's snapshot folder is +/// removed. pub fn inlineCapturedSnapshot(alloc: std.mem.Allocator, attachment: *types.ImageAttachment) !void { if (attachment.inline_data != null) return; const verified = try loadVerifiedSnapshot(alloc, attachment.*, .{}); @@ -1268,17 +1294,17 @@ pub fn inlineCapturedSnapshots(alloc: std.mem.Allocator, attachments: []types.Im /// Returns a new attachment with independently owned snapshot bytes for `new_id`. /// The source snapshot remains owned by the caller. An image whose bytes are -/// inline is those bytes (D84), so its clone carries them inline and writes -/// no file: a v2 session then saves it as a blob as it does any prompt -/// image, where a copy in the process's temporary folder would be gone once -/// fx exits (D79). +/// inline or in its session's blob is those bytes (D84), so its clone +/// carries them inline and writes no file: a saved session then saves it as +/// a blob as it does any prompt image, where a copy in the process's +/// temporary folder would be gone once fx exits (D89). pub fn cloneVerifiedImageAttachment( alloc: std.mem.Allocator, attachment: types.ImageAttachment, new_id: usize, ) !types.ImageAttachment { if (new_id == 0) return error.InvalidImageId; - if (attachment.inline_data != null) { + if (attachment.inline_data != null or attachment.blob_path != null) { const digest = attachment.snapshot_sha256 orelse return error.MissingImageSnapshot; var verified = try loadVerifiedSnapshot(alloc, attachment, .{}); errdefer verified.deinit(alloc); @@ -3236,59 +3262,37 @@ test "requests name the saved file of each attached image, never its source" { inline_bytes.id = 5; inline_bytes.inline_data = @constCast("png"); const in_memory: types.ImageAttachment = .{ .id = 6, .path = @constCast("inline://image-6"), .media_type = @constCast("image/png"), .snapshot_sha256 = attached.snapshot_sha256, .inline_data = @constCast("png") }; + // A v2 session's image keeps its bytes inline and is saved as its blob + // (D89); the note names the blob, never the temporary snapshot. + var in_blob = inline_bytes; + in_blob.id = 7; + in_blob.blob_path = @constCast("/Users/example/.fx/sessions/v2/s/blobs/0123abcd"); - const images = [_]types.ImageAttachment{ attached, pasted, recalled_file, recalled_paste, inline_bytes, in_memory }; - const prompt = "[Image #1] [Image #2] [Image #3] [Image #4] [Image #5] [Image #6]"; + const images = [_]types.ImageAttachment{ attached, pasted, recalled_file, recalled_paste, inline_bytes, in_memory, in_blob }; + const prompt = "[Image #1] [Image #2] [Image #3] [Image #4] [Image #5] [Image #6] [Image #7]"; const messages = [_]types.ChatMessage{ .{ .role = .user, .content = prompt, .images = &images }, .{ .role = .assistant, .content = "ok" }, }; - const projected = try noteAttachmentLocations(arena, &messages, null); + const projected = try noteAttachmentLocations(arena, &messages); const expected = try arena.print( "[Image #1 (image/png) is saved at \"{s}\".]\n" ++ "[Image #2 (image/png) is saved at \"{s}\".]\n" ++ "[Image #3 (image/png) is saved at \"{s}\".]\n" ++ "[Image #4 (image/png) is saved at \"{s}\".]\n" ++ + "[Image #7 (image/png) is saved at \"{s}\".]\n" ++ prompt, - .{ attached.snapshot_path.?, attached.snapshot_path.?, recalled_file.snapshot_path.?, recalled_paste.snapshot_path.? }, + .{ attached.snapshot_path.?, attached.snapshot_path.?, recalled_file.snapshot_path.?, recalled_paste.snapshot_path.?, in_blob.blob_path.? }, ); try std.testing.expectEqualStrings(expected, projected[0].content.?); try std.testing.expect(std.mem.find(u8, projected[0].content.?, source_path) == null); - try std.testing.expectEqual(@as(usize, 6), projected[0].images.len); + try std.testing.expectEqual(@as(usize, 7), projected[0].images.len); try std.testing.expectEqualStrings("ok", projected[1].content.?); try std.testing.expectEqualStrings(prompt, messages[0].content.?); // A request whose images have no saved file passes through untouched. - const unsaved = [_]types.ChatMessage{.{ .role = .user, .content = "[Image #5] [Image #6]", .images = images[4..] }}; - try std.testing.expect((try noteAttachmentLocations(arena, &unsaved, null)).ptr == (&unsaved).ptr); -} - -test "a v2 session's requests name the blob each inline prompt image is saved as (D79)" { - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - const digest = "5a84b42e05f3301948f95fb3f8255a8f5c2c51c70551b1726afd8e435def96d0"; - const images = [_]types.ImageAttachment{ - .{ .id = 1, .path = @constCast("inline://image-1"), .media_type = @constCast("image/png"), .snapshot_sha256 = @constCast(digest), .inline_data = @constCast("png") }, - // A digest that is not a lowercase SHA-256 names no file. - .{ .id = 2, .path = @constCast("inline://image-2"), .media_type = @constCast("image/png"), .snapshot_sha256 = @constCast("../../etc/passwd"), .inline_data = @constCast("png") }, - .{ .id = 3, .path = @constCast("inline://image-3"), .media_type = @constCast("image/png"), .inline_data = @constCast("png") }, - // An image without inline bytes keeps naming its snapshot. - .{ .id = 4, .path = @constCast("/Users/example/shot.png"), .media_type = @constCast("image/png"), .snapshot_path = @constCast("/Users/example/.fx/sessions/s/images/image-4.bin") }, - }; - const messages = [_]types.ChatMessage{.{ .role = .user, .content = "look", .images = &images }}; - const projected = try noteAttachmentLocations(arena, &messages, "/Users/example/.fx/sessions/v2/s/blobs"); - try std.testing.expectEqualStrings( - "[Image #1 (image/png) is saved at \"/Users/example/.fx/sessions/v2/s/blobs/" ++ digest ++ "\".]\n" ++ - "[Image #4 (image/png) is saved at \"/Users/example/.fx/sessions/s/images/image-4.bin\".]\n" ++ - "look", - projected[0].content.?, - ); - // Without a blob folder, as in libfx, inline bytes have no saved file. - try std.testing.expectEqualStrings( - "[Image #4 (image/png) is saved at \"/Users/example/.fx/sessions/s/images/image-4.bin\".]\nlook", - (try noteAttachmentLocations(arena, &messages, null))[0].content.?, - ); + const unsaved = [_]types.ChatMessage{.{ .role = .user, .content = "[Image #5] [Image #6]", .images = images[4..6] }}; + try std.testing.expect((try noteAttachmentLocations(arena, &unsaved)).ptr == (&unsaved).ptr); } test "temporary snapshot folders have names of one length" { @@ -3312,7 +3316,7 @@ test "image location notes quote the saved path" { .snapshot_path = @constCast("/Users/example/fx \"home\"]\n/sessions/s/images/image-7-0123456789abcdef.bin"), }}; const messages = [_]types.ChatMessage{.{ .role = .user, .content = "[Image #7]", .images = &images }}; - const projected = try noteAttachmentLocations(arena, &messages, null); + const projected = try noteAttachmentLocations(arena, &messages); try std.testing.expectEqualStrings( "[Image #7 (image/jpeg) is saved at \"/Users/example/fx \\\"home\\\"]\\n/sessions/s/images/image-7-0123456789abcdef.bin\".]\n[Image #7]", projected[0].content.?, @@ -3363,7 +3367,7 @@ test "requests leave out attachments over the model pixel limit and name their s }; var cache: AttachmentDimensionCache = .empty; - const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension, null); + const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension); const projected = projection.messages; try std.testing.expectEqual(@as(usize, 1), projected[0].images.len); @@ -3397,9 +3401,9 @@ test "request cache still withholds a snapshot deleted after dimension lookup" { }; var cache: AttachmentDimensionCache = .empty; - const first = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension, null); + const first = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension); try tmp.dir.deleteFile(std.testing.io, "image-1-0000000000000001.bin"); - const second = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension, null); + const second = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension); try std.testing.expectEqual(@as(usize, 0), first.messages[0].images.len); try std.testing.expectEqual(@as(usize, 0), second.messages[0].images.len); @@ -3417,7 +3421,7 @@ test "requests ask for a smaller copy of an oversized in-memory attachment" { const messages = [_]types.ChatMessage{.{ .role = .user, .content = "[Image #3]", .images = &images }}; var cache: AttachmentDimensionCache = .empty; - const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension, null); + const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension); try std.testing.expectEqual(@as(usize, 0), projection.messages[0].images.len); try std.testing.expectEqualStrings( @@ -3442,30 +3446,32 @@ test "requests keep verified attachments within the pixel limit unchanged" { const messages = [_]types.ChatMessage{.{ .role = .user, .content = "[Image #1] [Image #2]", .images = &images }}; var cache: AttachmentDimensionCache = .empty; - const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension, null); + const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension); try std.testing.expectEqual(@as([*]const types.ChatMessage, &messages), projection.messages.ptr); } -test "a withheld prompt image's notice names the saved copy the note names (D79)" { +test "a withheld prompt image's notice names the blob the note names (D89)" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); const wide = image_data.testPngHeader(9000, 10); - var captured = try captureInlineImageBytesInMemory(arena, 1, "image/png", &wide); - captured.snapshot_path = @constCast("/tmp/fx-image-snapshots-0123456789abcdef/image-1.bin"); - const images = [_]types.ImageAttachment{captured}; - const messages = [_]types.ChatMessage{.{ .role = .user, .content = "look", .images = &images }}; + var saved = try captureInlineImageBytesInMemory(arena, 1, "image/png", &wide); + saved.snapshot_path = @constCast("/tmp/fx-image-snapshots-0123456789abcdef/image-1.bin"); + saved.blob_path = @constCast("/Users/example/.fx/sessions/v2/s/blobs/0123abcd"); var cache: AttachmentDimensionCache = .empty; - const blob_dir = "/Users/example/.fx/sessions/v2/s/blobs"; - const saved = try withholdOversizedAttachments(arena, arena, &cache, &messages, 8000, blob_dir); - try std.testing.expectEqual(@as(usize, 0), saved.messages[0].images.len); - const blob = try std.Io.Dir.path.join(arena, &.{ blob_dir, captured.snapshot_sha256.? }); - try std.testing.expect(std.mem.find(u8, saved.messages[0].content.?, try arena.print("The original is saved at {s}. ", .{blob})) != null); - // Without a saved session it names the captured copy, which lasts - // while fx runs. - const unsaved = try withholdOversizedAttachments(arena, arena, &cache, &messages, 8000, null); - try std.testing.expect(std.mem.find(u8, unsaved.messages[0].content.?, "The original is saved at /tmp/fx-image-snapshots-0123456789abcdef/image-1.bin. ") != null); + const in_blob = [_]types.ChatMessage{.{ .role = .user, .content = "look", .images = &.{saved} }}; + const named = try withholdOversizedAttachments(arena, arena, &cache, &in_blob, 8000); + try std.testing.expectEqual(@as(usize, 0), named.messages[0].images.len); + try std.testing.expect(std.mem.find(u8, named.messages[0].content.?, "The original is saved at /Users/example/.fx/sessions/v2/s/blobs/0123abcd. ") != null); + // Inline bytes without a blob have no saved file, and the temporary + // snapshot beside them is never named. + var unsaved = saved; + unsaved.blob_path = null; + const inline_only = [_]types.ChatMessage{.{ .role = .user, .content = "look", .images = &.{unsaved} }}; + const unnamed = try withholdOversizedAttachments(arena, arena, &cache, &inline_only, 8000); + try std.testing.expect(std.mem.find(u8, unnamed.messages[0].content.?, "No local image file is available") != null); + try std.testing.expect(std.mem.find(u8, unnamed.messages[0].content.?, "fx-image-snapshots") == null); } test "reference-only attachments get host recovery guidance without attempting file access" { @@ -3480,7 +3486,7 @@ test "reference-only attachments get host recovery guidance without attempting f }}; const messages = [_]types.ChatMessage{.{ .role = .user, .images = &images }}; var cache: AttachmentDimensionCache = .empty; - const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, 8000, null); + const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, 8000); try std.testing.expectEqual(@as(usize, 0), projection.messages[0].images.len); const notice = projection.messages[0].content.?; try std.testing.expect(std.mem.find(u8, notice, "only a host source reference was supplied") != null); @@ -3502,7 +3508,7 @@ test "requests withhold snapshots whose dimensions cannot be verified" { }}; const messages = [_]types.ChatMessage{.{ .role = .user, .images = &images }}; var cache: AttachmentDimensionCache = .empty; - const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_single_image_dimension, null); + const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_single_image_dimension); try std.testing.expectEqual(@as(usize, 0), projection.messages[0].images.len); try std.testing.expect(std.mem.find(u8, projection.messages[0].content.?, "dimensions could not be verified") != null); try std.testing.expectEqual(@as(usize, 1), messages[0].images.len); @@ -3617,7 +3623,7 @@ test "requests find a JPEG frame header behind large metadata" { const messages = [_]types.ChatMessage{.{ .role = .user, .content = "[Image #1]", .images = &images }}; var cache: AttachmentDimensionCache = .empty; - const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension, null); + const projection = try withholdOversizedAttachments(arena, arena, &cache, &messages, image_data.max_image_dimension); try std.testing.expect(std.mem.startsWith(u8, projection.messages[0].content.?, "[Image #1 not sent: image/jpeg is 4032x3024 pixels")); } @@ -4496,7 +4502,7 @@ test "inline snapshot verification rejects corrupted retained bytes" { try std.testing.expectError(error.MissingImageSnapshot, loadVerifiedSnapshot(alloc, missing_digest, .{})); } -test "a recalled image whose bytes are inline keeps them inline and writes no file (D79, D84)" { +test "a recalled image whose bytes are inline keeps them inline and writes no file (D84, D89)" { const alloc = std.testing.allocator; const png = image_data.testPngHeader(10, 10); const attachment = try captureInlineImageBytesInMemory(alloc, 1, "image/png", &png); @@ -4517,3 +4523,69 @@ test "a recalled image whose bytes are inline keeps them inline and writes no fi corrupted.inline_data.?[9] ^= 1; try std.testing.expectError(error.ImageSnapshotCorrupt, cloneVerifiedImageAttachment(alloc, corrupted, 3)); } + +test "an image saved as a blob is only its handle, and readers load the blob, even one a fork hard-linked (D89)" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const png = image_data.testPngHeader(4, 3); + var digest: [Sha256.digest_length]u8 = undefined; + Sha256.hash(&png, &digest, .{}); + var hex = std.fmt.bytesToHex(digest, .lower); + try tmp.dir.writeFile(io_mod.getIo(), .{ .sub_path = &hex, .data = &png }); + // A fork hard-links its source's blobs, so this one has two links. + try std.Io.Dir.hardLink(tmp.dir, &hex, tmp.dir, "forked", io_mod.getIo(), .{ .follow_symlinks = false }); + const blob = try io_mod.dirRealpathAlloc(alloc, tmp.dir, &hex); + defer alloc.free(blob); + const handle: types.ImageAttachment = .{ + .id = 1, + .path = @constCast("/Users/example/shot.png"), + .media_type = @constCast("image/png"), + .snapshot_sha256 = &hex, + .blob_path = blob, + }; + try std.testing.expect(!handle.droppedFromSession()); + var read = try loadVerifiedSnapshot(alloc, handle, .{}); + defer read.deinit(alloc); + try std.testing.expectEqualSlices(u8, &png, read.bytes); + try std.testing.expectEqual(@as(?usize, png.len), attachmentRawBytes(handle)); + var cache_arena = std.heap.ArenaAllocator.init(alloc); + defer cache_arena.deinit(); + var cache: AttachmentDimensionCache = .empty; + const dimensions = (try attachmentDimensions(cache_arena.allocator(), &cache, handle)).?; + try std.testing.expectEqual(@as(u32, 4), dimensions.width); + + // A blob's folders may be reached through a symlink, as a home under + // macOS's /var is; a snapshot's may not. + try tmp.dir.symLink(io_mod.getIo(), ".", "linked", .{ .is_directory = true }); + const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); + defer alloc.free(root); + const through_link = try std.Io.Dir.path.join(alloc, &.{ root, "linked", &hex }); + defer alloc.free(through_link); + var linked = handle; + linked.blob_path = through_link; + var linked_read = try loadVerifiedSnapshot(alloc, linked, .{}); + defer linked_read.deinit(alloc); + try std.testing.expectEqualSlices(u8, &png, linked_read.bytes); + var linked_snapshot = handle; + linked_snapshot.blob_path = null; + linked_snapshot.snapshot_path = through_link; + try std.testing.expectError(error.ImageSnapshotPathUnsafe, loadVerifiedSnapshot(alloc, linked_snapshot, .{})); + + // A recall carries the bytes inline and writes no file. + const recalled = try cloneVerifiedImageAttachment(alloc, handle, 2); + defer discardImageAttachment(alloc, recalled); + try std.testing.expectEqualSlices(u8, &png, recalled.inline_data.?); + try std.testing.expectEqual(@as(?[]u8, null), recalled.blob_path); + + // Only a blob may have another link; a snapshot in the shared + // temporary folder may not. + var snapshot = handle; + snapshot.blob_path = null; + snapshot.snapshot_path = blob; + try std.testing.expectError(error.NotRegularFile, loadVerifiedSnapshot(alloc, snapshot, .{})); + + // A damaged blob fails its digest wherever it is read. + try tmp.dir.writeFile(io_mod.getIo(), .{ .sub_path = &hex, .data = "damaged" }); + try std.testing.expectError(error.ImageSnapshotCorrupt, loadVerifiedSnapshot(alloc, handle, .{})); +} diff --git a/src/core/images/image_route.zig b/src/core/images/image_route.zig new file mode 100644 index 000000000..5e9f56b3c --- /dev/null +++ b/src/core/images/image_route.zig @@ -0,0 +1,369 @@ +//! How images reach the model on one request. +//! +//! `step` is the whole decision: given the turn's state and one event, it +//! returns the next state and the effects to run. It performs no I/O and reads +//! no clock, environment, or global. The orchestrator is its shell: it runs +//! each effect and feeds the result back as an event. +//! +//! `tests/tla/image_route/spec/ImageRoute.tla` models `step` action for +//! action, and `abstract` maps a state onto that model's variables. The laws +//! below are the model's properties; `tests/tla/image_route/design/DESIGN.md` +//! lists where each one is enforced. +//! +//! This file imports only `std`, so the JSON-lines harness in +//! `tests/tla/drivers/image_route/` can build it on its own. +const std = @import("std"); + +/// Whether the selected model takes images inline. +pub const Support = enum { + /// The catalog was unavailable or did not list the model. + unknown, + /// The catalog lists the model without vision. + non_native, + /// The catalog lists the model with vision. + native, +}; + +/// What the model catalog said about one model. +pub const CatalogView = union(enum) { + /// The catalog could not be read: unreachable, refused, or malformed. + down, + /// The catalog was read and does not list the model. + missing, + listed: Tags, + + pub const Tags = struct { + vision: bool, + file_input: bool, + }; +}; + +/// Images reach the model inline exactly when the catalog lists `vision`. +/// +/// fx sends every image, user or tool, in an AI SDK `file` part with an +/// `image/*` media type, which is the AI Gateway's vision input. The +/// catalog's `file-input` tag covers documents such as PDFs and says nothing +/// about images, so it plays no part here. +pub fn supportFromCatalog(view: CatalogView) Support { + return switch (view) { + .down, .missing => .unknown, + .listed => |tags| if (tags.vision) .native else .non_native, + }; +} + +/// Where the turn's user images go. +pub const Route = enum { + /// Inline, as file parts. + native, + /// Through the vision tool, as text. + fallback, + /// Nowhere: a request with user images fails. + unavailable, +}; + +/// How the request offers the vision tool. +pub const VisionMode = enum { + unavailable, + optional, + required, +}; + +/// What happens to the tool images in the request's history. +pub const ToolImages = enum { + /// The history has none. + none, + /// They go inline. + attached, + /// Withheld; the model is told to call the vision tool. + withheld_fallback, + /// Withheld; the model is told no image path exists. + withheld_dead_end, + /// Withheld; the model is told support could not be confirmed. + withheld_unknown, +}; + +pub const FailReason = enum { + /// User images, and the catalog could not confirm image input. + capability_unknown, + /// User images, a model without vision, and no vision tool. + no_native_image, +}; + +/// What the shell knows about the request it is about to send. Only flags: +/// image bytes never enter the core. +pub const Request = struct { + /// The turn carries user images, attached now or authorized earlier. + user_images: bool, + /// Some of those images still wait for the model's first look. + pending_images: bool, + /// The request history holds tool images the session still has. + tool_images: bool, + /// The vision tool is configured and registered. + fallback: bool, + /// The catalog is known to be unreachable right now. + catalog_down: bool, +}; + +pub const State = struct { + /// The turn's one capability snapshot. Every decision reads it. + support: Support = .unknown, + /// This turn already asked for capabilities. + resolved: bool = false, + /// This turn already told the user the catalog is down. + notified: bool = false, + /// The request waiting for the capability reply. + pending: ?Request = null, +}; + +pub const Event = union(enum) { + /// A turn starts with the capabilities the shell holds. It also ends + /// whatever the previous turn left: a reply still owed is dropped. + turn_started: Support, + /// The shell has built a request's history and is about to send it. + request_ready: Request, + /// The reply to `resolve_capabilities`. + capabilities_resolved: Support, +}; + +/// One request's image handling, all read from one support value. +pub const Decision = struct { + route: Route, + mode: VisionMode, + tool_images: ToolImages, +}; + +pub const Effect = union(enum) { + /// Ask the catalog again, then send `capabilities_resolved`. + resolve_capabilities, + /// Tell the user once that tool images wait on the catalog. + notify_catalog_down, + /// Send the request as decided. + send: Decision, + /// Fail the turn without sending. + fail: FailReason, +}; + +pub const max_effects = 2; + +pub const Output = struct { + state: State, + buffer: [max_effects]Effect = undefined, + len: usize = 0, + + pub fn effects(self: *const Output) []const Effect { + return self.buffer[0..self.len]; + } +}; + +pub fn step(state: State, event: Event) Output { + return switch (event) { + .turn_started => |support| .{ .state = .{ .support = support } }, + .request_ready => |request| onRequest(state, request), + .capabilities_resolved => |support| onResolved(state, support), + }; +} + +fn onRequest(state: State, request: Request) Output { + if (state.pending != null) { + // Delivered while a reply is owed: the reply answers the newest + // request, and nothing runs twice. + var next = state; + next.pending = request; + return .{ .state = next }; + } + if (state.support == .unknown and !state.resolved and (request.user_images or request.tool_images)) { + var next = state; + next.resolved = true; + next.pending = request; + return withEffects(next, &.{.resolve_capabilities}); + } + return decide(state, request); +} + +fn onResolved(state: State, support: Support) Output { + // A reply nothing waits for, stale or duplicated, changes nothing. + const request = state.pending orelse return .{ .state = state }; + var next = state; + next.support = support; + next.pending = null; + return decide(next, request); +} + +fn decide(state: State, request: Request) Output { + const chosen = decision(state.support, request); + if (request.user_images and chosen.route == .unavailable) { + const reason: FailReason = if (state.support == .unknown) .capability_unknown else .no_native_image; + return withEffects(state, &.{.{ .fail = reason }}); + } + if (chosen.tool_images == .withheld_unknown and request.catalog_down and !state.notified) { + var next = state; + next.notified = true; + return withEffects(next, &.{ .notify_catalog_down, .{ .send = chosen } }); + } + return withEffects(state, &.{.{ .send = chosen }}); +} + +/// The image handling `support` implies for `request`. +pub fn decision(support: Support, request: Request) Decision { + const route: Route = switch (support) { + .native => .native, + .unknown => .unavailable, + .non_native => if (request.fallback) .fallback else .unavailable, + }; + const mode: VisionMode = if (route != .fallback) + .unavailable + else if (request.pending_images) + .required + else + .optional; + const tool_images: ToolImages = if (!request.tool_images) .none else switch (support) { + .native => .attached, + .unknown => .withheld_unknown, + .non_native => if (route == .fallback) .withheld_fallback else .withheld_dead_end, + }; + return .{ .route = route, .mode = mode, .tool_images = tool_images }; +} + +fn withEffects(state: State, list: []const Effect) Output { + var out: Output = .{ .state = state, .len = list.len }; + @memcpy(out.buffer[0..list.len], list); + return out; +} + +/// The spec's view of a state: `support`, `resolved`, `notified`, and +/// `pending` as a sequence of at most one request. +pub const Abstract = struct { + support: Support, + resolved: bool, + notified: bool, + pending: []const Request, +}; + +/// Maps `state` onto the variables of `ImageRoute.tla`. The result borrows +/// `state`. +pub fn abstract(state: *const State) Abstract { + return .{ + .support = state.support, + .resolved = state.resolved, + .notified = state.notified, + .pending = if (state.pending) |*request| request[0..1] else &.{}, + }; +} + +fn eachRequest(comptime body: anytype) !void { + for (0..32) |bits| { + const request: Request = .{ + .user_images = bits & 1 != 0, + .pending_images = bits & 2 != 0, + .tool_images = bits & 4 != 0, + .fallback = bits & 8 != 0, + .catalog_down = bits & 16 != 0, + }; + if (request.pending_images and !request.user_images) continue; + try body(request); + } +} + +const all_support = [_]Support{ .unknown, .non_native, .native }; + +test "a catalog that lists vision makes image input native with or without file input" { + for ([_]bool{ false, true }) |file_input| { + try std.testing.expectEqual(Support.native, supportFromCatalog(.{ .listed = .{ .vision = true, .file_input = file_input } })); + try std.testing.expectEqual(Support.non_native, supportFromCatalog(.{ .listed = .{ .vision = false, .file_input = file_input } })); + } + try std.testing.expectEqual(Support.unknown, supportFromCatalog(.down)); + try std.testing.expectEqual(Support.unknown, supportFromCatalog(.missing)); +} + +test "every decision reads one support value" { + try eachRequest(struct { + fn check(request: Request) !void { + for (all_support) |support| { + const chosen = decision(support, request); + try std.testing.expectEqual(chosen.route == .native, support == .native); + try std.testing.expectEqual(chosen.route == .fallback, chosen.mode != .unavailable); + try std.testing.expectEqual(chosen.tool_images == .none, !request.tool_images); + if (chosen.tool_images == .attached) try std.testing.expectEqual(Support.native, support); + if (chosen.tool_images == .withheld_unknown) try std.testing.expectEqual(Support.unknown, support); + if (chosen.tool_images == .withheld_fallback) try std.testing.expectEqual(Route.fallback, chosen.route); + if (chosen.tool_images == .withheld_dead_end) try std.testing.expect(support == .non_native and chosen.route == .unavailable); + if (chosen.route == .fallback) try std.testing.expectEqual(request.pending_images, chosen.mode == .required); + } + } + }.check); +} + +test "a turn asks for capabilities at most once and answers the request it held" { + try eachRequest(struct { + fn check(request: Request) !void { + const started = step(.{}, .{ .turn_started = .unknown }); + const first = step(started.state, .{ .request_ready = request }); + if (!request.user_images and !request.tool_images) { + try std.testing.expectEqual(@as(usize, 1), first.len); + try std.testing.expect(first.effects()[0] == .send); + return; + } + try std.testing.expectEqual(@as(usize, 1), first.len); + try std.testing.expect(first.effects()[0] == .resolve_capabilities); + // A duplicate delivery while the reply is owed runs nothing. + const duplicate = step(first.state, .{ .request_ready = request }); + try std.testing.expectEqual(@as(usize, 0), duplicate.len); + const answered = step(duplicate.state, .{ .capabilities_resolved = .unknown }); + try std.testing.expect(answered.state.pending == null); + for (answered.effects()) |effect| try std.testing.expect(effect != .resolve_capabilities); + // A stale reply changes nothing. + const stale = step(answered.state, .{ .capabilities_resolved = .native }); + try std.testing.expectEqual(@as(usize, 0), stale.len); + try std.testing.expectEqualDeep(answered.state, stale.state); + // The next request in the turn does not ask again. + const again = step(stale.state, .{ .request_ready = request }); + for (again.effects()) |effect| try std.testing.expect(effect != .resolve_capabilities); + } + }.check); +} + +test "the catalog-down notice reaches the user once per turn" { + const request: Request = .{ .user_images = false, .pending_images = false, .tool_images = true, .fallback = true, .catalog_down = true }; + var state = step(.{}, .{ .turn_started = .unknown }).state; + var notices: usize = 0; + for (0..3) |_| { + var out = step(state, .{ .request_ready = request }); + if (out.len == 1 and out.effects()[0] == .resolve_capabilities) out = step(out.state, .{ .capabilities_resolved = .unknown }); + for (out.effects()) |effect| { + if (effect == .notify_catalog_down) notices += 1; + } + state = out.state; + } + try std.testing.expectEqual(@as(usize, 1), notices); + const next_turn = step(state, .{ .turn_started = .unknown }); + try std.testing.expect(!next_turn.state.notified and !next_turn.state.resolved); +} + +test "user images without a route fail the request and never send" { + for (all_support) |support| { + const request: Request = .{ .user_images = true, .pending_images = true, .tool_images = true, .fallback = false, .catalog_down = false }; + const out = step(.{ .support = support, .resolved = true }, .{ .request_ready = request }); + try std.testing.expectEqual(@as(usize, 1), out.len); + switch (support) { + .native => try std.testing.expect(out.effects()[0] == .send), + .unknown => try std.testing.expectEqual(Effect{ .fail = .capability_unknown }, out.effects()[0]), + .non_native => try std.testing.expectEqual(Effect{ .fail = .no_native_image }, out.effects()[0]), + } + } +} + +test "abstract maps the pending request onto a sequence of at most one" { + const request: Request = .{ .user_images = true, .pending_images = false, .tool_images = false, .fallback = true, .catalog_down = false }; + const idle: State = .{}; + try std.testing.expectEqual(@as(usize, 0), abstract(&idle).pending.len); + const waiting: State = .{ .resolved = true, .pending = request }; + const view = abstract(&waiting); + try std.testing.expectEqual(@as(usize, 1), view.pending.len); + try std.testing.expectEqualDeep(request, view.pending[0]); +} + +test "requests carry flags, not image bytes" { + inline for (@typeInfo(Request).@"struct".field_types) |field_type| { + try std.testing.expect(field_type == bool); + } +} diff --git a/src/core/input/paste_framing.zig b/src/core/input/paste_framing.zig index 0bd01ffa5..6288cdb39 100644 --- a/src/core/input/paste_framing.zig +++ b/src/core/input/paste_framing.zig @@ -48,19 +48,23 @@ pub const ResetReason = union(enum) { input_limit: Owner, invalid_utf8, new_paste_stale_state, - unsafe_suffix, }; pub const BoundaryState = enum { capturing, end_candidate, - unsafe, }; -pub const Settlement = enum { +pub const Settlement = union(enum) { none, - finish, - reject, + finish: Finish, +}; + +/// A paste that ended in this delivery epoch. Bytes that followed its end +/// marker in the same epoch are never input: they may be clipboard content +/// after an end marker embedded in the paste, so they are only counted. +const Finish = struct { + dropped_suffix_bytes: usize, }; pub const State = struct { @@ -98,11 +102,6 @@ pub const State = struct { switch (self.boundary) { .capturing => {}, .end_candidate => { - self.boundary = .unsafe; - self.unsafe_suffix_bytes +|= 1; - return; - }, - .unsafe => { self.unsafe_suffix_bytes +|= 1; return; }, @@ -161,8 +160,7 @@ pub const State = struct { if (!self.active()) return .none; return switch (self.boundary) { .capturing => .none, - .end_candidate => .finish, - .unsafe => if (self.confirmedOverflowBytes() > 0) .finish else .reject, + .end_candidate => .{ .finish = .{ .dropped_suffix_bytes = self.unsafe_suffix_bytes } }, }; } @@ -178,17 +176,6 @@ pub const State = struct { return self.buffer.items.len +| self.overflow_bytes; } - pub fn observedBytes(self: *const State) usize { - return self.attemptedBytes() +| - self.decision_bytes +| - self.unsafe_suffix_bytes; - } - - pub fn confirmedOverflowBytes(self: *const State) usize { - if (self.end_match_len > 0) return self.end_candidate_overflow_bytes; - return self.overflow_bytes; - } - /// Ends framing while retaining captured bytes for synchronous handling. pub fn beginHandling(self: *State) void { self.owner = .none; @@ -269,11 +256,6 @@ pub const State = struct { ); } }, - .unsafe_suffix => debug_trace.logf( - "input", - "{s} paste dropped bytes={d} reason=unsafe_suffix", - .{ ownerLabel(self.owner), self.observedBytes() }, - ), .decision_prompt_active, .session_reset, .shutdown => { switch (self.owner) { .decision_prompt => debug_trace.logf( @@ -339,7 +321,6 @@ fn resetReasonName(reason: ResetReason) []const u8 { .input_limit => "input_limit", .invalid_utf8 => "invalid_utf8", .new_paste_stale_state => "new_paste_stale_state", - .unsafe_suffix => "unsafe_suffix", }; } @@ -386,7 +367,7 @@ test "decision paste counts content bytes and exact end marker only" { try std.testing.expectEqual(@as(usize, 3), state.decision_bytes); try std.testing.expectEqual(Owner.decision_prompt, state.owner); try std.testing.expectEqual(BoundaryState.end_candidate, state.boundary); - try std.testing.expectEqual(Settlement.finish, state.settleDeliveryEpoch()); + try std.testing.expectEqual(Settlement{ .finish = .{ .dropped_suffix_bytes = 0 } }, state.settleDeliveryEpoch()); } test "decision paste treats incomplete and parameterized end candidates as content" { @@ -402,7 +383,7 @@ test "decision paste treats incomplete and parameterized end candidates as conte try state.consumeByte(std.testing.allocator, byte); } try std.testing.expectEqual(payload.len, state.decision_bytes); - try std.testing.expectEqual(Settlement.finish, state.settleDeliveryEpoch()); + try std.testing.expectEqual(Settlement{ .finish = .{ .dropped_suffix_bytes = 0 } }, state.settleDeliveryEpoch()); } test "composer paste preserves safe bytes and tracks bounded overflow" { @@ -427,12 +408,10 @@ test "composer overflow excludes a possible exact end marker" { exact.begin(.composer, 1); try exact.consumeByte(std.testing.allocator, 'x'); - for (paste_end_marker[0 .. paste_end_marker.len - 1]) |byte| { + for (paste_end_marker) |byte| { try exact.consumeByte(std.testing.allocator, byte); - try std.testing.expectEqual(@as(usize, 0), exact.confirmedOverflowBytes()); } - try exact.consumeByte(std.testing.allocator, paste_end_marker[paste_end_marker.len - 1]); - try std.testing.expectEqual(@as(usize, 0), exact.confirmedOverflowBytes()); + try std.testing.expectEqual(@as(usize, 0), exact.overflow_bytes); var mismatch: State = .{}; defer mismatch.deinit(std.testing.allocator); @@ -442,10 +421,10 @@ test "composer overflow excludes a possible exact end marker" { for ("\x1b[20x") |byte| { try mismatch.consumeByte(std.testing.allocator, byte); } - try std.testing.expect(mismatch.confirmedOverflowBytes() > 0); + try std.testing.expect(mismatch.overflow_bytes > 0); } -test "composer overflow rejection takes precedence over an unsafe suffix" { +test "composer overflow survives an ignored suffix" { var state: State = .{}; defer state.deinit(std.testing.allocator); state.begin(.composer, 1); @@ -458,9 +437,9 @@ test "composer overflow rejection takes precedence over an unsafe suffix" { } try state.consumeByte(std.testing.allocator, 'z'); - try std.testing.expectEqual(BoundaryState.unsafe, state.boundary); - try std.testing.expect(state.confirmedOverflowBytes() > 0); - try std.testing.expectEqual(Settlement.finish, state.settleDeliveryEpoch()); + try std.testing.expectEqual(BoundaryState.end_candidate, state.boundary); + try std.testing.expect(state.overflow_bytes > 0); + try std.testing.expectEqual(Settlement{ .finish = .{ .dropped_suffix_bytes = 1 } }, state.settleDeliveryEpoch()); } test "discarding remainder preserves buffered bytes and end-marker progress" { @@ -475,7 +454,7 @@ test "discarding remainder preserves buffered bytes and end-marker progress" { try state.consumeByte(std.testing.allocator, byte); } - try std.testing.expectEqual(Settlement.finish, state.settleDeliveryEpoch()); + try std.testing.expectEqual(Settlement{ .finish = .{ .dropped_suffix_bytes = 0 } }, state.settleDeliveryEpoch()); try std.testing.expectEqualStrings("x", state.buffer.items); try std.testing.expectEqual(@as(usize, 0), state.decision_bytes); } @@ -542,10 +521,10 @@ test "end candidate stays paste owned until delivery epoch settlement" { try std.testing.expect(state.active()); try std.testing.expectEqualStrings("safe", state.buffer.items); try std.testing.expectEqual(BoundaryState.end_candidate, state.boundary); - try std.testing.expectEqual(Settlement.finish, state.settleDeliveryEpoch()); + try std.testing.expectEqual(Settlement{ .finish = .{ .dropped_suffix_bytes = 0 } }, state.settleDeliveryEpoch()); } -test "any same-epoch suffix makes the candidate sticky unsafe" { +test "a same-epoch suffix is counted and never captured" { var state: State = .{}; defer state.deinit(std.testing.allocator); state.begin(.composer, std.math.maxInt(usize)); @@ -559,9 +538,11 @@ test "any same-epoch suffix makes the candidate sticky unsafe" { try std.testing.expect(state.active()); try std.testing.expectEqualStrings("safe", state.buffer.items); - try std.testing.expectEqual(BoundaryState.unsafe, state.boundary); - try std.testing.expectEqual(@as(usize, 1 + paste_end_marker.len), state.unsafe_suffix_bytes); - try std.testing.expectEqual(Settlement.reject, state.settleDeliveryEpoch()); + try std.testing.expectEqual(BoundaryState.end_candidate, state.boundary); + try std.testing.expectEqual( + Settlement{ .finish = .{ .dropped_suffix_bytes = 1 + paste_end_marker.len } }, + state.settleDeliveryEpoch(), + ); } test "traced reset reports dropped captures and clears only paste state" { diff --git a/src/core/session/session.zig b/src/core/session/session.zig index ffee95116..f470d873e 100644 --- a/src/core/session/session.zig +++ b/src/core/session/session.zig @@ -220,12 +220,12 @@ pub fn collect_image_catalog( for (canonical_history) |turn| { for (images_for_history_turn(turn)) |attachment| { - catalog[copied] = try dupeImageAttachment(alloc, attachment); + catalog[copied] = try core_types.dupeImageAttachment(alloc, attachment); copied += 1; } } for (current_images) |attachment| { - catalog[copied] = try dupeImageAttachment(alloc, attachment); + catalog[copied] = try core_types.dupeImageAttachment(alloc, attachment); copied += 1; } @@ -278,12 +278,12 @@ pub fn merge_image_catalog_history_turn( core_types.freeImageAttachment(alloc, attachment); }; for (catalog) |attachment| { - merged[copied] = try dupeImageAttachment(alloc, attachment); + merged[copied] = try core_types.dupeImageAttachment(alloc, attachment); copied += 1; } for (additions) |attachment| { if (image_attachment_for_id(catalog, attachment.id) != null) continue; - merged[copied] = try dupeImageAttachment(alloc, attachment); + merged[copied] = try core_types.dupeImageAttachment(alloc, attachment); copied += 1; } sort_utils.sort(ImageAttachment, merged, {}, image_id_less_than); @@ -1814,45 +1814,7 @@ pub fn freeHistoryTurnSlice(alloc: Allocator, turns: []HistoryTurn) void { if (turns.len > 0) alloc.free(turns); } /// Deep-copies an image attachment slice; caller owns the returned slice and frees with freeImageAttachmentSlice. -pub fn dupeImageAttachmentSlice(alloc: Allocator, attachments: []const ImageAttachment) ![]ImageAttachment { - if (attachments.len == 0) return &.{}; - const copy = try alloc.alloc(ImageAttachment, attachments.len); - errdefer alloc.free(copy); - var copied: usize = 0; - errdefer { - var i: usize = 0; - while (i < copied) : (i += 1) { - core_types.freeImageAttachment(alloc, copy[i]); - } - } - for (attachments, 0..) |attachment, i| { - copy[i] = try dupeImageAttachment(alloc, attachment); - copied += 1; - } - return copy; -} -fn dupeImageAttachment(alloc: Allocator, src: ImageAttachment) !ImageAttachment { - const path = try alloc.dupe(u8, src.path); - errdefer alloc.free(path); - const media_type = try alloc.dupe(u8, src.media_type); - errdefer alloc.free(media_type); - const snapshot_path = if (src.snapshot_path) |value| - try alloc.dupe(u8, value) - else - null; - errdefer if (snapshot_path) |value| alloc.free(value); - const snapshot_sha256 = if (src.snapshot_sha256) |value| - try alloc.dupe(u8, value) - else - null; - return .{ - .id = src.id, - .path = path, - .media_type = media_type, - .snapshot_path = snapshot_path, - .snapshot_sha256 = snapshot_sha256, - }; -} +pub const dupeImageAttachmentSlice = core_types.dupeImageAttachmentSlice; /// Deep-copies one history turn; caller owns the returned turn and frees with freeHistoryTurn. pub const dupeHistoryTurn = core_types.dupeHistoryTurn; pub fn appendAssistantTurnWithExecution( @@ -2595,27 +2557,6 @@ test "appendHistoryMessages frees owned projection text when append fails" { var compact_messages: std.ArrayList(message.Message) = .empty; try std.testing.expectError(error.OutOfMemory, appendHistoryMessages(compact_failing.allocator(), &compact_messages, &compact)); } -test "dupeImageAttachment frees path when media_type allocation fails" { - var failing = std.testing.FailingAllocator.init(testing_allocator.no_resize, .{ .fail_index = 1 }); - try std.testing.expectError(error.OutOfMemory, dupeImageAttachment(failing.allocator(), .{ - .path = @constCast("/tmp/a.png"), - .media_type = @constCast("image/png"), - })); -} -test "dupeImageAttachment preserves stable image id" { - const alloc = std.testing.allocator; - const copy = try dupeImageAttachment(alloc, .{ - .id = 41, - .path = @constCast("/tmp/a.png"), - .media_type = @constCast("image/png"), - }); - defer { - alloc.free(copy.path); - alloc.free(copy.media_type); - } - - try std.testing.expectEqual(@as(usize, 41), copy.id); -} test "conversation-language inference matches script signals" { try std.testing.expectEqual(ConversationLanguage.literal("und-Latn"), inferConversationLanguage("puoi aprire la landing page", ConversationLanguage.default())); try std.testing.expectEqual(ConversationLanguage.literal("ja"), inferConversationLanguage("ランディングページを開いて", ConversationLanguage.default())); @@ -4320,6 +4261,35 @@ test "active context and image catalog retain historical images" { try std.testing.expectEqualStrings("/tmp/historical.png", catalog[0].path); } +test "the image catalog keeps each image's handle and bytes, so Vision can read a resumed one (D89)" { + const alloc = std.testing.allocator; + const digest = "abababababababababababababababababababababababababababababababab"; + var resumed = [_]ImageAttachment{.{ + .id = 1, + .path = @constCast("/Users/example/old.png"), + .media_type = @constCast("image/png"), + .snapshot_sha256 = @constCast(digest), + .blob_path = @constCast("/Users/example/.fx/sessions/v2/s/blobs/" ++ digest), + }}; + const history = [_]HistoryTurn{.{ .assistant = .{ + .user = .{ .text = @constCast("old image"), .images = &resumed }, + .assistant = @constCast("old answer"), + } }}; + var unsaved_bytes = "png".*; + const current = [_]ImageAttachment{.{ + .id = 2, + .path = @constCast("/Users/example/new.png"), + .media_type = @constCast("image/png"), + .inline_data = &unsaved_bytes, + .source_ref = @constCast("host:new"), + }}; + const catalog = try collect_image_catalog(alloc, &history, ¤t); + defer core_types.freeImageAttachmentSlice(alloc, catalog); + try std.testing.expectEqualStrings(resumed[0].blob_path.?, catalog[0].blob_path.?); + try std.testing.expectEqualStrings("png", catalog[1].inline_data.?); + try std.testing.expectEqualStrings("host:new", catalog[1].source_ref.?); +} + test "full image catalog merges canonical history and current images" { const alloc = std.testing.allocator; var historical_images = [_]ImageAttachment{.{ diff --git a/src/core/session/session_adapter.zig b/src/core/session/session_adapter.zig index 69e06a989..8d3d58252 100644 --- a/src/core/session/session_adapter.zig +++ b/src/core/session/session_adapter.zig @@ -41,6 +41,7 @@ const sm = @import("session_manager"); const debug_trace = @import("../shared/debug_trace.zig"); const io_mod = @import("../shared/io.zig"); const types = @import("../shared/types.zig"); +const image_attachments = @import("../images/image_attachments.zig"); const image_data = @import("../images/image_data.zig"); const profile_paths = @import("../shared/profile_paths.zig"); const session_event = @import("session_event.zig"); @@ -1207,10 +1208,6 @@ pub const Session = struct { /// `~/.fx/session-files/{id}`, owned: where an older session kept its /// side files, read once by the move (D47). files_path: []u8, - /// The folder of this session's blobs once it is published, owned, as - /// `Manager.blobPath` joins it: each prompt image is the file named by - /// its hash there (D79). - blobs_path: []u8, /// The session's blobs, shared with every capability over it (D44). host: *BlobHost, capability: ?session_child_store.SessionChildCapability = null, @@ -1223,6 +1220,10 @@ pub const Session = struct { stored_results: std.StringHashMapUnmanaged([]u8) = .empty, /// Ids of the open turn's calls already saved as running (D28), owned. running: std.ArrayList([]u8) = .empty, + /// Prompt images saved as blobs (D89), digest to blob path, both owned. + /// The store deletes no blob of a live session, so an entry stays true, + /// and every copy of a turn the host passes encodes its images alike. + prompt_blobs: std.array_hash_map.String([]u8) = .empty, turn_open: bool = false, /// Highest turn number started; the manager numbers turns the same way. last_turn: u64 = 0, @@ -1441,12 +1442,10 @@ pub const Session = struct { fn init(alloc: Allocator, store: *Store, handle: sm.Session, fresh: bool) !*Session { const files_path = try std.Io.Dir.path.join(alloc, &.{ store.home, profile_paths.root_dir_name, files_dir_name, handle.id() }); errdefer alloc.free(files_path); - const blobs_path = try std.Io.Dir.path.join(alloc, &.{ store.manager.root_path, handle.id(), "blobs" }); - errdefer alloc.free(blobs_path); const host = try BlobHost.create(alloc, store, handle); errdefer BlobHost.release(host); const self = try alloc.create(Session); - self.* = .{ .alloc = alloc, .store = store, .handle = handle, .files_path = files_path, .blobs_path = blobs_path, .host = host, .fresh = fresh }; + self.* = .{ .alloc = alloc, .store = store, .handle = handle, .files_path = files_path, .host = host, .fresh = fresh }; return self; } @@ -1454,12 +1453,6 @@ pub const Session = struct { return self.handle.id(); } - /// Where requests say a prompt image is saved (D79): the blob a turn's - /// user item keeps it as, written before any of the turn's tools run. - pub fn blobsPath(self: *const Session) []const u8 { - return self.blobs_path; - } - /// Appends through the manager; a host session also traces the turn and /// child lines `Wiring.tla` models. The first item lists every blob the /// stores kept since the last such line, so verify and recover see a @@ -1564,12 +1557,16 @@ pub const Session = struct { self.streamed.deinit(alloc); self.stored_results.deinit(alloc); self.running.deinit(alloc); + for (self.prompt_blobs.keys(), self.prompt_blobs.values()) |digest, path| { + alloc.free(digest); + alloc.free(path); + } + self.prompt_blobs.deinit(alloc); self.turn_numbers.deinit(alloc); if (self.language) |value| alloc.free(value); if (self.instructions) |value| alloc.free(value); if (self.first_title) |value| alloc.free(value); alloc.free(self.files_path); - alloc.free(self.blobs_path); alloc.destroy(self); } @@ -1610,35 +1607,6 @@ pub const Session = struct { /// Opens the turn on disk if it is not open yet, so a body stored for it /// is a blob of a session on disk (D44). Caller holds the mutex. - /// Keeps each inline prompt image as one of the session's blobs too and - /// saves the prompt naming that file, which every request names for the - /// model and an image too large to send names in its notice (D79). The - /// bytes stay in the item (D44), and `promptImageBlobs` lists the blob - /// on it. Storing the same bytes again costs one stat, so a streamed - /// prompt encodes the same each time. The path - /// is the one `blobPath` gives once the session is published, so an - /// import's staged blobs (D59) get theirs too. The caller holds the - /// mutex. - fn keepPromptImages(self: *Session, a: Allocator, events: []Event) !void { - for (events) |*event| switch (event.*) { - .user => |*user| { - var kept: ?[]types.ImageAttachment = null; - for (user.images, 0..) |image, index| { - const bytes = image.inline_data orelse continue; - if (kept == null) { - // A blob needs the turn on disk first (D44). - try self.openTurnLocked(); - kept = try a.dupe(types.ImageAttachment, user.images); - } - const hash = try self.handle.putBlob(bytes); - kept.?[index].snapshot_path = try std.Io.Dir.path.join(a, &.{ self.blobs_path, &hash }); - } - if (kept) |list| user.images = list; - }, - else => {}, - }; - } - fn openTurnLocked(self: *Session) !void { if (self.turn_open) return; _ = try self.write(&.{.turn_started}); @@ -1708,14 +1676,13 @@ pub const Session = struct { var events: std.ArrayList(Event) = .empty; try session_event.appendHistoryTurnConversationEvents(a, &events, turn); if (events.items.len < 2) return error.InvalidConversationFrame; - try self.keepPromptImages(a, events.items); const pieces = events.items[0 .. events.items.len - 1]; const end = events.items[events.items.len - 1]; // The end piece joins the others; its line ends the turn. const all = events.items; const encoded = try a.alloc([]u8, all.len); - for (all, encoded) |piece, *bytes| bytes.* = try encodePiece(a, piece); + for (all, encoded) |piece, *bytes| bytes.* = try self.encodeEvent(a, piece); const start = try self.streamedPrefix(encoded[0..pieces.len]); var tail: std.ArrayList(sm.Event) = .empty; @@ -1755,6 +1722,96 @@ pub const Session = struct { self.fresh = false; } + /// Saves each prompt image of the turn as a blob named by its digest + /// before the turn's first request is built, and gives the image that + /// blob's path for the model's note (`AgentRuntimeDeps.save_prompt_images`, + /// D89, `tests/tla/session_manager/PromptImages.tla`). Each save lists + /// the blob again on the turn's next item, so a fork hard-links it even + /// after a crash lost an earlier listing. A saved image then keeps only + /// that handle, and readers load its bytes from the blob. An image the + /// store refuses keeps its bytes inline, as before D89, and gets no + /// path. An image that arrives as only a handle, perhaps to another + /// session's blob, is saved from that file. `alloc` owns `images` and + /// what this sets on them; the bytes an image arrived with are the + /// caller's and are not freed. + pub fn savePromptImages(self: *Session, alloc: Allocator, images: []types.ImageAttachment) !void { + self.mutex.lockUncancelable(io_mod.getIo()); + defer self.mutex.unlock(io_mod.getIo()); + for (images) |*image| { + // Bytes this read from the file a handle names, which `alloc` owns. + var loaded: ?[]u8 = null; + if (image.inline_data == null) { + const verified = image_attachments.loadVerifiedSnapshot(alloc, image.*, .{}) catch |err| switch (err) { + error.OutOfMemory => return err, + else => { + debug_trace.logf("session", "event=sessions_v2_prompt_image_unreadable session={s} image={d} reason={s}", .{ self.id(), image.id, @errorName(err) }); + image.blob_path = null; + continue; + }, + }; + loaded = verified.bytes; + image.inline_data = verified.bytes; + } + image.blob_path = null; + const bytes = image.inline_data.?; + const digest = image.snapshot_sha256 orelse continue; + const path = self.savePromptImageLocked(bytes, digest) catch |err| switch (err) { + error.OutOfMemory => return err, + else => { + debug_trace.logf("session", "event=sessions_v2_prompt_image_inline session={s} image={d} reason={s}", .{ self.id(), image.id, @errorName(err) }); + continue; + }, + }; + image.blob_path = try alloc.dupe(u8, path); + image.inline_data = null; + if (loaded) |value| alloc.free(value); + } + } + + /// The path of the blob holding `bytes`, owned by `prompt_blobs`. A turn + /// the store cannot start fails the save, as a refused blob does, so the + /// image stays inline and the prompt goes on. + fn savePromptImageLocked(self: *Session, bytes: []const u8, digest: []const u8) ![]const u8 { + // A blob needs the turn on disk (D44). + try self.openTurnLocked(); + const hash = try BlobHost.put(self.host, bytes); + // The digest names the blob: bytes that do not match it are not + // saved under it. + if (!std.mem.eql(u8, &hash, digest)) return error.ImageDigestMismatch; + if (self.prompt_blobs.get(digest)) |path| return path; + // The blob's path once the session is published, as + // `Manager.blobPath` joins it, so a conversion's staged import (D59) + // names its images too. The put above just stored the file. + const path = try std.Io.Dir.path.join(self.alloc, &.{ self.store.manager.root_path, self.id(), "blobs", &hash }); + errdefer self.alloc.free(path); + const key = try self.alloc.dupe(u8, digest); + errdefer self.alloc.free(key); + try self.prompt_blobs.put(self.alloc, key, path); + return path; + } + + /// `event` as this session writes it: an image is written as a blob + /// exactly when this session saved it as one, whichever copy of the turn + /// the host passes and whatever path the copy carries. So a commit + /// matches what was streamed, and an item never names another session's + /// blob (D89, `tests/tla/session_manager/PromptImages.tla` RefHasBlob). + fn encodeEvent(self: *Session, a: Allocator, event: Event) ![]u8 { + const user = switch (event) { + .user => |value| value, + else => return encodePiece(a, event), + }; + if (user.images.len == 0) return encodePiece(a, event); + const images = try a.dupe(types.ImageAttachment, user.images); + for (images) |*image| { + const digest = image.snapshot_sha256 orelse { + image.blob_path = null; + continue; + }; + image.blob_path = self.prompt_blobs.get(digest); + } + return encodePiece(a, .{ .user = .{ .text = user.text, .images = images, .work_id = user.work_id } }); + } + /// Streams the turn so far (`AgentRuntimeDeps.append_turn_piece`): the /// first call starts the turn, later calls append only the pieces /// completed since. A tool result gets the side file the commit would @@ -1776,7 +1833,6 @@ pub const Session = struct { const a = arena.allocator(); var events: std.ArrayList(Event) = .empty; try events.append(a, .{ .user = .{ .text = user.text, .images = user.images, .work_id = user.work_id } }); - try self.keepPromptImages(a, events.items); // A result's blob needs the turn on disk first (D44). if (storesBodies(execution)) try self.openTurnLocked(); // On a missing handle the list keeps the pieces before it. @@ -1785,7 +1841,7 @@ pub const Session = struct { else => return err, }; const encoded = try a.alloc([]u8, events.items.len); - for (events.items, encoded) |event, *bytes| bytes.* = try encodePiece(a, event); + for (events.items, encoded) |event, *bytes| bytes.* = try self.encodeEvent(a, event); const start = try self.streamedPrefix(encoded); if (!self.turn_open or start < encoded.len) { try self.writePieces(a, events.items[start..], encoded[start..], &.{}); @@ -1877,9 +1933,7 @@ pub const Session = struct { } } for (events, encoded) |event, bytes| { - var piece = try self.item(a, std.meta.activeTag(event), bytes); - if (event == .user) piece.blobs = try self.promptImageBlobs(a, piece.blobs, event.user.images); - try batch.append(a, .{ .item = piece }); + try batch.append(a, .{ .item = try self.item(a, std.meta.activeTag(event), bytes) }); } try batch.appendSlice(a, tail); if (batch.items.len == 0) return; @@ -1887,24 +1941,6 @@ pub const Session = struct { if (!self.turn_open) self.startedTurn(); } - /// `blobs` and the blob each kept prompt image names, so every blob is - /// listed on an item of the turn that stored it (D44, D79). - fn promptImageBlobs(self: *Session, a: Allocator, blobs: []const []const u8, images: []const types.ImageAttachment) ![]const []const u8 { - const folder = self.blobs_path; - var list: std.ArrayList([]const u8) = .empty; - for (images) |image| { - const path = image.snapshot_path orelse continue; - if (path.len != folder.len + 1 + @sizeOf(sm.BlobHash) or !std.mem.startsWith(u8, path, folder) or path[folder.len] != '/') continue; - const hash = path[folder.len + 1 ..]; - if (list.items.len == 0) try list.appendSlice(a, blobs); - const listed = for (list.items) |ref| { - if (std.mem.eql(u8, ref, hash)) break true; - } else false; - if (!listed) try list.append(a, hash); - } - return if (list.items.len == 0) blobs else list.items; - } - fn startedTurn(self: *Session) void { self.last_turn += 1; self.turn_open = true; @@ -1937,13 +1973,32 @@ pub const Session = struct { try self.importTurn(a, events.items); } + /// Saves each image of `events`' user items as this session's blob, as + /// a turn's start saves a prompt's (D89), so a converted session names + /// its images by blob too. An image the store refuses keeps its bytes + /// inline. The caller holds the mutex. + fn saveImportedImages(self: *Session, events: []const Event) !void { + for (events) |event| switch (event) { + .user => |user| for (user.images) |image| { + const bytes = image.inline_data orelse continue; + const digest = image.snapshot_sha256 orelse continue; + _ = self.savePromptImageLocked(bytes, digest) catch |err| { + if (err == error.OutOfMemory) return error.OutOfMemory; + debug_trace.logf("session", "event=sessions_v2_prompt_image_inline session={s} image={d} reason={s}", .{ self.id(), image.id, @errorName(err) }); + continue; + }; + }, + else => {}, + }; + } + /// `events`, a turn's pieces, ended by the last one. fn importTurn(self: *Session, a: Allocator, turn_events: []const Event) !void { if (turn_events.len < 2) return error.InvalidConversationFrame; const events = try a.dupe(Event, turn_events); - try self.keepPromptImages(a, events); + try self.saveImportedImages(events); const encoded = try a.alloc([]u8, events.len); - for (events, encoded) |event, *bytes| bytes.* = try encodePiece(a, event); + for (events, encoded) |event, *bytes| bytes.* = try self.encodeEvent(a, event); const end: sm.Event = switch (events[events.len - 1]) { .turn_completed => .turn_committed, .interrupted => |interruption| .{ .turn_interrupted = switch (interruption.reason) { @@ -2732,10 +2787,10 @@ const Source = struct { } } - /// `user` with the bytes of each image an older session kept in its side - /// folder (D47), from its blob: inside the turn, as a v2 session keeps - /// images now (D44). Other images are unchanged; an unreadable one keeps - /// its old path, as a lost snapshot does. + /// `user` with the blob path of each image an older session kept in its + /// side folder (D47), as a v2 session's own prompt images have it + /// (D89). Other images are unchanged; one whose blob is gone keeps its + /// old path, as a lost snapshot does. fn withMovedImages(src: Source, pa: Allocator, user: session_event.ConversationUser) !session_event.ConversationUser { const host = src.host orelse return user; const files_path = src.files_path orelse return user; @@ -2746,20 +2801,49 @@ const Source = struct { const path = image.snapshot_path orelse continue; if (path.len <= files_path.len + 1 or !std.mem.startsWith(u8, path, files_path) or path[files_path.len] != '/') continue; const hash = host.moved.get(path[files_path.len + 1 ..]) orelse continue; - const bytes = src.store.manager.getBlob(pa, src.id, &hash) catch |err| { - debug_trace.logf("session", "event=sessions_v2_moved_image_unreadable session={s} err={s}", .{ src.id, @errorName(err) }); - continue; + // The image is its own blob now, which readers load and the + // model's note names (D89). + const blob_path = src.store.manager.blobPath(pa, src.id, &hash) catch |err| switch (err) { + error.OutOfMemory => return err, + else => { + debug_trace.logf("session", "event=sessions_v2_moved_image_unreadable session={s} err={s}", .{ src.id, @errorName(err) }); + continue; + }, }; if (images == null) images = try pa.dupe(types.ImageAttachment, user.images); - images.?[index].inline_data = bytes; - // The v1 file went with its folder; the blob is the original now - // (D79). - images.?[index].snapshot_path = try src.store.manager.blobPath(pa, src.id, &hash); + images.?[index].blob_path = blob_path; } const list = images orelse return user; return .{ .text = user.text, .images = list, .work_id = user.work_id }; } + /// A `user` item, each image saved as its own blob given only that + /// blob's path, from which readers load its bytes when they need them + /// (D89). A fork reads the blobs it hard-linked here too. + fn decodeUser(src: Source, pa: Allocator, data: []const u8) !session_event.ConversationUser { + const wire = try std.json.parseFromSliceLeaky(WireUser, pa, data, .{ .allocate = .alloc_if_needed }); + const images = try wire.toImages(pa); + for (wire.images, images) |image, *out| { + const hash = image.blob orelse continue; + out.blob_path = src.store.manager.blobPath(pa, src.id, hash) catch |err| return imageBlobFailed(src, image.id, err); + } + return .{ .text = wire.text, .images = images, .work_id = wire.work_id }; + } + + /// An image blob the log names that is gone or misnamed damages the + /// session, as a piece's blob does (D39). One whose bytes were damaged is + /// found by `verify`, and by any reader, which checks them against the + /// image's digest (D89). + fn imageBlobFailed(src: Source, image_id: usize, err: sm.BlobError) (sm.BlobError || error{InvalidSessionFormat}) { + return switch (err) { + error.NotFound, error.Corrupt, error.InvalidArgument => { + debug_trace.logf("session", "event=sessions_v2_prompt_image_unreadable session={s} image={d} err={s}", .{ src.id, image_id, @errorName(err) }); + return error.InvalidSessionFormat; + }, + else => |e| e, + }; + } + /// A piece's bytes, from its blob when the line holds a reference. The /// item may list more blobs, the stores' bodies (D44); the reference is /// the one its data names. @@ -4186,8 +4270,13 @@ noinline fn replay( const data = try src.pieceData(pa, piece); // The builder copies what it keeps, as it does for v1's // reader, so strings may point into the page. + if (kind == .user) { + try builder.begin(try src.withMovedImages(pa, try src.decodeUser(pa, data))); + continue; + } switch (try decodePiece(pa, kind, data, .alloc_if_needed)) { - .user => |value| try builder.begin(try src.withMovedImages(pa, value)), + // Read above, with its images' blobs. + .user => unreachable, .assistant => |value| { running_text = null; try builder.appendAssistant(value); @@ -4656,7 +4745,9 @@ const WireUser = struct { images: []const WireImage = &.{}, work_id: ?[]const u8 = null, - /// `types.ImageAttachment` field for field, with `inline_data` in base64. + /// `types.ImageAttachment` field for field, with `inline_data` in base64, + /// or the image's own blob (D89): `blob` names it by its digest, and the + /// bytes, the digest, and the snapshot are not written again. const WireImage = struct { id: usize = 0, path: []const u8, @@ -4665,11 +4756,20 @@ const WireUser = struct { snapshot_sha256: ?[]const u8 = null, inline_data: ?[]const u8 = null, source_ref: ?[]const u8 = null, + blob: ?[]const u8 = null, pub fn jsonStringify(self: WireImage, writer: *std.json.Stringify) !void { try writer.beginObject(); inline for (@typeInfo(WireImage).@"struct".field_names) |field_name| { - if (!std.mem.eql(u8, field_name, "source_ref") or self.source_ref != null) { + // Written only when set, so an image without them encodes + // as it did before they existed. + const optional = comptime std.mem.eql(u8, field_name, "source_ref") or std.mem.eql(u8, field_name, "blob"); + // An image in its blob has no bytes, digest or snapshot here. + const in_blob = comptime std.mem.eql(u8, field_name, "snapshot_path") or + std.mem.eql(u8, field_name, "snapshot_sha256") or + std.mem.eql(u8, field_name, "inline_data"); + const write = if (optional) @field(self, field_name) != null else !(in_blob and self.blob != null); + if (write) { try writer.objectField(field_name); try writer.write(@field(self, field_name)); } @@ -4686,6 +4786,16 @@ const WireUser = struct { if (image.source_ref) |value| { if (!image_data.validSourceRef(value)) return error.InvalidSessionFormat; } + if (image.blob_path != null) { + wire.* = .{ + .id = image.id, + .path = image.path, + .media_type = image.media_type, + .source_ref = image.source_ref, + .blob = image.snapshot_sha256 orelse return error.InvalidSessionFormat, + }; + continue; + } wire.* = .{ .id = image.id, .path = image.path, @@ -4703,11 +4813,28 @@ const WireUser = struct { } fn toUser(wire: WireUser, a: Allocator) !session_event.ConversationUser { + return .{ .text = wire.text, .images = try wire.toImages(a), .work_id = wire.work_id }; + } + + /// The images, without the bytes of those in blobs: `Source.decodeUser` + /// reads those. + fn toImages(wire: WireUser, a: Allocator) ![]types.ImageAttachment { const images = try a.alloc(types.ImageAttachment, wire.images.len); for (wire.images, images) |image, *out| { if (image.source_ref) |value| { if (!image_data.validSourceRef(value)) return error.InvalidSessionFormat; } + if (image.blob) |hash| { + if (image.inline_data != null or image.snapshot_path != null or image.snapshot_sha256 != null) return error.InvalidSessionFormat; + out.* = .{ + .id = image.id, + .path = try a.dupe(u8, image.path), + .media_type = try a.dupe(u8, image.media_type), + .snapshot_sha256 = try a.dupe(u8, hash), + .source_ref = if (image.source_ref) |value| try a.dupe(u8, value) else null, + }; + continue; + } out.* = .{ .id = image.id, .path = try a.dupe(u8, image.path), @@ -4723,7 +4850,7 @@ const WireUser = struct { .source_ref = if (image.source_ref) |value| try a.dupe(u8, value) else null, }; } - return .{ .text = wire.text, .images = images, .work_id = wire.work_id }; + return images; } }; @@ -6392,11 +6519,24 @@ test "an older session moves off its side folder on its first writable open, and for (try records.list(arena)) |name| count += @intFromBool(std.mem.eql(u8, name, "compacted-T9.txt")); try testing.expectEqual(@as(usize, 1), count); } - // The image is inside the turn now, and the client's settings are settings. + // The image is its blob now, read back on demand, and the client's + // settings are settings. var restored = try r.restore(testing.allocator); defer restored.deinit(testing.allocator); const image = restored.history[0].assistant.user.images[0]; - try testing.expectEqualSlices(u8, &OldSideFolder.png, image.inline_data.?); + try testing.expectEqual(@as(?[]u8, null), image.inline_data); + { + var read = try image_attachments.loadVerifiedSnapshot(testing.allocator, image, .{}); + defer read.deinit(testing.allocator); + try testing.expectEqualSlices(u8, &OldSideFolder.png, read.bytes); + } + // Its blob is the file the model's note names (D89). + { + const hash = r.host.moved.get("images/" ++ OldSideFolder.image_name).?; + const path = try t.store.manager.blobPath(testing.allocator, id, &hash); + defer testing.allocator.free(path); + try testing.expectEqualStrings(path, image.blob_path.?); + } const prompt = (try r.clientPrompt(testing.allocator)).?; defer testing.allocator.free(prompt); try testing.expectEqualStrings("You run inside Mini.", prompt); @@ -6414,6 +6554,7 @@ test "an older session moves off its side folder on its first writable open, and defer without.deinit(testing.allocator); const lost = without.history[0].assistant.user.images[0]; try testing.expectEqual(@as(?[]u8, null), lost.inline_data); + try testing.expectEqual(@as(?[]u8, null), lost.blob_path); try testing.expect(std.mem.endsWith(u8, lost.snapshot_path.?, "images/" ++ OldSideFolder.image_name)); } @@ -6574,7 +6715,7 @@ fn userItemLists(t: *TestHome, id: []const u8, hash: []const u8) !bool { return error.TestUnexpectedResult; } -test "a prompt image's bytes ride inside the user's item, its saved path names its blob, and an image without them encodes as v1's (D44, D79)" { +test "a prompt image's bytes ride inside the user's item, and an image without them encodes as v1's (D44)" { var t: TestHome = undefined; try t.init(); defer t.deinit(); @@ -6613,16 +6754,7 @@ test "a prompt image's bytes ride inside the user's item, its saved path names i try testing.expectEqualSlices(u8, &bytes, image.inline_data.?); try testing.expectEqualStrings(&digest_hex, image.snapshot_sha256.?); try testing.expectEqualStrings(images[0].path, image.path); - // The saved prompt names the session's blob of the image, which the - // notice for an image too large to send names after a resume (D79). - const blob_path = try t.store.manager.blobPath(testing.allocator, id, &digest_hex); - defer testing.allocator.free(blob_path); - try testing.expectEqualStrings(blob_path, image.snapshot_path.?); - const kept = try t.store.manager.getBlob(testing.allocator, id, &digest_hex); - defer testing.allocator.free(kept); - try testing.expectEqualSlices(u8, &bytes, kept); - // Every blob is listed on an item of its turn (D44). - try testing.expect(try userItemLists(&t, id, &digest_hex)); + try testing.expectEqualStrings(images[0].snapshot_path.?, image.snapshot_path.?); if (source_ref) |value| { try testing.expectEqualStrings(value, image.source_ref.?); } else try testing.expectEqual(@as(?[]u8, null), image.source_ref); @@ -6647,6 +6779,240 @@ test "a prompt image's bytes ride inside the user's item, its saved path names i } } +/// Session `id`'s first `user` item, whether it lists blob `hash`, and how +/// many turns were superseded. The caller frees `data`. +const UserItem = struct { data: []u8, lists: bool, superseded: usize }; + +fn firstUserItem(manager: *sm.Manager, id: []const u8, hash: []const u8) !UserItem { + var page = try manager.read(testing.allocator, id, .start, .forward, 1000); + defer page.deinit(); + var found: ?UserItem = null; + var superseded: usize = 0; + for (page.entries) |entry| { + const body = entry.body orelse continue; + if (body != .item) continue; + if (std.mem.eql(u8, body.item.type, superseded_type)) superseded += 1; + if (found != null or !std.mem.eql(u8, body.item.type, itemType(.user).?)) continue; + const lists = for (body.item.blobs) |ref| { + if (std.mem.eql(u8, ref, hash)) break true; + } else false; + found = .{ .data = try testing.allocator.dupe(u8, body.item.data), .lists = lists, .superseded = 0 }; + } + var item = found orelse return error.TestUnexpectedResult; + item.superseded = superseded; + return item; +} + +test "a prompt image is saved as its own blob before the request, named in the user's item, and read back with its path (D89)" { + var t: TestHome = undefined; + try t.init(); + defer t.deinit(); + var model = "m".*; + const bytes = OldSideFolder.png ++ [_]u8{ 0x00, 0xff }; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(&bytes, &digest, .{}); + var digest_hex = std.fmt.bytesToHex(digest, .lower); + const original = [_]types.ImageAttachment{.{ + .id = 1, + .path = @constCast("/Users/example/shot.png"), + .media_type = @constCast("image/png"), + .snapshot_path = @constCast("/tmp/fx-image-snapshots-0000000000000001/image-1.bin"), + .snapshot_sha256 = &digest_hex, + .inline_data = @constCast(&bytes), + }}; + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const s = try Session.create(testing.allocator, &t.store, "/w", .app, testSeed(&model)); + const id = try testing.allocator.dupe(u8, s.id()); + defer testing.allocator.free(id); + + var images = original; + try s.savePromptImages(arena.allocator(), &images); + const blob_path = try t.store.manager.blobPath(testing.allocator, id, &digest_hex); + defer testing.allocator.free(blob_path); + try testing.expectEqualStrings(blob_path, images[0].blob_path.?); + // The turn keeps only the handle; the host's copy keeps its bytes. + try testing.expectEqual(@as(?[]u8, null), images[0].inline_data); + try testing.expect(original[0].inline_data != null); + // Before any request: the file the note names already holds the image. + { + var file = try std.Io.Dir.openFileAbsolute(io_mod.getIo(), blob_path, .{}); + defer file.close(io_mod.getIo()); + var buffer: [256]u8 = undefined; + const len = try file.readPositionalAll(io_mod.getIo(), &buffer, 0); + try testing.expectEqualSlices(u8, &bytes, buffer[0..len]); + } + // Saving the handle again reads it back and keeps the one blob and its + // path. + try s.savePromptImages(arena.allocator(), &images); + try testing.expectEqualStrings(blob_path, images[0].blob_path.?); + try testing.expectEqual(@as(?[]u8, null), images[0].inline_data); + + // Streamed from the saved copy, committed from the host's copy without + // paths: the same item, so the turn is not superseded. + try s.appendProgress(.{ .text = @constCast("look"), .images = &images }, .{}, .{}); + var turn = assistantTurn("look", "a png"); + var host_copy = original; + turn.assistant.user.images = &host_copy; + try s.commitTurn(turn, types.ConversationLanguage.default()); + s.close(); + + const item = try firstUserItem(t.store.manager, id, &digest_hex); + defer testing.allocator.free(item.data); + try testing.expectEqual(@as(usize, 0), item.superseded); + try testing.expect(item.lists); + const blob_field = try arena.allocator().print("\"blob\":\"{s}\"", .{&digest_hex}); + try testing.expect(std.mem.find(u8, item.data, blob_field) != null); + // Neither the bytes nor the process's temporary snapshot are written. + try testing.expect(std.mem.find(u8, item.data, "inline_data") == null); + try testing.expect(std.mem.find(u8, item.data, "fx-image-snapshots") == null); + try testing.expectEqual(@as(u64, 0), (try t.store.manager.verify(id)).bad_blobs); + + // Resumed, and in a fork that hard-links the blob, the image has only + // the path of its own session's blob, which reads back as its bytes. + const fork = try t.store.manager.openFork(.{ .source = id, .at = .{ .turn = 1 }, .workspace = "/w", .host = .app }); + const fork_id = try testing.allocator.dupe(u8, fork.id()); + defer testing.allocator.free(fork_id); + fork.release(); + for ([_][]const u8{ id, fork_id }) |which| { + const r = try Session.resumeSession(testing.allocator, &t.store, .{ .id = which }, "/w", .app); + defer r.close(); + var restored = try r.restore(testing.allocator); + defer restored.deinit(testing.allocator); + try testing.expectEqual(@as(usize, 1), restored.history.len); + const image = restored.history[0].assistant.user.images[0]; + try testing.expectEqual(@as(?[]u8, null), image.inline_data); + try testing.expectEqualStrings(&digest_hex, image.snapshot_sha256.?); + try testing.expectEqual(@as(?[]u8, null), image.snapshot_path); + try testing.expectEqualStrings(original[0].path, image.path); + const path = try t.store.manager.blobPath(testing.allocator, which, &digest_hex); + defer testing.allocator.free(path); + try testing.expectEqualStrings(path, image.blob_path.?); + var read = try image_attachments.loadVerifiedSnapshot(testing.allocator, image, .{}); + defer read.deinit(testing.allocator); + try testing.expectEqualSlices(u8, &bytes, read.bytes); + } + + // A blob the item names that is gone damages the session (D39). + try std.Io.Dir.deleteFileAbsolute(io_mod.getIo(), blob_path); + const r = try Session.resumeSession(testing.allocator, &t.store, .{ .id = id }, "/w", .app); + defer r.close(); + try testing.expectError(error.InvalidSessionFormat, r.restore(testing.allocator)); +} + +test "a prompt image the store does not save keeps its bytes inline and gets no path (D89)" { + var t: TestHome = undefined; + try t.init(); + defer t.deinit(); + var model = "m".*; + const bytes = OldSideFolder.png; + // A digest that is not the bytes' cannot name their blob. + const wrong_digest: [32]u8 = @splat(0xab); + var wrong = std.fmt.bytesToHex(wrong_digest, .lower); + var images = [_]types.ImageAttachment{ + .{ .id = 1, .path = @constCast("/Users/example/a.png"), .media_type = @constCast("image/png"), .snapshot_sha256 = &wrong, .inline_data = @constCast(&bytes) }, + // Bytes the session does not hold are not its to save. + .{ .id = 2, .path = @constCast("/Users/example/b.png"), .media_type = @constCast("image/png"), .snapshot_path = @constCast("/Users/example/.fx/sessions/s/images/image-2.bin") }, + }; + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const s = try Session.create(testing.allocator, &t.store, "/w", .app, testSeed(&model)); + const id = try testing.allocator.dupe(u8, s.id()); + defer testing.allocator.free(id); + try s.savePromptImages(arena.allocator(), &images); + try testing.expectEqual(@as(?[]u8, null), images[0].blob_path); + try testing.expectEqual(@as(?[]u8, null), images[1].blob_path); + var turn = assistantTurn("look", "two"); + turn.assistant.user.images = &images; + try s.commitTurn(turn, types.ConversationLanguage.default()); + s.close(); + + const item = try firstUserItem(t.store.manager, id, &wrong); + defer testing.allocator.free(item.data); + try testing.expect(std.mem.find(u8, item.data, "\"blob\"") == null); + const r = try Session.resumeSession(testing.allocator, &t.store, .{ .id = id }, "/w", .app); + defer r.close(); + var restored = try r.restore(testing.allocator); + defer restored.deinit(testing.allocator); + const kept = restored.history[0].assistant.user.images; + try testing.expectEqualSlices(u8, &bytes, kept[0].inline_data.?); + try testing.expectEqual(@as(?[]u8, null), kept[0].blob_path); + try testing.expectEqual(@as(?[]u8, null), kept[1].blob_path); + try testing.expectEqualStrings(images[1].snapshot_path.?, kept[1].snapshot_path.?); +} + +test "a turn the store cannot start keeps its prompt images inline and does not fail the prompt (D89)" { + var t: TestHome = undefined; + try t.init(); + defer t.deinit(); + var model = "m".*; + const bytes = OldSideFolder.png; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(&bytes, &digest, .{}); + var digest_hex = std.fmt.bytesToHex(digest, .lower); + var images = [_]types.ImageAttachment{.{ .id = 1, .path = @constCast("/Users/example/a.png"), .media_type = @constCast("image/png"), .snapshot_sha256 = &digest_hex, .inline_data = @constCast(&bytes) }}; + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const s = try Session.create(testing.allocator, &t.store, "/w", .app, testSeed(&model)); + defer s.close(); + // A new session reaches the disk with its first line, the turn's start, + // which a read-only store refuses. + const io = io_mod.getIo(); + try std.Io.Dir.cwd().createDirPath(io, t.store.manager.root_path); + var root = try std.Io.Dir.openDirAbsolute(io, t.store.manager.root_path, .{}); + defer root.close(io); + root.setPermissions(io, .fromMode(0o500)) catch return error.SkipZigTest; + defer root.setPermissions(io, .fromMode(0o700)) catch {}; + try s.savePromptImages(arena.allocator(), &images); + try testing.expectEqual(@as(?[]u8, null), images[0].blob_path); + try testing.expectEqualSlices(u8, &bytes, images[0].inline_data.?); +} + +test "an image that arrives with another session's blob path is saved here or written without a blob (D89)" { + var t: TestHome = undefined; + try t.init(); + defer t.deinit(); + var model = "m".*; + const bytes = OldSideFolder.png; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(&bytes, &digest, .{}); + var digest_hex = std.fmt.bytesToHex(digest, .lower); + const other_digest: [32]u8 = @splat(0xcd); + var other_hex = std.fmt.bytesToHex(other_digest, .lower); + const elsewhere = "/Users/example/.fx/sessions/v2/other/blobs/elsewhere"; + const original = [_]types.ImageAttachment{ + .{ .id = 1, .path = @constCast("/Users/example/a.png"), .media_type = @constCast("image/png"), .snapshot_sha256 = &digest_hex, .inline_data = @constCast(&bytes), .blob_path = @constCast(elsewhere) }, + // No bytes here to save, only another session's blob. + .{ .id = 2, .path = @constCast("/Users/example/b.png"), .media_type = @constCast("image/png"), .snapshot_path = @constCast("/Users/example/b.bin"), .snapshot_sha256 = &other_hex, .blob_path = @constCast(elsewhere) }, + }; + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const s = try Session.create(testing.allocator, &t.store, "/w", .app, testSeed(&model)); + const id = try testing.allocator.dupe(u8, s.id()); + defer testing.allocator.free(id); + var images = original; + try s.savePromptImages(arena.allocator(), &images); + const own = try t.store.manager.blobPath(testing.allocator, id, &digest_hex); + defer testing.allocator.free(own); + try testing.expectEqualStrings(own, images[0].blob_path.?); + try testing.expectEqual(@as(?[]u8, null), images[1].blob_path); + + // Committed from a copy that still carries the other session's paths. + var turn = assistantTurn("look", "two"); + var host_copy = original; + turn.assistant.user.images = &host_copy; + try s.commitTurn(turn, types.ConversationLanguage.default()); + s.close(); + const item = try firstUserItem(t.store.manager, id, &digest_hex); + defer testing.allocator.free(item.data); + const own_ref = try arena.allocator().print("\"blob\":\"{s}\"", .{&digest_hex}); + try testing.expect(std.mem.find(u8, item.data, own_ref) != null); + const other_ref = try arena.allocator().print("\"blob\":\"{s}\"", .{&other_hex}); + try testing.expect(std.mem.find(u8, item.data, other_ref) == null); + try testing.expect(std.mem.find(u8, item.data, elsewhere) == null); + try testing.expectEqual(@as(u64, 0), (try t.store.manager.verify(id)).bad_blobs); +} + test "v2 image source refs reject malformed metadata" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -6934,7 +7300,7 @@ test "a converted v1 session keeps its title, permission rules and preferences ( try testing.expectEqualStrings(saved.get("model").?.string, restored.preferences.?.model); } -test "a converted v1 session's side files keep their old names, and its prompt images ride inside the prompt and name their blob (D44, D47, D79)" { +test "a converted v1 session's side files keep their old names, and its prompt images become its blobs (D44, D47, D89)" { const alloc = testing.allocator; var t: TestHome = undefined; try t.init(); @@ -6957,14 +7323,23 @@ test "a converted v1 session's side files keep their old names, and its prompt i var restored = try s.restore(alloc); defer restored.deinit(alloc); const image = restored.history[0].assistant.user.images[0]; - try testing.expectEqualSlices(u8, image_bytes, image.inline_data.?); - // Its v1 file went with the folder; it names its blob (D79). + try testing.expectEqual(@as(?[]u8, null), image.inline_data); + { + var read = try image_attachments.loadVerifiedSnapshot(alloc, image, .{}); + defer read.deinit(alloc); + try testing.expectEqualSlices(u8, image_bytes, read.bytes); + } + // Its v1 file went with the folder. The converted prompt names the + // image by its blob, listed on the user's item, as a new prompt does + // (D89). var digest: [32]u8 = undefined; std.crypto.hash.sha2.Sha256.hash(image_bytes, &digest, .{}); const image_hash = std.fmt.bytesToHex(digest, .lower); const blob_path = try t.store.manager.blobPath(alloc, id, &image_hash); defer alloc.free(blob_path); - try testing.expectEqualStrings(blob_path, image.snapshot_path.?); + try testing.expectEqualStrings(blob_path, image.blob_path.?); + try testing.expectEqual(@as(?[]u8, null), image.snapshot_path); + try testing.expect(try userItemLists(&t, id, &image_hash)); } test "a converted v1 session keeps its ACP client's prompt and tool identities (D46)" { @@ -7740,10 +8115,14 @@ test "an image the oldest format saved as only its path is captured from its fil try testing.expectEqualStrings("look [Image #1] [Image #2]", user.text); try testing.expectEqual(@as(usize, 1), user.images.len); try testing.expectEqual(@as(usize, 1), user.images[0].id); - try testing.expectEqualSlices(u8, bytes, user.images[0].inline_data.?); var digest: [32]u8 = undefined; std.crypto.hash.sha2.Sha256.hash(bytes, &digest, .{}); try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), user.images[0].snapshot_sha256.?); + // The captured image converts to the session's blob, as any does (D89). + try testing.expect(user.images[0].blob_path != null); + var read = try image_attachments.loadVerifiedSnapshot(alloc, user.images[0], .{}); + defer read.deinit(alloc); + try testing.expectEqualSlices(u8, bytes, read.bytes); } test "images the oldest format saved without an id get the numbers v1's resume gave them (D69)" { diff --git a/src/core/session/session_store.zig b/src/core/session/session_store.zig index c69c147c8..b2d37eba9 100644 --- a/src/core/session/session_store.zig +++ b/src/core/session/session_store.zig @@ -42,9 +42,9 @@ pub const validateSessionId = paths.validateSessionId; pub const generateSessionId = paths.generateSessionId; /// The process's temporary folder for captured prompt images, made on first -/// use. The turn keeps the bytes (D44); until the prompt is saved, a -/// withheld image's notice names the copy here (D79). The caller frees the -/// result; `temp_dir` keeps the folder's path. +/// use. The turn keeps the bytes (D44), and a saved session saves them as +/// its blob (D89), which requests name instead of a copy here. The caller +/// frees the result; `temp_dir` keeps the folder's path. pub fn imageSnapshotTempDir(alloc: Allocator, temp_dir: *?[]u8) ![]u8 { if (temp_dir.* == null) temp_dir.* = try image_attachments.createTempSnapshotDir(alloc); return alloc.dupe(u8, temp_dir.*.?); diff --git a/src/core/shared/types.zig b/src/core/shared/types.zig index d3456518e..d443ac00f 100644 --- a/src/core/shared/types.zig +++ b/src/core/shared/types.zig @@ -1519,12 +1519,19 @@ pub const ImageAttachment = struct { inline_data: ?[]u8 = null, /// Owned opaque host reference, never filesystem or network authority. source_ref: ?[]u8 = null, + /// Owned path of the v2 session's blob that holds these bytes (D89), + /// which the model's note names so its tools can open the image. An + /// image saved as a blob keeps only this handle: readers load its bytes + /// from the blob when they need them, checked against its digest. The + /// session store owns the file: fx never writes or deletes through this + /// path. + blob_path: ?[]u8 = null, /// Whether the session no longer holds this image: a libfx session with /// an image store keeps only its newest images, and an older one keeps /// its digest, which names it in the host's store, without its bytes. pub fn droppedFromSession(self: ImageAttachment) bool { - return self.inline_data == null and self.snapshot_path == null and self.snapshot_sha256 != null; + return self.inline_data == null and self.snapshot_path == null and self.blob_path == null and self.snapshot_sha256 != null; } }; @@ -3302,47 +3309,59 @@ pub fn dupeImageAttachmentSlice(alloc: std.mem.Allocator, attachments: []const I } for (attachments, 0..) |attachment, i| { - const path = try alloc.dupe(u8, attachment.path); - errdefer alloc.free(path); - - const media_type = try alloc.dupe(u8, attachment.media_type); - errdefer alloc.free(media_type); - - const snapshot_path = if (attachment.snapshot_path) |value| - try alloc.dupe(u8, value) - else - null; - errdefer if (snapshot_path) |value| alloc.free(value); - - const snapshot_sha256 = if (attachment.snapshot_sha256) |value| - try alloc.dupe(u8, value) - else - null; - errdefer if (snapshot_sha256) |value| alloc.free(value); - - const inline_data = if (attachment.inline_data) |value| - try alloc.dupe(u8, value) - else - null; - errdefer if (inline_data) |value| alloc.free(value); - const source_ref = if (attachment.source_ref) |value| - try alloc.dupe(u8, value) - else - null; - copy[i] = .{ - .id = attachment.id, - .path = path, - .media_type = media_type, - .snapshot_path = snapshot_path, - .snapshot_sha256 = snapshot_sha256, - .inline_data = inline_data, - .source_ref = source_ref, - }; + copy[i] = try dupeImageAttachment(alloc, attachment); copied += 1; } return copy; } +/// A copy of `attachment` that owns every field it has; free it with +/// `freeImageAttachment`. +pub fn dupeImageAttachment(alloc: std.mem.Allocator, attachment: ImageAttachment) !ImageAttachment { + const path = try alloc.dupe(u8, attachment.path); + errdefer alloc.free(path); + + const media_type = try alloc.dupe(u8, attachment.media_type); + errdefer alloc.free(media_type); + + const snapshot_path = if (attachment.snapshot_path) |value| + try alloc.dupe(u8, value) + else + null; + errdefer if (snapshot_path) |value| alloc.free(value); + + const snapshot_sha256 = if (attachment.snapshot_sha256) |value| + try alloc.dupe(u8, value) + else + null; + errdefer if (snapshot_sha256) |value| alloc.free(value); + + const inline_data = if (attachment.inline_data) |value| + try alloc.dupe(u8, value) + else + null; + errdefer if (inline_data) |value| alloc.free(value); + const source_ref = if (attachment.source_ref) |value| + try alloc.dupe(u8, value) + else + null; + errdefer if (source_ref) |value| alloc.free(value); + const blob_path = if (attachment.blob_path) |value| + try alloc.dupe(u8, value) + else + null; + return .{ + .id = attachment.id, + .path = path, + .media_type = media_type, + .snapshot_path = snapshot_path, + .snapshot_sha256 = snapshot_sha256, + .inline_data = inline_data, + .source_ref = source_ref, + .blob_path = blob_path, + }; +} + /// Frees the owned fields in one attachment; caller owns any containing storage. pub fn freeImageAttachment(alloc: std.mem.Allocator, attachment: ImageAttachment) void { alloc.free(attachment.path); @@ -3351,6 +3370,7 @@ pub fn freeImageAttachment(alloc: std.mem.Allocator, attachment: ImageAttachment if (attachment.snapshot_sha256) |sha256| alloc.free(sha256); if (attachment.inline_data) |data| alloc.free(data); if (attachment.source_ref) |source_ref| alloc.free(source_ref); + if (attachment.blob_path) |path| alloc.free(path); } pub fn freeImageAttachmentSlice(alloc: std.mem.Allocator, attachments: []ImageAttachment) void { @@ -3644,6 +3664,29 @@ test "ImageAttachment helpers duplicate empty and populated slices" { freeImageAttachmentSlice(alloc, originals); } +test "dupeImageAttachment frees path when media_type allocation fails" { + var failing = std.testing.FailingAllocator.init(testing_allocator.no_resize, .{ .fail_index = 1 }); + try std.testing.expectError(error.OutOfMemory, dupeImageAttachment(failing.allocator(), .{ + .path = @constCast("/tmp/a.png"), + .media_type = @constCast("image/png"), + })); +} + +test "dupeImageAttachment preserves stable image id" { + const alloc = std.testing.allocator; + const copy = try dupeImageAttachment(alloc, .{ + .id = 41, + .path = @constCast("/tmp/a.png"), + .media_type = @constCast("image/png"), + }); + defer { + alloc.free(copy.path); + alloc.free(copy.media_type); + } + + try std.testing.expectEqual(@as(usize, 41), copy.id); +} + test "ImageAttachment source references clone independently and clean up allocation failures" { const originals = [_]ImageAttachment{ .{ diff --git a/src/core/terminal/client.zig b/src/core/terminal/client.zig index 346d9c743..cd9d0df7a 100644 --- a/src/core/terminal/client.zig +++ b/src/core/terminal/client.zig @@ -1139,6 +1139,15 @@ const LiveTerminalFixture = struct { /// Starts `command` in a clean bash and returns its owned session id. fn start(fixture: *LiveTerminalFixture, command: []const u8) ![]u8 { + return fixture.startInShell("/bin/bash", command); + } + + /// Starts `command` in a clean `shell_path` and returns its owned session id. + fn startInShell( + fixture: *LiveTerminalFixture, + shell_path: []const u8, + command: []const u8, + ) ![]u8 { const alloc = std.testing.allocator; if (fixture.persistence) |*previous| previous.deinit(); fixture.persistence = try operation.prepareStartPersistence(alloc, .{ @@ -1155,7 +1164,7 @@ const LiveTerminalFixture = struct { var started = try fixture.run(.{ .start = .{ .cwd = fixture.home, .command = command, - .shell = .{ .executable = .{ .path = "/bin/bash", .clean_start = true } }, + .shell = .{ .executable = .{ .path = shell_path, .clean_start = true } }, .backend = .native, .return_when = .started, .wait_ceiling_ms = 15_000, @@ -1375,6 +1384,42 @@ test "in-process registry starts writes to and stops a real terminal" { ); } +test "zsh terminal commands keep words that start with = literal" { + std.Io.Dir.accessAbsolute(io_mod.getIo(), "/bin/zsh", .{}) catch + return error.SkipZigTest; + var fixture: LiveTerminalFixture = undefined; + try fixture.init(); + defer fixture.deinit(); + const alloc = std.testing.allocator; + + const session_id = try fixture.startInShell( + "/bin/zsh", + "echo before; echo ======; [ x == x ] && echo same; echo after", + ); + defer alloc.free(session_id); + try std.testing.expectEqual( + contracts.ReturnOutcome{ .exited = 0 }, + try fixture.waitForExit(session_id), + ); + + var read = try fixture.run(.{ .read = .{ + .session_id = session_id, + .cursor = .{ .segment = 1, .offset = 0 }, + .authority = fixture.claim(), + } }); + defer read.deinit(alloc); + const output = switch (read.view()) { + .success => |success| switch (success) { + .read => |value| value.output, + else => return error.TestUnexpectedResult, + }, + .failure => return error.TestUnexpectedResult, + }; + try std.testing.expect(std.mem.find(u8, output, "======") != null); + try std.testing.expect(std.mem.find(u8, output, "same") != null); + try std.testing.expect(std.mem.find(u8, output, "after") != null); +} + test "closing owned terminals for exit leaves no orphan and records them ended" { var fixture: LiveTerminalFixture = undefined; try fixture.init(); diff --git a/src/core/terminal/native_session.zig b/src/core/terminal/native_session.zig index 3ff7a7983..c407f16c6 100644 --- a/src/core/terminal/native_session.zig +++ b/src/core/terminal/native_session.zig @@ -1682,6 +1682,7 @@ const Session = struct { defer self.alloc.free(executable); const bootstrap = try shell_resolver.buildBootstrap( self.alloc, + shell_resolver.shellKind(invocation.path) orelse return error.UnsupportedShell, executable, control_path, &nonce, diff --git a/src/core/terminal/shell_resolver.zig b/src/core/terminal/shell_resolver.zig index f45dac5ad..4cc5a6c47 100644 --- a/src/core/terminal/shell_resolver.zig +++ b/src/core/terminal/shell_resolver.zig @@ -185,6 +185,14 @@ pub fn profileShell( const captured_zsh_user_prelude = "\\builtin trap - TERM; "; +/// zsh's `equals` option expands an unquoted word that starts with `=`, such +/// as a `======` separator or the `==` in `[ a == b ]`, to the path of the +/// command it names, and a word that names no command aborts the rest of the +/// command line. Commands are written for POSIX shells, so every zsh launch +/// turns the option off right before the command, after startup files or a +/// snapshot replay have run. +const zsh_command_prelude = "\\builtin unsetopt equals; "; + pub fn capturedInvocation( alloc: Allocator, environment_value: Environment, @@ -198,7 +206,10 @@ pub fn capturedInvocation( .clean_start = true, } }); removeInteractiveFlag(&invocation); - invocation.setCommand(command); + invocation.setCommand(if (shellKind(path) == .zsh) + try std.mem.concat(alloc, u8, &.{ zsh_command_prelude, command }) + else + command); return invocation; }, .user => |path| { @@ -209,7 +220,7 @@ pub fn capturedInvocation( invocation.append("expand_aliases"); } const effective_command = if (shellKind(path) == .zsh) - try std.mem.concat(alloc, u8, &.{ captured_zsh_user_prelude, command }) + try std.mem.concat(alloc, u8, &.{ captured_zsh_user_prelude, zsh_command_prelude, command }) else command; invocation.setCommand(effective_command); @@ -270,6 +281,7 @@ pub fn snapshotInvocation( /// runs with stdin from /dev/null so it cannot read the rest of the script. pub fn snapshotScript( alloc: Allocator, + kind: ShellKind, replay: []const u8, command: []const u8, ) Allocator.Error![]u8 { @@ -277,7 +289,9 @@ pub fn snapshotScript( errdefer output.deinit(alloc); try output.appendSlice(alloc, replay); if (replay.len != 0 and replay[replay.len - 1] != '\n') try output.append(alloc, '\n'); - try output.appendSlice(alloc, "__fx_snapshot_restored=1\n\\builtin eval -- "); + try output.appendSlice(alloc, "__fx_snapshot_restored=1\n"); + if (kind == .zsh) try output.appendSlice(alloc, zsh_command_prelude); + try output.appendSlice(alloc, "\\builtin eval -- "); try appendShellWord(&output, alloc, command); try output.appendSlice(alloc, " if (whole_output and std.mem.eql(u8, std.mem.trim(u8, output, " \t\r\n"), trimmed)) return zsh_parse_retry_guidance, + .line_by_line => return line_shell_parse_retry_guidance, + } } - return false; + return null; } // Shell error prefixes may carry an absolute argv[0] path (fx spawns the // resolved login shell as /bin/bash, /bin/zsh, ...), so match the basename // of the token before the first colon rather than a raw line prefix. -fn hasShellErrorPrefix(line: []const u8) bool { - const colon = std.mem.findScalar(u8, line, ':') orelse return false; +// Snapshot and terminal runs pass the command to `eval`, so zsh names the +// source `(eval)`, as it does for an `eval` inside the command. Under the +// snapshot, bash names the script it sources from `/dev/fd/0` +// (`snapshotScript` in shell_resolver.zig), followed by `eval:`. Anchoring +// bash to that path keeps the `eval` errors of a script the command runs out. +fn shellErrorFamily(line: []const u8) ?ShellFamily { + const colon = std.mem.findScalar(u8, line, ':') orelse return null; const token = line[0..colon]; - if (token.len == 0 or std.mem.findScalar(u8, token, ' ') != null) return false; + if (token.len == 0 or std.mem.findScalar(u8, token, ' ') != null) return null; + if (std.mem.eql(u8, token, "(eval)")) return .zsh; + if (std.mem.eql(u8, token, "/dev/fd/0") and std.mem.startsWith(u8, line[colon + 1 ..], " eval:")) return .line_by_line; const base = if (std.mem.findLast(u8, token, "/")) |slash| token[slash + 1 ..] else token; - const names = [_][]const u8{ "zsh", "bash", "sh", "dash" }; - for (names) |name| { - if (std.mem.eql(u8, base, name)) return true; + if (std.mem.eql(u8, base, "zsh")) return .zsh; + const line_shells = [_][]const u8{ "bash", "sh", "dash" }; + for (line_shells) |name| { + if (std.mem.eql(u8, base, name)) return .line_by_line; } - return false; + return null; } fn hasUsageBannerOutput(output: []const u8) bool { @@ -1706,7 +1739,11 @@ fn formatSnapshotRaw( "Command output is incomplete. Inspect external state and available output before retrying; do not blindly rerun a command that may have changed state." else switch (snapshot.state) { .lost => "Execution status is indeterminate. Inspect external state before retrying; do not blindly rerun a command that may have changed state.", - .completed => failureRetryGuidance(projection.exit_code, snapshot.output_delta), + .completed => failureRetryGuidance( + projection.exit_code, + snapshot.output_delta, + snapshot.stdout_bytes +| snapshot.stderr_bytes == snapshot.output_delta.len, + ), .running, .stopped => null, }; var out: std.Io.Writer.Allocating = .init(alloc); @@ -2386,6 +2423,7 @@ test "completed shell snapshot with shell parse error carries rewrite guidance" .state = .{ .completed = .{ .exit_code = 1 } }, .output_delta = @constCast("zsh:1: unmatched '\n"), .output_truncated = false, + .stderr_bytes = "zsh:1: unmatched '\n".len, }, null); defer alloc.free(body); @@ -2395,11 +2433,33 @@ test "completed shell snapshot with shell parse error carries rewrite guidance" try std.testing.expectEqualStrings("completed", object.get("state").?.string); try std.testing.expectEqual(@as(i64, 1), object.get("exit_code").?.integer); try std.testing.expectEqualStrings( - shell_parse_retry_guidance, + zsh_parse_retry_guidance, object.get("retry_guidance").?.string, ); } +test "completed zsh parse error after earlier delivered output carries no guidance" { + // An earlier `running` observation already delivered `built\n`, so the + // command ran before an `eval` inside it failed to parse. + const alloc = std.testing.allocator; + const body = try formatSnapshot(alloc, .{ + .execution_id = @constCast("shell-parse-error-later"), + .command = @constCast("echo built; sleep 2; eval 'echo \"'"), + .cwd = @constCast("/tmp"), + .retained = false, + .state = .{ .completed = .{ .exit_code = 1 } }, + .output_delta = @constCast("(eval):1: unmatched \"\n"), + .output_truncated = false, + .stdout_bytes = "built\n".len, + .stderr_bytes = "(eval):1: unmatched \"\n".len, + }, null); + defer alloc.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, alloc, body, .{}); + defer parsed.deinit(); + try std.testing.expect(parsed.value.object.get("retry_guidance").? == .null); +} + test "completed shell snapshot with usage error carries argument guidance" { const alloc = std.testing.allocator; const body = try formatSnapshot(alloc, .{ @@ -2488,53 +2548,101 @@ test "incomplete output guidance takes precedence over failure guidance" { test "failure guidance detectors match only their signatures" { try std.testing.expectEqual( - shell_parse_retry_guidance, - failureRetryGuidance(1, "zsh:1: unmatched '\n"), + zsh_parse_retry_guidance, + failureRetryGuidance(1, "zsh:1: unmatched '\n", true), ); // fx spawns the resolved login shell with an absolute argv[0], so bash // errors carry a /bin/bash prefix; bare-basename forms also occur when a // command spawns a shell subprocess itself. try std.testing.expectEqual( - shell_parse_retry_guidance, - failureRetryGuidance(2, "/bin/bash: -c: line 1: syntax error: unexpected end of file\n"), + line_shell_parse_retry_guidance, + failureRetryGuidance(2, "/bin/bash: -c: line 1: syntax error: unexpected end of file\n", true), ); try std.testing.expectEqual( - shell_parse_retry_guidance, - failureRetryGuidance(2, "bash: -c: line 1: syntax error near unexpected token `)'\n"), + line_shell_parse_retry_guidance, + failureRetryGuidance(2, "bash: -c: line 1: syntax error near unexpected token `)'\n", true), ); try std.testing.expectEqual( - shell_parse_retry_guidance, - failureRetryGuidance(2, "/bin/dash: 1: Syntax error: Unterminated quoted string\n"), + line_shell_parse_retry_guidance, + failureRetryGuidance(2, "/bin/dash: 1: Syntax error: Unterminated quoted string\n", true), ); try std.testing.expectEqual( - shell_parse_retry_guidance, - failureRetryGuidance(1, "zsh: parse error near '\\n'\n"), + zsh_parse_retry_guidance, + failureRetryGuidance(1, "zsh: parse error near '\\n'\n", true), + ); + // Snapshot and terminal runs evaluate the command, so zsh reports `(eval)` + // and bash reports the sourced snapshot script followed by `eval:`. + try std.testing.expectEqual( + zsh_parse_retry_guidance, + failureRetryGuidance(1, "(eval):1: unmatched \"\n", true), + ); + try std.testing.expectEqual( + zsh_parse_retry_guidance, + failureRetryGuidance(1, "(eval):1: parse error near `)'\r\n", true), + ); + try std.testing.expectEqual( + line_shell_parse_retry_guidance, + failureRetryGuidance(1, "/dev/fd/0: eval: line 2: unexpected EOF while looking for matching `\"'\n/dev/fd/0: eval: line 3: syntax error: unexpected end of file\n", true), + ); + // bash 5.2 prints only the unexpected EOF line for an unclosed quote. + try std.testing.expectEqual( + line_shell_parse_retry_guidance, + failureRetryGuidance(1, "/dev/fd/0: eval: line 40: unexpected EOF while looking for matching `\"'\n", true), + ); + try std.testing.expectEqual( + line_shell_parse_retry_guidance, + failureRetryGuidance(2, "/bin/bash: -c: line 1: unexpected EOF while looking for matching `\"'\n", true), ); try std.testing.expectEqual( usage_error_retry_guidance, - failureRetryGuidance(2, "usage: grep [-abcdDEFGHhIiJLlMmnOopqRSsUVvwXxZz]\n"), + failureRetryGuidance(2, "usage: grep [-abcdDEFGHhIiJLlMmnOopqRSsUVvwXxZz]\n", true), ); try std.testing.expectEqual( usage_error_retry_guidance, - failureRetryGuidance(64, "Usage: ls [-ABCFGHabcdfghiklmnopqrstuvwx1] [file ...]\n"), + failureRetryGuidance(64, "Usage: ls [-ABCFGHabcdfghiklmnopqrstuvwx1] [file ...]\n", true), ); try std.testing.expectEqual( usage_error_retry_guidance, - failureRetryGuidance(2, "grep: unknown option\nusage: grep [-abcdDEFGHhIiJLlMmnOopqRSsUVvwXxZz]\n"), + failureRetryGuidance(2, "grep: unknown option\nusage: grep [-abcdDEFGHhIiJLlMmnOopqRSsUVvwXxZz]\n", true), ); // Exit zero never carries failure guidance. - try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(0, "usage: grep\n")); - try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(null, "zsh:1: unmatched '\n")); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(0, "usage: grep\n", true)); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(null, "zsh:1: unmatched '\n", true)); // A compiler-style syntax error without a shell prefix is not a shell parse error. - try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "main.c:4:5: error: expected ';' after expression (syntax error)\n")); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "main.c:4:5: error: expected ';' after expression (syntax error)\n", true)); // A non-shell token before the colon is not a shell prefix even when a shell name appears later. - try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "error: bash: syntax error\n")); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "error: bash: syntax error\n", true)); + // An eval source alone is not enough; the line must still name a parse error. + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "(eval):1: no such file or directory: missing\n", true)); + // A bash run-time error under the snapshot names `/dev/fd/0` without `eval:`. + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "/dev/fd/0: line 3: 1 + : syntax error: operand expected (error token is \" \")\n", true)); + // zsh's `[` builtin reports a bad expression at run time, after the command + // parsed. A `[[` test missing an operand prints the same text and is dropped too. + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "(eval):3: parse error: condition expected: =\n", true)); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "zsh:2: parse error: condition expected: $a\n", true)); + // Other output beside a zsh parse error means something ran, so the error + // may have come from an `eval` in a startup file or in the command. + // Captured runs can also place later stdout before stderr, so the order + // of that output proves nothing. + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "(eval):1: unmatched \"\r\nls: /missing: No such file or directory\r\n", true)); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "BEFORE\nzsh:1: unmatched '\nAFTER\n", true)); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "RAN\n(eval):1: unmatched \"\n", true)); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "zsh:1: unmatched \"\nbye\n", true)); + // Output an earlier observation delivered counts as other output for zsh. + // bash guidance already allows for lines that ran. + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "(eval):1: unmatched \"\n", false)); + try std.testing.expectEqual( + line_shell_parse_retry_guidance, + failureRetryGuidance(1, "/dev/fd/0: eval: line 2: unexpected EOF while looking for matching `\"'\n", false), + ); + // The `eval` of a bash script the command runs is not the snapshot's. + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(3, "./build.sh: eval: line 4: unexpected EOF while looking for matching `\"'\n", true)); // Ordinary failures keep null guidance. - try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(2, "grep: /nonexistent: No such file or directory\n")); - try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "")); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(2, "grep: /nonexistent: No such file or directory\n", true)); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "", true)); // A usage-looking line deep in output is not an argument rejection banner. - try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "line one\nline two\nline three\nline four\nusage: not a banner\n")); + try std.testing.expectEqual(@as(?[]const u8, null), failureRetryGuidance(1, "line one\nline two\nline three\nline four\nusage: not a banner\n", true)); } test "shell snapshot keeps bounded head tail and control metadata" { diff --git a/tests/e2e/acp.test.ts b/tests/e2e/acp.test.ts index 432fa1ec9..5a313dd46 100644 --- a/tests/e2e/acp.test.ts +++ b/tests/e2e/acp.test.ts @@ -1073,6 +1073,118 @@ for (const [name, finalText] of [ }, 45000); } +test("ACP shell completed and replayed updates show the command output", async () => { + const fixture = createConfiguredProviderFixture(body => body.messages.some((message: any) => message.role === "tool") + ? configuredCompletion(body.model, "done") + : configuredToolCompletion(body.model, "shell", { request: { action: "run", command: "echo hello-$((6*7))" } }, "call_shell")); + const shellText = (lines: any[]) => lines + .filter((message: any) => message.method === "session/update" && message.params?.update?.toolCallId === "call_shell") + .map((message: any) => message.params.update) + .filter((update: any) => update.status === "completed") + .map((update: any) => update.content); + let client: AcpClient | undefined; + try { + client = await AcpClient.create({ cwd: fixture.workspace, env: fixture.env }); + await client.request("initialize", { protocolVersion: 1 }, 1); + const created = await client.request("session/new", { cwd: fixture.workspace, mcpServers: [] }, 2) as any; + if (created.error) throw new Error(JSON.stringify(created)); + await client.request("session/set_mode", { modeId: "full-access" }, 3); + const prompt = await runPrompt(client, "Run the command.", TIMEOUT); + expect(prompt.promptResult.result.stopReason).toBe("end_turn"); + const expected = [{ type: "content", content: { type: "text", text: "hello-42\n" } }]; + expect(shellText(prompt.messages)).toEqual([expected]); + expect(client.stderr).toBe(""); + client.endStdin(); + expect(await client.waitForExit()).toBe(0); + await client.close(); + + client = await AcpClient.create({ cwd: fixture.workspace, env: fixture.env }); + await client.request("initialize", { protocolVersion: 1 }, 1); + const loaded = await client.request("session/load", { sessionId: created.result.sessionId, cwd: fixture.workspace, mcpServers: [] }, 2) as any; + if (loaded.error) throw new Error(JSON.stringify(loaded)); + expect(shellText(client.rawLines.map(line => JSON.parse(line)))).toEqual([expected]); + expect(client.stderr).toBe(""); + } finally { + await client?.close(); + fixture.close(); + } +}, 45000); + +test("ACP finishes tool calls that a cancel stopped before they ran", async () => { + const root = createIsolatedRoot("fx-acp-cancel-cards-"); + const gateway = startFakeGateway([ + fakeGatewaySse([ + { type: "tool-call", toolCallId: "call_slow", toolName: "shell", input: { request: { action: "run", yield_time_ms: 30_000, command: "sleep 5; echo slow" } } }, + { type: "tool-call", toolCallId: "call_skipped", toolName: "shell", input: { request: { action: "run", command: "echo skipped" } } }, + { type: "finish", finishReason: { unified: "tool-calls", raw: "tool-calls" } }, + ]), + ]); + let client: AcpClient | undefined; + try { + client = await AcpClient.create({ cwd: root.workspace, env: fakeGatewayEnv(root, gateway) }); + await client.request("initialize", { protocolVersion: 1 }, 1); + const created = await client.request("session/new", { cwd: root.workspace, mcpServers: [] }, 2) as any; + if (created.error) throw new Error(JSON.stringify(created)); + await client.request("session/set_mode", { modeId: "full-access" }, 3); + sendPrompt(client, 4, "Run both commands."); + await waitForCondition( + "slow command running", + () => client!.rawLines.some(line => line.includes("\"call_slow\"") && line.includes("\"in_progress\"")), + 10_000, + ); + client.send({ jsonrpc: "2.0", method: "session/cancel", params: {} }); + const { responses } = await readUntilResponses(client, [4]); + expect(responses.get(4)!.result.stopReason).toBe("cancelled"); + const updates = (id: string) => client!.rawLines + .map(line => JSON.parse(line)) + .filter((message: any) => message.method === "session/update" && message.params?.update?.toolCallId === id) + .map((message: any) => message.params.update); + const slowFinals = updates("call_slow").filter((update: any) => update.status === "completed" || update.status === "failed"); + expect(slowFinals.map((update: any) => update.status)).toEqual(["failed"]); + const skipped = updates("call_skipped"); + expect(skipped.map((update: any) => update.status)).toEqual(["pending", "failed"]); + expect(acpContentText(skipped[1].content)).toBe("Cancelled"); + expect(client.stderr).toBe(""); + } finally { + await client?.close(); + gateway.stop(); + rmSync(root.root, { recursive: true, force: true }); + } +}, 45000); + +test("ACP finishes a provider search the turn ended before it finished", async () => { + const root = createIsolatedRoot("fx-acp-open-search-"); + // Every attempt starts a provider search, then the stream breaks off, so + // recovery gives up with the search announced and never finished. + const brokenSearch = () => new Response( + `data: ${JSON.stringify({ type: "tool-input-start", id: "search_open", toolName: "exa_search" })}\n\n` + + `data: ${JSON.stringify({ type: "tool-input-delta", id: "search_open", delta: "{\"query\":" })}\n\n`, + { headers: { "content-type": "text/event-stream" } }, + ); + const gateway = startFakeGateway(Array.from({ length: 12 }, brokenSearch)); + let client: AcpClient | undefined; + try { + client = await AcpClient.create({ cwd: root.workspace, env: fakeGatewayEnv(root, gateway) }); + await client.request("initialize", { protocolVersion: 1 }, 1); + const created = await client.request("session/new", { cwd: root.workspace, mcpServers: [] }, 2) as any; + if (created.error) throw new Error(JSON.stringify(created)); + sendPrompt(client, 3, "Search the web."); + const { responses } = await readUntilResponses(client, [3]); + expect(responses.get(3)!.result.stopReason).toBe("refused"); + const search = client.rawLines + .map(line => JSON.parse(line)) + .filter((message: any) => message.method === "session/update" && message.params?.update?.toolCallId === "search_open") + .map((message: any) => message.params.update); + expect(search.map((update: any) => update.status)).toEqual(["pending", "failed"]); + expect(acpContentText(search[1].content)).toBe("Tool state preserved for recovery"); + expect(client.stderr).toBe(""); + } finally { + await client?.close(); + gateway.stop(); + rmSync(root.root, { recursive: true, force: true }); + } +}, 45000); + function createIsolatedRoot(prefix: string) { const root = realpathSync(mkdtempSync(join(tmpdir(), prefix))); const home = join(root, "home"); @@ -1293,13 +1405,16 @@ function sendSteeringPrompt(client: AcpClient, id: number, text: string): void { } // A withheld image's original: the session's blob, which the notice names -// and its saved prompt lists (D79). +// (D79) and its saved prompt names by hash, without the image's bytes (D89). function expectSavedOriginal(home: string, sessionId: string, body: string, original: Buffer): void { const named = /The original is saved at (\S+)\. Use/.exec(body)?.[1]; - const blob = join(sessionDir(home, sessionId), "blobs", createHash("sha256").update(original).digest("hex")); + const hash = createHash("sha256").update(original).digest("hex"); + const blob = join(sessionDir(home, sessionId), "blobs", hash); expect(named).toBe(blob); expect(readFileSync(blob)).toEqual(original); - expect(savedText(home, sessionId)).toContain(blob); + const saved = savedText(home, sessionId); + expect(saved).toContain(hash); + expect(saved).not.toContain(original.toString("base64")); } async function startAutoSession(client: AcpClient, hooks = false) { @@ -6655,8 +6770,14 @@ describe("acp: model-independent", () => { { type: "image", data: imageData, mimeType: "image/png" }, ], TIMEOUT); expect(JSON.stringify(failed)).toContain("RequiredVisionToolCallMissing"); - // The failed turn keeps its image inside the saved prompt (D44). - expect(savedText(root.home, sessionId)).toContain(imageData); + // The failed turn keeps its image as the session's blob, which its + // saved prompt names by hash, without the image's bytes (D89). + const imageBytes = Buffer.from(imageData, "base64"); + const hash = createHash("sha256").update(imageBytes).digest("hex"); + expect(readFileSync(join(sessionDir(root.home, sessionId), "blobs", hash))).toEqual(imageBytes); + const saved = savedText(root.home, sessionId); + expect(saved).toContain(hash); + expect(saved).not.toContain(imageData); await client.close(); client = await AcpClient.create({ cwd: root.workspace, env: fakeGatewayEnv(root, gateway) }); await client.request("initialize", { protocolVersion: 1 }, 10); diff --git a/tests/e2e/ask-presentation.test.ts b/tests/e2e/ask-presentation.test.ts index ff6b06fde..b065f161b 100644 --- a/tests/e2e/ask-presentation.test.ts +++ b/tests/e2e/ask-presentation.test.ts @@ -1017,6 +1017,27 @@ describe("fx ask shell startup files", () => { expect(result.stderr).not.toContain("shell snapshot unavailable"); }, TIMEOUT); + test("a user-profile command with a quoting error carries rewrite guidance", async () => { + const root = createRoot(); + const gateway = startFakeGateway([ + userShellRun("unmatched", "echo \"oops"), + fakeGatewayFinalText("Guidance checked.\n"), + ]); + gateways.push(gateway); + + const result = await runFx( + ["ask", "--json", "--yolo", "--no-save", "Check the guidance."], + { cwd: root.workspace, env: { ...gatewayEnv(root.home, gateway), FX_SOUND: "0" }, timeoutMs: TIMEOUT }, + ); + + expect(result.code).toBe(0); + // The snapshot runs the command through eval, so zsh reports the error as + // `(eval):1:` and bash as `/dev/fd/0: eval:` instead of under the shell's name. + expect(result.stderr).not.toContain("shell snapshot unavailable"); + expect(gateway.requests[1]!.body).toMatch(/\(eval\):1:|\/dev\/fd\/0: eval:/); + expect(gateway.requests[1]!.body).toContain("The shell could not parse this command"); + }, TIMEOUT); + test("startup files the snapshot cannot capture fall back with one notice", async () => { const root = createRoot(); // A 9 MB alias exceeds the 8 MiB capture limit, while full startup still diff --git a/tests/e2e/file-tool-paths.test.ts b/tests/e2e/file-tool-paths.test.ts index 52f6436b6..5121ac97f 100644 --- a/tests/e2e/file-tool-paths.test.ts +++ b/tests/e2e/file-tool-paths.test.ts @@ -349,11 +349,28 @@ async function runTerminalToolScenario(args: { } describe("filesystem path handling", () => { - for (const supportsImages of [true, false]) { + const readImageCases = [ + { + name: "read_file attaches a workspace image inline for a vision model", + modelTags: ["tool-use", "vision", "file-input"], + inline: true, + }, + { + // The catalog lists the default model, spacexai/grok-4.7, with vision + // and without file-input, which covers documents rather than images. + name: "read_file attaches a workspace image inline for a vision model without file input", + modelTags: ["reasoning", "tool-use", "implicit-caching", "vision", "structured-output"], + inline: true, + }, + { + name: "read_file image is withheld with guidance for a text-only model", + modelTags: ["tool-use"], + inline: false, + }, + ]; + for (const imageCase of readImageCases) { test( - supportsImages - ? "read_file attaches a workspace image inline for a vision model" - : "read_file image is withheld with guidance for a text-only model", + imageCase.name, async () => { const root = createIsolatedRoot(); const pngBase64 = @@ -363,11 +380,7 @@ describe("filesystem path handling", () => { toolCall("read_image_1", "read_file", { path: "pixel.png" }), finalText("image inspected"), ], - { - modelTags: supportsImages - ? ["tool-use", "vision", "file-input"] - : ["tool-use"], - }, + { modelTags: imageCase.modelTags }, ); try { writeFileSync( @@ -398,7 +411,7 @@ describe("filesystem path handling", () => { ) as Record | undefined; expect(part).toBeDefined(); const output = part!.output as Record; - if (supportsImages) { + if (imageCase.inline) { expect(output.type).toBe("content"); expect(contentText(output)).toContain("image attached"); expect(contentText(output)).not.toContain("binary or non-utf8"); diff --git a/tests/e2e/gateway-stream-lifecycle.test.ts b/tests/e2e/gateway-stream-lifecycle.test.ts index dd0db670f..e673c6b34 100644 --- a/tests/e2e/gateway-stream-lifecycle.test.ts +++ b/tests/e2e/gateway-stream-lifecycle.test.ts @@ -1153,7 +1153,7 @@ describe("gateway stream lifecycle", () => { expectOnlyLeadingSystemMessages(gateway.requests[0]!.body); expect(contentText(request.prompt[1]?.content)).toBe(WEB_SEARCH_GUIDANCE); expect(toolByName(oracleRequest, "shell")?.description).toBe( - "Run every command with shell.run. Fast commands complete in one call; commands still running after yield_time_ms return one owned session_id and remain available across turns. Use shell.interact with that exact session_id: omit chars to observe, or provide chars to send exact input and then observe. Use shell.stop only when termination is requested. output_delta is always terminal-safe; unsafe bytes are escaped while full_output_handle retains exact output, so do not run a separate command merely to test output safety or shell usability. Never detach with &, nohup, setsid, or double-forking. Each call starts a new shell with the user's startup files applied: their aliases, functions, and PATH are available, but cd, export, and alias changes do not carry over to the next call. In zsh, quote glob patterns meant for another program (for example '--include=*.zig') because unmatched globs are errors, and quote words that begin with =.", + "Run every command with shell.run. Fast commands complete in one call; commands still running after yield_time_ms return one owned session_id and remain available across turns. Use shell.interact with that exact session_id: omit chars to observe, or provide chars to send exact input and then observe. Use shell.stop only when termination is requested. output_delta is always terminal-safe; unsafe bytes are escaped while full_output_handle retains exact output, so do not run a separate command merely to test output safety or shell usability. Never detach with &, nohup, setsid, or double-forking. Each call starts a new shell with the user's startup files applied: their aliases, functions, and PATH are available, but cd, export, and alias changes do not carry over to the next call. In zsh, quote glob patterns meant for another program (for example '--include=*.zig') because unmatched globs are errors.", ); expect(toolByName(oracleRequest, "skill")?.description).toContain( "the task clearly matches one", @@ -9960,6 +9960,100 @@ printf '%s' ${JSON.stringify(trailingMarker)} > ${JSON.stringify(effectPath)} } }); + test("a restarted reply refused as too long compacts and finishes the turn", async () => { + const root = createFixtureRoot("restarted-reply-overflow"); + const tracePath = join(root.root, "trace.log"); + writeFileSync(join(root.workspace, "seed-notes.txt"), "RESTARTED_OVERFLOW_SEED_NOTES\n"); + let phase: "seed" | "attempt" = "seed"; + let attempt = 0; + const gateway = startDynamicFakeGateway((raw) => { + const body = JSON.parse(raw); + if (body.toolChoice?.type === "none" && body.tools?.length === 0) { + return fakeGatewayFinalText("Facts:\nF1: RESTARTED_OVERFLOW_SUMMARY: the earlier turn read seed-notes.txt."); + } + if (phase === "seed") { + // The earlier turn reads a file, which leaves work to compact. + return raw.includes('"seed-read"') + ? fakeGatewayFinalText("SEED_DONE") + : fakeGatewayToolCall("seed-read", "read_file", { path: "seed-notes.txt" }); + } + attempt += 1; + // The reply is cut off before its finish, so fx restarts it. + if (attempt === 1) return sse('data: {"type":"text-delta","id":"answer","delta":"visible partial text"}\n\n'); + // The provider refuses the restart for its size, before any output. + if (attempt === 2) { + return Response.json({ error: { + type: "invalid_request_error", + code: "context_length_exceeded", + message: "This model's maximum context length is 128000 tokens. The request exceeds the context window.", + } }, { status: 400 }); + } + return fakeGatewayFinalText("RESTARTED_OVERFLOW_RECOVERED"); + }, { + models: [{ id: MODEL, type: "language", tags: ["tool-use"], context_window: 128_000, max_tokens: 8192 }], + }); + try { + const seed = await runFx(["ask", "--json", "--auto", "Read seed-notes.txt and reply."], { + cwd: root.workspace, + env: fixtureEnv(root, gateway, tracePath), + timeoutMs: 15_000, + }); + expect(seed.code).toBe(0); + const sessionId = parseAskJson(seed.stdout).session_id; + const seedRequests = gateway.requests.length; + phase = "attempt"; + + const result = await runFx(["ask", "--json", "--auto", "--resume-id", sessionId, "Continue."], { + cwd: root.workspace, + env: { ...fixtureEnv(root, gateway, tracePath), FX_TRACE_SCOPES: "agent,core,gateway,stream,context_compaction" }, + timeoutMs: 15_000, + }); + const json = parseAskJson(result.stdout); + + expect(result.stderr).toContain("Response interrupted. Restarting."); + expect(result.stderr).not.toContain("HTTP 400"); + expect(result.code).toBe(0); + expect(json.exit_code).toBe(0); + expect(json.error).toBeUndefined(); + expect(json.output).toBe("RESTARTED_OVERFLOW_RECOVERED"); + + // The cut-off reply, its refused restart, the compaction's summary + // requests, and the restart sent again over the compacted conversation. + const requests = gateway.requests.slice(seedRequests).map((request) => request.body); + const restart = "Restart that response from the beginning"; + const isSummary = (raw: string) => { + const body = JSON.parse(raw); + return body.toolChoice?.type === "none" && body.tools?.length === 0; + }; + expect(requests.length).toBeGreaterThanOrEqual(4); + expect(promptText(requests[0]!)).not.toContain(restart); + expect(promptText(requests[1]!)).toContain(restart); + expect(requests[1]).not.toContain("compacted_conversation"); + const summaries = requests.slice(2, -1); + expect(summaries.every(isSummary)).toBe(true); + expect(summaries[0]).toContain("seed-notes.txt"); + const retry = requests.at(-1)!; + expect(isSummary(retry)).toBe(false); + expect(promptText(retry)).toContain(restart); + expect(retry).toContain("compacted_conversation"); + expect(retry).toContain("RESTARTED_OVERFLOW_SUMMARY"); + expect(retry).not.toContain("RESTARTED_OVERFLOW_SEED_NOTES"); + + // The session keeps the compaction and the finished reply. + const saved = await savedSession(root.home, sessionId); + expect(saved.history.some((turn) => turn.kind === "compacted_summary")).toBe(true); + expect(saved.history.at(-1)).toEqual(expect.objectContaining({ + kind: "assistant", + user: expect.objectContaining({ text: "Continue." }), + assistant: "RESTARTED_OVERFLOW_RECOVERED", + })); + expect(readFileSync(tracePath, "utf8")).toContain("provider_overflow_recovery"); + } finally { + gateway.stop(); + rmSync(root.root, { recursive: true, force: true }); + } + }); + test("post-tool provider retry keeps instructions leading", async () => { const root = createFixtureRoot("post-tool-retry-order"); const tracePath = join(root.root, "trace.log"); diff --git a/tests/e2e/sessions.test.ts b/tests/e2e/sessions.test.ts index 44e9ad0a7..a04638dd2 100644 --- a/tests/e2e/sessions.test.ts +++ b/tests/e2e/sessions.test.ts @@ -35,7 +35,7 @@ import { TmuxSession, tmuxAvailable, } from "./tmux-helpers"; -import { childLines, childWork, compactionRecordPath, type LogLine, logLines, onlySessionId, rootSessionIds, savedSessionIds, storeRoot } from "./saved-sessions"; +import { childLines, childWork, compactionRecordPath, type LogLine, logLines, onlySessionId, rootSessionIds, savedSessionIds, sessionDir, storeRoot } from "./saved-sessions"; // The session store: every `fx ask` entry and exit, the files it writes, and the faults a real disk and a real // crash produce: kills mid-stream and mid-tool, a torn tail, a flipped @@ -1666,22 +1666,42 @@ function itemData(line: any): any { return typeof line.data === "string" ? JSON.parse(line.data) : line.data; } -/// The images a session keeps inside its user items (D44), as the bytes -/// each decodes to. -function inlineImages(fixture: Fixture, id: string): Buffer[] { - const found: Buffer[] = []; +/// The images a session's user items name as its own blobs (D89): each +/// blob's path and bytes, and whether its item lists it. No user item may +/// keep an image's bytes or a snapshot path. +function savedImages(fixture: Fixture, id: string): { path: string; bytes: Buffer; listed: boolean }[] { + const found: { path: string; bytes: Buffer; listed: boolean }[] = []; for (const line of logLines(fixture.home, id) as any[]) { if (line.kind !== "item" || line.type !== "user") continue; for (const image of itemData(line).images ?? []) { - if (image.inline_data) found.push(Buffer.from(image.inline_data, "base64")); + expect(image.inline_data).toBeUndefined(); + expect(image.snapshot_path).toBeUndefined(); + expect(typeof image.blob).toBe("string"); + const path = join(sessionDir(fixture.home, id), "blobs", image.blob); + found.push({ path, bytes: readFileSync(path), listed: (line.blobs ?? []).includes(image.blob) }); } } return found; } +/// The saved files a request's notes name, in order. +function notedImagePaths(body: string): string[] { + const paths: string[] = []; + for (const message of JSON.parse(body).prompt as { content?: unknown }[]) { + if (!Array.isArray(message.content)) continue; + for (const part of message.content as { type?: string; text?: string }[]) { + if (part.type !== "text") continue; + for (const match of String(part.text).matchAll(/^\[Image #\d+ \([^)]*\) is saved at ("(?:[^"\\]|\\.)*")\.\]$/gm)) { + paths.push(JSON.parse(match[1]!)); + } + } + } + return paths; +} + const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); -test("fx ask keeps an image inside its turn, and resume in a new process sends it again", async () => { +test("fx ask saves an image as its session's blob, names it to the model, and resume sends it again (D89)", async () => { const fixture = createFixture("fx-v2-image-"); const gateway = replyToLatest([ ["Describe the image.", "IMAGE_SEEN"], @@ -1694,14 +1714,20 @@ test("fx ask keeps an image inside its turn, and resume in a new process sends i expect(created.code).toBe(0); expect(JSON.parse(created.stdout).output).toBe("IMAGE_SEEN"); const id = JSON.parse(created.stdout).session_id; - const images = inlineImages(fixture, id); + const images = savedImages(fixture, id); expect(images.length).toBe(1); - expect(images[0]!.subarray(0, 8).equals(PNG_SIGNATURE)).toBe(true); + expect(images[0]!.bytes.equals(Buffer.from(PNG_1X1, "base64"))).toBe(true); + expect(images[0]!.listed).toBe(true); + // The model is told where the blob is, and never the attached file. + expect(notedImagePaths(gateway.requests[0]!.body)).toEqual([images[0]!.path]); + expect(gateway.requests[0]!.body).not.toContain(image); const resumed = await ask(fixture, gateway, ["--resume-id", id, "And again."]); expect(resumed.code).toBe(0); // The prompt text says "image" too, so match the part's media type. expect(gateway.requests[0]!.body).toContain("image/png"); expect(gateway.requests.at(-1)!.body).toContain("image/png"); + // Read back from its blob, the image keeps its note in a new process. + expect(notedImagePaths(gateway.requests.at(-1)!.body)).toEqual([images[0]!.path]); expectWholeLog(fixture, id); expectNoV1Sessions(fixture); } finally { @@ -1710,6 +1736,33 @@ test("fx ask keeps an image inside its turn, and resume in a new process sends i } }, TIMEOUT * 2); +test.skipIf(!tmuxAvailable())("an image pasted into the app is saved as its session's blob and named to the model (D89)", async () => { + const fixture = createFixture("fx-v2-app-image-"); + const gateway = replyToLatest([["Describe the pasted image.", "APP_IMAGE_SEEN"]], undefined, VISION_MODEL); + try { + const image = join(fixture.workspace, "dot.png"); + writeFileSync(image, Buffer.from(PNG_1X1, "base64")); + const app = await startApp(fixture, gateway, []); + await app.session.pasteText(image); + await app.session.waitForText("[Image 1]", TIMEOUT); + await app.session.sendText(" Describe the pasted image."); + await app.session.waitForText("APP_IMAGE_SEEN", TIMEOUT); + await quitApp(app); + const id = onlySessionId(fixture.home); + const saved = savedImages(fixture, id); + expect(saved.length).toBe(1); + expect(saved[0]!.bytes.equals(Buffer.from(PNG_1X1, "base64"))).toBe(true); + expect(saved[0]!.listed).toBe(true); + const request = gateway.requests.find((r: { body: string }) => r.body.includes("Describe the pasted image."))!; + expect(notedImagePaths(request.body)).toEqual([saved[0]!.path]); + expect(request.body).not.toContain(image); + expectWholeLog(fixture, id); + } finally { + gateway.stop(); + rmSync(fixture.root, { recursive: true, force: true }); + } +}, TIMEOUT * 2); + test("ACP keeps an image prompt and replays it with the image on load", async () => { const fixture = createFixture("fx-v2-acp-image-"); const gateway = replyToLatest([["Describe this ACP image.", "ACP_IMAGE_SEEN"]], undefined, VISION_MODEL); @@ -1725,7 +1778,10 @@ test("ACP keeps an image prompt and replays it with the image on load", async () ], }); expect(prompted.stopReason).toBe("end_turn"); - expect(inlineImages(fixture, id).length).toBe(1); + const saved = savedImages(fixture, id); + expect(saved.length).toBe(1); + expect(saved[0]!.bytes.equals(Buffer.from(PNG_1X1, "base64"))).toBe(true); + expect(notedImagePaths(gateway.requests.at(-1)!.body)).toEqual([saved[0]!.path]); expect(await client.close()).toBe(0); client = await AcpRpc.start(fixture, gateway); diff --git a/tests/e2e/tui-composer-edit-contracts.test.ts b/tests/e2e/tui-composer-edit-contracts.test.ts index 4bcec3f8e..7ac7dd806 100644 --- a/tests/e2e/tui-composer-edit-contracts.test.ts +++ b/tests/e2e/tui-composer-edit-contracts.test.ts @@ -1,4 +1,5 @@ import { afterEach, expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, @@ -30,6 +31,7 @@ const COMPOSER_LIMIT_PASTE_TIMEOUT = TIMEOUT * 3; const COMPOSER_BYTE_LIMIT = 8 * 1024 * 1024; const PASTE_START = ["1b", "5b", "32", "30", "30", "7e"] as const; const PASTE_END = ["1b", "5b", "32", "30", "31", "7e"] as const; +const SUFFIX_IGNORED_NOTICE = "Input that arrived right after the paste was ignored."; const MANAGED_REVIEW_BODY = "GROUP_C_MANAGED_REVIEW_BODY"; const WORKSPACE_REVIEW_BODY = "GROUP_C_WORKSPACE_REVIEW_BODY"; @@ -276,6 +278,21 @@ async function pasteExact(active: TmuxSession, text: string): Promise { ]); } +/// Pastes `text` and presses `key` in one tmux command, so the key reaches fx +/// right behind the paste's end marker, as when a user presses a key while a +/// large paste is still arriving. +function pasteThenKey(active: TmuxSession, text: string, key: string): void { + const buffer = `${active.name}-paste-then-key`; + execFileSync("tmux", ["load-buffer", "-b", buffer, "-"], { + input: text, + stdio: ["pipe", "pipe", "pipe"], + }); + execFileSync("tmux", [ + "paste-buffer", "-d", "-p", "-b", buffer, "-t", active.name, + ";", "send-keys", "-t", active.name, key, + ], { stdio: "pipe" }); +} + async function selectReviewSkill( active: TmuxSession, selectWorkspace = false, @@ -318,6 +335,7 @@ tmuxTest( await waitForGatewayRequest(); expect(finalUserText()).toBe("CRLF_SENTINEL_A\nCRLF_SENTINEL_B"); + expect(await active.captureFullScrollback()).not.toContain(SUFFIX_IGNORED_NOTICE); expectCleanRuntime(active); }, TIMEOUT, @@ -336,10 +354,18 @@ tmuxTest( "0d", ...PASTE_END, ]); - await waitForTraceOrExit(active, "reason=unsafe_suffix"); + // "/version", "\r", and the terminal's real end marker all follow the + // embedded one in the same delivery. + await waitForTraceOrExit( + active, + "paste suffix dropped owner=composer bytes=15 reason=same_delivery_as_end_marker", + ); + await active.waitForText(SUFFIX_IGNORED_NOTICE, TIMEOUT); expect(gateway?.requestCount()).toBe(0); - expect(await active.captureFullScrollback()).not.toMatch(/[*✓!✗⊘i] version:/); + const scrollback = await active.captureFullScrollback(); + expect(scrollback).not.toMatch(/[*✓!✗⊘i] version:/); + expect(composerContains(scrollback, "/version")).toBe(false); await active.waitForText("PRESERVED_DRAFT", TIMEOUT); await active.sendKeys("Enter"); @@ -364,13 +390,109 @@ tmuxTest( "5b", ...PASTE_END, ]); - await waitForTraceOrExit(active, "reason=unsafe_suffix"); + await waitForTraceOrExit( + active, + "paste suffix dropped owner=composer bytes=8 reason=same_delivery_as_end_marker", + ); expect(gateway?.requestCount()).toBe(0); - await active.waitForText("ESCAPE_DRAFT", TIMEOUT); + await active.waitForPane((pane) => composerContains(pane, "ESCAPE_DRAFTpaste"), TIMEOUT); await active.sendKeys("Enter"); await waitForGatewayRequest(); - expect(finalUserText()).toBe("ESCAPE_DRAFT"); + expect(finalUserText()).toBe("ESCAPE_DRAFTpaste"); + expectCleanRuntime(active); + }, + TIMEOUT, +); + +tmuxTest( + "Enter in the same delivery as a paste keeps the paste and does not submit", + async () => { + const active = await startFx(true, 1, false, "input"); + await active.sendLiteralText("DRAFT "); + + await active.sendHexBytes([ + ...PASTE_START, + ...textHex("PASTED_BODY"), + ...PASTE_END, + "0d", + ]); + await waitForTraceOrExit(active, "paste end owner=composer bytes=11"); + await waitForTraceOrExit( + active, + "paste suffix dropped owner=composer bytes=1 reason=same_delivery_as_end_marker", + ); + await active.waitForText(SUFFIX_IGNORED_NOTICE, TIMEOUT); + await active.waitForPane((pane) => composerContains(pane, "DRAFT PASTED_BODY"), TIMEOUT); + expect(gateway?.requestCount()).toBe(0); + + await active.sendKeys("Enter"); + await waitForGatewayRequest(); + expect(finalUserText()).toBe("DRAFT PASTED_BODY"); + expectCleanRuntime(active); + }, + TIMEOUT, +); + +tmuxTest( + "keys typed in the same delivery as a paste are not added to the draft", + async () => { + const active = await startFx(true, 1, false, "input"); + + await active.sendHexBytes([ + ...PASTE_START, + ...textHex("PASTED_BODY"), + ...PASTE_END, + ...textHex("TYPED"), + ]); + await waitForTraceOrExit( + active, + "paste suffix dropped owner=composer bytes=5 reason=same_delivery_as_end_marker", + ); + await active.waitForText(SUFFIX_IGNORED_NOTICE, TIMEOUT); + await active.waitForPane((pane) => composerContains(pane, "PASTED_BODY"), TIMEOUT); + expect(composerContains(await active.capturePane(), "TYPED")).toBe(false); + + // Keys typed after the paste has been read are ordinary input again. + await active.sendLiteralText(" AFTER"); + await active.waitForPane((pane) => composerContains(pane, "PASTED_BODY AFTER"), TIMEOUT); + await active.sendKeys("Enter"); + await waitForGatewayRequest(); + expect(finalUserText()).toBe("PASTED_BODY AFTER"); + expectCleanRuntime(active); + }, + TIMEOUT, +); + +tmuxTest( + "a large paste with Enter pressed right behind it stays in the draft", + async () => { + const active = await startFx(true, 1, false, "input"); + const lines = Array.from( + { length: 400 }, + (_, index) => `log line ${String(index).padStart(5, "0")} GET /assets/chunk.js 200 340ms`, + ); + const body = lines.join("\n"); + await active.sendLiteralText("is there a download optimization to make? "); + + pasteThenKey(active, body, "Enter"); + await waitForTraceOrExit(active, `paste end owner=composer bytes=${body.length}`); + await waitForTraceOrExit( + active, + "paste suffix dropped owner=composer bytes=1 reason=same_delivery_as_end_marker", + ); + await active.waitForText(SUFFIX_IGNORED_NOTICE, TIMEOUT); + await active.waitForPane( + (pane) => + composerContains(pane, "is there a download optimization to make?") && + composerContains(pane, "[Pasted text #1, 400 lines]"), + TIMEOUT, + ); + expect(gateway?.requestCount()).toBe(0); + + await active.sendKeys("Enter"); + await waitForGatewayRequest(); + expect(finalUserText()).toBe(`is there a download optimization to make? ${body}`); expectCleanRuntime(active); }, TIMEOUT, diff --git a/tests/e2e/vision-route-fake-gateway.test.ts b/tests/e2e/vision-route-fake-gateway.test.ts index edb7c3905..c356ee488 100644 --- a/tests/e2e/vision-route-fake-gateway.test.ts +++ b/tests/e2e/vision-route-fake-gateway.test.ts @@ -298,6 +298,33 @@ function writeMarkedImage(imagePath: string, marker: string) { return readFileSync(imagePath).toString("base64"); } +// Damages image 1 of a copied v1 session past repair: its file no longer +// holds the image, and its digest is one no other image there shares, so +// an identical copy saved as a blob cannot restore it (D89). +function damageFirstV1Image(home: string, sessionId: string): void { + const folder = join(home, ".fx", "sessions", sessionId); + const snapshot = readdirSync(join(folder, "images")).find((name) => name.startsWith("image-1-")); + expect(snapshot).toBeDefined(); + writeFileSync(join(folder, "images", snapshot!), "corrupt"); + const log = join(folder, "events.jsonl"); + const lines = readFileSync(log, "utf8").split("\n"); + const first = JSON.parse(lines[0]!); + first.event.user.images[0].snapshot_sha256 = createHash("sha256").update("an image no other copy holds").digest("hex"); + lines[0] = JSON.stringify(first); + writeFileSync(log, lines.join("\n")); +} + +// A prompt image session `id` saved: its blob, which the saved prompt names +// by hash, without the image's bytes (D89). +function expectSavedBlob(home: string, id: string, base64: string): void { + const bytes = Buffer.from(base64, "base64"); + const hash = createHash("sha256").update(bytes).digest("hex"); + expect(readFileSync(join(sessionDir(home, id), "blobs", hash))).toEqual(bytes); + const saved = savedText(home, id); + expect(saved).toContain(hash); + expect(saved).not.toContain(base64); +} + function writeLegacyZeroImageSession( root: ReturnType, sessionId: string, @@ -2388,10 +2415,7 @@ describe("Vision route fake Gateway", () => { // Twenty prompt images only an older fx's session file holds; one is // damaged before the session converts on its next open. const [sessionId] = copyV1Fixture(root.home, "many-images", 1, root.workspace) as [string]; - const imageDir = join(root.home, ".fx", "sessions", sessionId, "images"); - const firstSnapshot = readdirSync(imageDir).find((name) => name.startsWith("image-1-")); - expect(firstSnapshot).toBeDefined(); - writeFileSync(join(imageDir, firstSnapshot!), "corrupt"); + damageFirstV1Image(root.home, sessionId); const gateway = startImageGateway([ sseToolCall("vision", { image_ids: [1, 2], focus: "inspect both" }, "vision_partial"), sseText(visionBatchResult([2])), @@ -2457,10 +2481,7 @@ describe("Vision route fake Gateway", () => { // Twenty prompt images only an older fx's session file holds; one is // damaged before the session converts on its next open. const [sessionId] = copyV1Fixture(root.home, "many-images", 1, root.workspace) as [string]; - const imageDir = join(root.home, ".fx", "sessions", sessionId, "images"); - const firstSnapshot = readdirSync(imageDir).find((name) => name.startsWith("image-1-")); - expect(firstSnapshot).toBeDefined(); - writeFileSync(join(imageDir, firstSnapshot!), "corrupt"); + damageFirstV1Image(root.home, sessionId); const allIds = Array.from({ length: 20 }, (_, index) => index + 1); const gateway = startImageGateway([ sseToolCall("vision", { image_ids: allIds, focus: "inspect all" }, "vision_partial_twenty"), @@ -3233,8 +3254,8 @@ for (const sourceChange of ["removed", "changed"] as const) { expect(failed.code).toBe(1); expect(failed.stdout).toContain("RequiredVisionToolCallMissing"); const id = JSON.parse(failed.stdout).session_id; - // The failed turn keeps the captured image inside its saved prompt (D44). - expect(savedText(root.home, id)).toContain(original); + // The failed turn keeps the captured image as its session's blob (D89). + expectSavedBlob(root.home, id, original); if (sourceChange === "removed") rmSync(input); else writeMarkedImage(input, "REPLACEMENT_MUST_NOT_BE_USED"); const resumed = await runFx(["ask", "--json", "--auto", "--resume-id", id, "Describe the saved image again."], options); @@ -3275,8 +3296,8 @@ test.skipIf(!tmuxAvailable())("TUI keeps a failed turn's image through a cold re await session.waitForText("RequiredVisionToolCallMissing", TIMEOUT); await session.waitForStableComposer(TIMEOUT); const [id] = rootSessionIds(root.home) as [string]; - // The failed turn keeps the captured image inside its saved prompt (D44). - expect(savedText(root.home, id)).toContain(original); + // The failed turn keeps the captured image as its session's blob (D89). + expectSavedBlob(root.home, id, original); await session.sendText("/quit"); await session.waitForPane(() => session!.paneStatus().dead, TIMEOUT); expect(session.paneStatus().status).toBe(0); @@ -3303,7 +3324,7 @@ test.skipIf(!tmuxAvailable())("TUI keeps a failed turn's image through a cold re await session.sendText("/quit"); await session.waitForPane(() => session!.paneStatus().dead, TIMEOUT); expect(session.paneStatus().status).toBe(0); - expect(savedText(root.home, id)).toContain(original); + expectSavedBlob(root.home, id, original); } finally { await session?.kill(); gateway.stop(); diff --git a/tests/tla/README.md b/tests/tla/README.md index f1949596b..7f62c3833 100644 --- a/tests/tla/README.md +++ b/tests/tla/README.md @@ -7,11 +7,12 @@ TLA+ specifications for fx, checked with TLC, together with the scripts and Bun | `acp/` | The ACP permission modes, which match the CLI's `/permissions`, and the per-session Standard, Fast, and Ultrafast speed (`src/acp/`). | `acp/check.sh`, then the drivers in `drivers/acp/` | | `cli/` | How `fx` and `fx ask` handle `--model`, `--fast`, `--ultrafast`, and `--effort` at launch, for the current design and a proposed one that validates them. | `cli/check.sh`, `bun test drivers/cli/` | | `durable/` | Durable libfx sessions (`sdk/fx-sdk.js`): the journal, tool replay, parallel batches, fencing, the inbox, resumption, and snapshots. | `durable/check.sh`, `durable/mutants.py` | +| `image_route/` | How images reach the model on each request (`src/core/images/image_route.zig`): whether the catalog's `vision` tag makes image input native, the user-image route and vision tool mode, tool images attached or withheld with a notice, one capability request and one catalog-down notice per turn, and crashes, duplicate requests, and lost or stale replies. `design/` holds the goal, vision, context, and design the checks trace to, and `mutations.json` the planted bugs. | `image_route/check.sh`, which runs `bun run verify`: TLC, every graph edge replayed against the real core, code traces checked by TLC, every planted bug, and `design/DESIGN.md` traceability | | `indexer/` | When a scanned root's listing may be reused, merging parallel grep results, the file picker's loading row, and listing snapshots (`src/core/indexer/`). | `indexer/check.sh` | | `mcp/` | The MCP engine (`src/mcp/`): wire format, requests, stdio, version detection, the tool catalog, both HTTP transports, input-required rounds, authorization, subscriptions, elicitation, and the engine. `rules.md` lists the MCP specification rules the models cite by ID. | `mcp/check.sh`, then the drivers in `drivers/mcp/` | | `mcp_host/` | fx's MCP host (`src/core/mcp_host/`): host lifecycle, saved credentials, logins shared between processes, sign-in, elicitation routing, and calls held for the URLs a server asks the user to open. | `mcp_host/check.sh`, `bun test drivers/mcp_host/` | | `sdk/` | libfx durable sessions (`DurableSession.tla`), how a session object takes prompts in (`PromptIntake.tla`), images saved once by name (`ImageStore.tla`), agent names and routing (`AgentRouting.tla`), tenants whose agents share one process (`TenantIsolation.tla`), steering (`DurableSteer.tla`), and the provider credential edge (`ProviderAuth.tla`). `check.sh` also writes the TLC state graphs the drivers walk. | `sdk/check.sh`, then the drivers | -| `session_manager/` | The session log, turn lifecycle, writer lock, snapshots, fork, subagents, catalog, host wiring, the side-file move, and v1 conversion. `rollout/` holds the models of the changes that moved fx onto the session manager. `system-design.md` is the design record the specs check and defines the `D1` to `D88` decision numbers that code comments cite. | `session_manager/check.sh`, `session_manager/rollout/run.py`, `bun test drivers/session_manager/` | +| `session_manager/` | The session log, turn lifecycle, writer lock, snapshots, fork, subagents, catalog, host wiring, the side-file move, v1 conversion, and prompt images saved as blobs (`PromptImages.tla`). `rollout/` holds the models of the changes that moved fx onto the session manager. `system-design.md` is the design record the specs check and defines the `D1` to `D89` decision numbers that code comments cite. | `session_manager/check.sh`, `session_manager/mutants.py`, `session_manager/rollout/run.py`, `bun test drivers/session_manager/` | | `shell_command/` | How an agent git command is admitted and run (`src/core/shell_command/safe_git.zig`), and one model turn through the shell tool, for the current design and a redesign. | `shell_command/check.sh`, `bun test drivers/shell_command/` | | `terminal/` | The in-memory shell snapshot owner (`src/core/terminal/shell_snapshot.zig`) and the lifecycle of hosted terminals. | `terminal/check.sh` | | `usage/` | `/usage` refreshes and the shared usage cache (`src/usage/`): `LeanUsage.tla` and `UsageModel.tla`. | `usage/check.sh`, then the drivers in `drivers/usage/` | @@ -65,6 +66,7 @@ From the repository root: tests/tla/session_manager/check.sh # every session spec tests/tla/session_manager/check.sh SessionLog Fork # selected specs tests/tla/session_manager/mutants.py # every planted bug must be caught +tests/tla/session_manager/mutants.py PromptImages # the planted bugs of selected specs tests/tla/session_manager/trace/selftest.sh # the trace validator must accept and reject tests/tla/session_manager/rollout/run.py check # the rollout models; mutate for their planted bugs tests/tla/indexer/check.sh # every config and its expected verdict diff --git a/tests/tla/drivers/image_route/code-to-spec.ts b/tests/tla/drivers/image_route/code-to-spec.ts new file mode 100644 index 000000000..ed009349d --- /dev/null +++ b/tests/tla/drivers/image_route/code-to-spec.ts @@ -0,0 +1,146 @@ +// Code to spec: runs event sequences through the real core, with faults, +// records each run as a JSON trace, and has TLC check that every trace is a +// behavior of ImageRoute and keeps its laws (TraceImageRoute.tla). +// +// Each trace starts in a new harness process. The scripted traces are the +// adversarial cases the design names; the random ones mix every event with +// the faults the model allows: a crash at any step (the process is killed), +// a request delivered twice, a reply that is lost (the next turn starts +// first), stale, or duplicated, and a catalog that is down, missing the +// model, or flapping between answers. +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { Harness, actFor, checkTlc, seededRandom, specFiles, toTla, type CheckedTlc, type Event, type Request, type View } from "./lib.ts"; + +const listed = (vision: boolean, file_input: boolean): View => ({ kind: "listed", vision, file_input }); +const down: View = { kind: "down", vision: false, file_input: false }; +const missing: View = { kind: "missing", vision: false, file_input: false }; +// The live catalog's entry for the default model, spacexai/grok-4.7. +const visionOnly = listed(true, false); +const views: View[] = [listed(true, true), visionOnly, listed(false, true), listed(false, false), missing, down]; + +function req(flags: Partial): Request { + return { user_images: false, pending_images: false, tool_images: false, fallback: false, catalog_down: false, ...flags }; +} + +const requests: Request[] = []; +for (let bits = 0; bits < 32; bits += 1) { + const r = req({ + user_images: (bits & 1) !== 0, + pending_images: (bits & 2) !== 0, + tool_images: (bits & 4) !== 0, + fallback: (bits & 8) !== 0, + catalog_down: (bits & 16) !== 0, + }); + if (r.pending_images && !r.user_images) continue; + requests.push(r); +} + +const turn = (view: View): Event => ({ event: "turn_started", view }); +const ready = (flags: Partial): Event => ({ event: "request_ready", req: req(flags) }); +const reply = (view: View): Event => ({ event: "capabilities_resolved", view }); +const crash: Event = { event: "crash" }; + +/// The adversarial cases, by name. +export const scripted: Record = { + // The reported bug: a tool image under the default model's catalog entry. + "default model tool image": [turn(visionOnly), ready({ tool_images: true, fallback: true }), ready({ tool_images: true, user_images: true })], + // A down catalog: one notice per turn, however many requests. + "catalog down notice once per turn": [ + turn(down), ready({ tool_images: true, catalog_down: true }), reply(down), + ready({ tool_images: true, catalog_down: true }), ready({ tool_images: true, catalog_down: true }), + turn(down), ready({ tool_images: true, catalog_down: true }), reply(down), + ], + // A request delivered again, then another, while the reply is owed. + "duplicate request while a reply is owed": [ + turn(missing), ready({ user_images: true }), ready({ user_images: true }), ready({ tool_images: true }), reply(visionOnly), + ], + // The reply is lost when the turn ends first, then arrives stale. + "lost reply then stale reply": [turn(down), ready({ tool_images: true }), turn(visionOnly), reply(down), ready({ tool_images: true })], + // The process dies while a reply is owed. + "crash while a reply is owed": [turn(down), ready({ user_images: true, pending_images: true }), crash, reply(visionOnly), ready({ user_images: true }), reply(visionOnly)], + // A duplicated reply, the second with a different answer. + "duplicated reply": [turn(missing), ready({ tool_images: true, fallback: true }), reply(listed(false, false)), reply(visionOnly), ready({ tool_images: true, fallback: true })], + // User images on a model without vision and no vision tool fail. + "user images with no route": [turn(listed(false, true)), ready({ user_images: true, pending_images: true }), ready({ user_images: true, fallback: true, pending_images: true })], + // The catalog recovers within the turn: the one resolve picks it up. + "catalog recovers within the turn": [turn(down), ready({ tool_images: true, user_images: true, fallback: true }), reply(listed(true, false)), ready({ tool_images: true })], +}; + +function randomEvents(random: () => number, length: number): Event[] { + const pick = (items: T[]): T => items[Math.floor(random() * items.length)]!; + const pickView = () => (random() < 0.3 ? visionOnly : pick(views)); + const events: Event[] = [turn(pickView())]; + for (let i = 1; i < length; i += 1) { + const roll = random(); + const previous = events[events.length - 1]!; + if (roll < 0.1 && previous.event === "request_ready") events.push(previous); + else if (roll < 0.5) events.push({ event: "request_ready", req: pick(requests) }); + else if (roll < 0.75) events.push(reply(pickView())); + else if (roll < 0.93) events.push(turn(pickView())); + else events.push(crash); + } + return events; +} + +export type CodeToSpecResult = { + traces: number; + steps: number; + tracesPath: string; + tlc: CheckedTlc; +}; + +/// Runs the traces through `bin` and checks them with `cfgName` from +/// `cfgText` (TraceImageRoute.cfg by default). +export async function codeToSpec(options: { + bin: string; + dir: string; + seed: number; + randomTraces: number; + maxLength: number; + cfgName?: string; + cfgText?: string; +}): Promise { + const random = seededRandom(options.seed); + const runs: { name: string; events: Event[] }[] = Object.entries(scripted).map(([name, events]) => ({ name, events })); + for (let i = 0; i < options.randomTraces; i += 1) { + runs.push({ name: `random ${i + 1} (seed ${options.seed})`, events: randomEvents(random, 2 + Math.floor(random() * (options.maxLength - 1))) }); + } + + const harness = await Harness.start(options.bin); + const traces: { name: string; states: Record[] }[] = []; + let steps = 0; + try { + for (const run of runs) { + await harness.crash(); + const states: Record[] = [{ ...harness.last, act: { name: "Init" } }]; + for (const event of run.events) { + const before = harness.last; + const shown = await harness.send(event); + states.push({ ...shown, act: actFor(event, before) }); + steps += 1; + } + traces.push({ name: run.name, states }); + } + } finally { + await harness.close(); + } + + mkdirSync(options.dir, { recursive: true }); + const tracesPath = join(options.dir, "traces.json"); + writeFileSync(tracesPath, `${JSON.stringify(traces, null, 1)}\n`); + const data = [ + "---- MODULE TraceData ----", + "EXTENDS Sequences", + "Traces == <<", + traces.map((trace) => ` \\* ${trace.name}\n ${toTla(trace.states)}`).join(",\n"), + ">>", + "====", + "", + ].join("\n"); + const files = { ...specFiles(), "TraceData.tla": data }; + const cfgName = options.cfgName ?? "TraceImageRoute.cfg"; + if (options.cfgText) files[cfgName as keyof typeof files] = options.cfgText; + const tlc = await checkTlc(join(options.dir, "tlc"), "TraceImageRoute", cfgName, files); + return { traces: traces.length, steps, tracesPath, tlc }; +} diff --git a/tests/tla/drivers/image_route/harness.zig b/tests/tla/drivers/image_route/harness.zig new file mode 100644 index 000000000..377362769 --- /dev/null +++ b/tests/tla/drivers/image_route/harness.zig @@ -0,0 +1,103 @@ +//! A JSON-lines shell around the real image route core, for the Bun drivers. +//! +//! Built from `src/core/images/image_route.zig` (or a planted-bug copy of it) +//! as the `image_route` module. On start it prints the core's zero state. Then +//! it reads one event per line and prints one line per event: +//! +//! in: {"event":"turn_started","view":{"kind":"listed","vision":true,"file_input":false}} +//! {"event":"request_ready","req":{"user_images":false,...}} +//! {"event":"capabilities_resolved","view":{"kind":"down","vision":false,"file_input":false}} +//! out: {"state":{"support":"unknown","resolved":false,"notified":false,"pending":[]},"out":[...]} +//! +//! `state` is `image_route.abstract`, and `out` is the effects `step` +//! returned, in the spec's record shapes. The harness is a shell: it turns a +//! catalog view into a support value with the core's own +//! `supportFromCatalog`, exactly as `model_capabilities.mergeCapabilities` +//! does, and holds the state between lines. A crash is the driver killing +//! this process. +const std = @import("std"); +const image_route = @import("image_route"); + +const ViewJson = struct { + kind: []const u8, + vision: bool = false, + file_input: bool = false, +}; + +const Input = struct { + event: []const u8, + view: ?ViewJson = null, + req: ?image_route.Request = null, +}; + +const EffectJson = struct { + kind: []const u8, + route: ?image_route.Route = null, + mode: ?image_route.VisionMode = null, + tool_images: ?image_route.ToolImages = null, + reason: ?image_route.FailReason = null, +}; + +const Line = struct { + state: image_route.Abstract, + out: []const EffectJson, +}; + +fn catalogView(view: ViewJson) error{InvalidView}!image_route.CatalogView { + if (std.mem.eql(u8, view.kind, "down")) return .down; + if (std.mem.eql(u8, view.kind, "missing")) return .missing; + if (std.mem.eql(u8, view.kind, "listed")) return .{ .listed = .{ .vision = view.vision, .file_input = view.file_input } }; + return error.InvalidView; +} + +fn event(input: Input) error{ InvalidView, InvalidEvent }!image_route.Event { + if (std.mem.eql(u8, input.event, "turn_started")) { + return .{ .turn_started = image_route.supportFromCatalog(try catalogView(input.view orelse return error.InvalidEvent)) }; + } + if (std.mem.eql(u8, input.event, "capabilities_resolved")) { + return .{ .capabilities_resolved = image_route.supportFromCatalog(try catalogView(input.view orelse return error.InvalidEvent)) }; + } + if (std.mem.eql(u8, input.event, "request_ready")) { + return .{ .request_ready = input.req orelse return error.InvalidEvent }; + } + return error.InvalidEvent; +} + +fn effectJson(effect: image_route.Effect) EffectJson { + return switch (effect) { + .resolve_capabilities => .{ .kind = "resolve" }, + .notify_catalog_down => .{ .kind = "notify" }, + .send => |chosen| .{ .kind = "send", .route = chosen.route, .mode = chosen.mode, .tool_images = chosen.tool_images }, + .fail => |reason| .{ .kind = "fail", .reason = reason }, + }; +} + +fn writeLine(writer: *std.Io.Writer, state: *const image_route.State, effects: []const image_route.Effect) !void { + var buffer: [image_route.max_effects]EffectJson = undefined; + for (effects, 0..) |effect, index| buffer[index] = effectJson(effect); + try std.json.Stringify.value(Line{ .state = image_route.abstract(state), .out = buffer[0..effects.len] }, .{ .emit_null_optional_fields = false }, writer); + try writer.writeByte('\n'); + try writer.flush(); +} + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const alloc = init.gpa; + + var in_buffer: [64 * 1024]u8 = undefined; + var stdin = std.Io.File.stdin().reader(io, &in_buffer); + var out_buffer: [64 * 1024]u8 = undefined; + var stdout = std.Io.File.stdout().writer(io, &out_buffer); + + var state: image_route.State = .{}; + try writeLine(&stdout.interface, &state, &.{}); + while (try stdin.interface.takeDelimiter('\n')) |raw| { + const text = std.mem.trim(u8, raw, " \r\t"); + if (text.len == 0) continue; + const parsed = try std.json.parseFromSlice(Input, alloc, text, .{}); + defer parsed.deinit(); + const out = image_route.step(state, try event(parsed.value)); + state = out.state; + try writeLine(&stdout.interface, &state, out.effects()); + } +} diff --git a/tests/tla/drivers/image_route/lib.ts b/tests/tla/drivers/image_route/lib.ts new file mode 100644 index 000000000..0526bad44 --- /dev/null +++ b/tests/tla/drivers/image_route/lib.ts @@ -0,0 +1,286 @@ +// Shared pieces of the image route drivers: where the model and the core +// live, building and running the JSON-lines harness around the real core, a +// seeded random source, TLA+ value printing, and TLC runs on the pinned jar. +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { REPO_ROOT, TLA_ROOT, runTlc, tlcToolchain, type TlcRun } from "../lib/tlc.ts"; + +export const SPEC_DIR = join(TLA_ROOT, "image_route", "spec"); +export const MODEL_DIR = join(TLA_ROOT, "image_route"); +export const CORE_SOURCE = join(REPO_ROOT, "src", "core", "images", "image_route.zig"); +export const HARNESS_SOURCE = join(import.meta.dirname, "harness.zig"); + +// TLC 1.7.4 (TLC2 2.19 of 08 August 2024, rev 5a47802), from +// https://github.com/tlaplus/tlaplus/releases/download/v1.7.4/tla2tools.jar +export const PINNED_JAR_SHA256 = "936a262061c914694dfd669a543be24573c45d5aa0ff20a8b96b23d01e050e88"; +export const PINNED_TLC_VERSION = "TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802)"; + +export type View = { kind: "listed" | "missing" | "down"; vision: boolean; file_input: boolean }; +export type Request = { + user_images: boolean; + pending_images: boolean; + tool_images: boolean; + fallback: boolean; + catalog_down: boolean; +}; +export type Effect = { kind: string; route?: string; mode?: string; tool_images?: string; reason?: string }; +export type CoreState = { support: string; resolved: boolean; notified: boolean; pending: Request[] }; +export type Shown = CoreState & { out: Effect[] }; +export type Act = + | { name: "Init" | "Crash" } + | { name: "TurnStarted"; view: View } + | { name: "RequestReady"; req: Request } + | { name: "CapabilitiesResolved"; view: View; answered: Request[] }; +export type Event = + | { event: "turn_started"; view: View } + | { event: "request_ready"; req: Request } + | { event: "capabilities_resolved"; view: View } + | { event: "crash" }; + +/// The pinned toolchain, or a reason it is not the pinned one. +export function pinnedToolchain() { + const toolchain = tlcToolchain(); + if (toolchain.jarSha256 !== PINNED_JAR_SHA256) { + throw new Error( + `tla2tools.jar at ${toolchain.jar} has sha256 ${toolchain.jarSha256}, not the pinned ${PINNED_JAR_SHA256}. ` + + "Download v1.7.4 from https://github.com/tlaplus/tlaplus/releases/download/v1.7.4/tla2tools.jar " + + "and point TLA2TOOLS_JAR at it.", + ); + } + return toolchain; +} + +/// A canonical JSON form with sorted keys, so equal values print equally. +export function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (value && typeof value === "object") { + const entries = Object.entries(value as Record).filter(([, v]) => v !== undefined); + entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`; + } + return JSON.stringify(value); +} + +/// A JSON value as TLA+: objects are records, arrays are sequences. +export function toTla(value: unknown): string { + if (Array.isArray(value)) return `<<${value.map(toTla).join(", ")}>>`; + if (typeof value === "boolean") return value ? "TRUE" : "FALSE"; + if (typeof value === "number") return String(value); + if (typeof value === "string") return JSON.stringify(value); + if (value && typeof value === "object") { + const entries = Object.entries(value as Record).filter(([, v]) => v !== undefined); + return `[${entries.map(([k, v]) => `${k} |-> ${toTla(v)}`).join(", ")}]`; + } + throw new Error(`cannot print ${String(value)} as TLA+`); +} + +/// Mulberry32: a small seeded generator, so a failing run can be replayed. +export function seededRandom(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/// Builds the harness around `core` (the real core, or a planted-bug copy) +/// into `dir`. Throws with the compiler's output when the build fails. +export function buildHarness(dir: string, core = CORE_SOURCE): string { + mkdirSync(dir, { recursive: true }); + const bin = join(dir, "image-route-harness"); + const build = Bun.spawnSync([ + "zig", "build-exe", "-ODebug", + "--dep", "image_route", `-Mroot=${HARNESS_SOURCE}`, `-Mimage_route=${core}`, + `-femit-bin=${bin}`, "--cache-dir", join(REPO_ROOT, ".zig-cache"), + ], { cwd: REPO_ROOT, stdout: "pipe", stderr: "pipe" }); + if (build.exitCode !== 0 || !existsSync(bin)) { + throw new Error(`harness build failed for ${core}:\n${build.stderr.toString()}${build.stdout.toString()}`); + } + return bin; +} + +/// One running harness process. `crash()` kills it and starts a new one, +/// which begins from the core's zero state: the only way the harness loses +/// state is the way a real process does. +export class Harness { + private proc!: ReturnType; + private lines: string[] = []; + private waiters: ((line: string) => void)[] = []; + private buffered = ""; + last!: Shown; + + private constructor(private readonly bin: string) {} + + static async start(bin: string): Promise { + const harness = new Harness(bin); + await harness.spawn(); + return harness; + } + + private async spawn() { + this.lines = []; + this.waiters = []; + this.buffered = ""; + this.proc = Bun.spawn([this.bin], { stdin: "pipe", stdout: "pipe", stderr: "inherit" }); + const reader = (this.proc.stdout as ReadableStream).getReader(); + const decoder = new TextDecoder(); + (async () => { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + this.buffered += decoder.decode(value, { stream: true }); + let newline = this.buffered.indexOf("\n"); + while (newline >= 0) { + const line = this.buffered.slice(0, newline); + this.buffered = this.buffered.slice(newline + 1); + const waiter = this.waiters.shift(); + if (waiter) waiter(line); + else this.lines.push(line); + newline = this.buffered.indexOf("\n"); + } + } + })(); + this.last = this.parse(await this.nextLine()); + } + + private nextLine(): Promise { + const line = this.lines.shift(); + if (line !== undefined) return Promise.resolve(line); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("the harness did not answer within 5 s")), 5000); + this.waiters.push((value) => { clearTimeout(timer); resolve(value); }); + }); + } + + private parse(line: string): Shown { + const parsed = JSON.parse(line) as { state: CoreState; out: Effect[] }; + return { ...parsed.state, out: parsed.out }; + } + + async send(event: Event): Promise { + if (event.event === "crash") { + await this.crash(); + return this.last; + } + const stdin = this.proc.stdin as import("bun").FileSink; + stdin.write(`${JSON.stringify(event)}\n`); + stdin.flush(); + this.last = this.parse(await this.nextLine()); + return this.last; + } + + async crash() { + this.proc.kill("SIGKILL"); + await this.proc.exited; + await this.spawn(); + } + + async close() { + (this.proc.stdin as import("bun").FileSink).end(); + await this.proc.exited; + } +} + +/// The event that performs `act` on the core. +export function eventFor(act: Act): Event { + switch (act.name) { + case "TurnStarted": return { event: "turn_started", view: act.view }; + case "RequestReady": return { event: "request_ready", req: act.req }; + case "CapabilitiesResolved": return { event: "capabilities_resolved", view: act.view }; + case "Crash": return { event: "crash" }; + case "Init": throw new Error("Init is not an event"); + } +} + +/// The spec's act for `event`, applied from a core that showed `before`. +export function actFor(event: Event, before: Shown): Act { + switch (event.event) { + case "turn_started": return { name: "TurnStarted", view: event.view }; + case "request_ready": return { name: "RequestReady", req: event.req }; + case "capabilities_resolved": return { name: "CapabilitiesResolved", view: event.view, answered: before.pending }; + case "crash": return { name: "Crash" }; + } +} + +/// The part of a spec state the code shows. +export function shownOf(state: Record): Shown { + const { support, resolved, notified, pending, out } = state as Shown; + return { support, resolved, notified, pending, out }; +} + +export type CheckedTlc = TlcRun & { verdict: "pass" | "violated" | "error"; property: string | null; tlcVersion: string | null }; + +/// Runs TLC and names the property it reports violated, including a +/// temporal property, which TLC reports without a name. +export async function checkTlc( + dir: string, + module: string, + cfg: string, + files: Record, + extraArgs: string[] = [], +): Promise { + pinnedToolchain(); + mkdirSync(dir, { recursive: true }); + for (const [name, text] of Object.entries(files)) writeFileSync(join(dir, name), text); + const run = await runTlc(dir, module, cfg, [], dir, ["-deadlock", ...extraArgs]); + const tlcVersion = /^(TLC2 Version [^\n]*?)\s*$/m.exec(run.output)?.[1] ?? null; + // A property TraceImageRoute re-exports from ImageRoute is reported under + // its instance name, such as IR!ResolveOncePerTurn. + let property = /Error: (?:Invariant|Action property) ([\w!]+) is violated/.exec(run.output)?.[1]?.replace(/^\w+!/, "") ?? null; + if (!property && /Temporal properties were violated/.test(run.output)) { + const cfgText = files[cfg] ?? readFileSync(join(dir, cfg), "utf8"); + const listed = /PROPERTY\s+([\s\S]*)$/.exec(cfgText)?.[1]?.split(/\s+/).filter(Boolean) ?? []; + property = listed.length === 1 ? listed[0]! : "temporal"; + } + const passed = run.exitCode === 0 && /Model checking completed\. No error has been found\./.test(run.output); + const verdict = passed ? "pass" : property ? "violated" : "error"; + return { ...run, verdict, property, tlcVersion }; +} + +/// The spec files with `ImageRoute.tla` replaced by `spec`. +export function specFiles(spec = readFileSync(join(SPEC_DIR, "ImageRoute.tla"), "utf8")): Record { + return { + "ImageRoute.tla": spec, + "MCImageRoute.cfg": readFileSync(join(SPEC_DIR, "MCImageRoute.cfg"), "utf8"), + "TraceImageRoute.tla": readFileSync(join(SPEC_DIR, "TraceImageRoute.tla"), "utf8"), + "TraceImageRoute.cfg": readFileSync(join(SPEC_DIR, "TraceImageRoute.cfg"), "utf8"), + }; +} + +/// A config that checks only `property`, which is an invariant or property +/// of MCImageRoute.cfg, against `specification`. +export function singleConfig(property: string, specification: string, invariants: string[]): string { + const kind = invariants.includes(property) ? "INVARIANT" : "PROPERTY"; + return `SPECIFICATION ${specification}\n${specification === "TraceSpec" ? "CHECK_DEADLOCK FALSE\n" : ""}${kind}\n ${property}\n`; +} + +/// The names MCImageRoute.cfg lists, by section. +export function configNames(cfgText = readFileSync(join(SPEC_DIR, "MCImageRoute.cfg"), "utf8")) { + const sections: Record = {}; + let current = ""; + for (const raw of cfgText.split("\n")) { + const line = raw.replace(/\\\*.*$/, "").trim(); + if (!line) continue; + const [first, ...rest] = line.split(/\s+/); + if (/^[A-Z_]+$/.test(first!)) { + current = first!; + sections[current] ??= []; + sections[current]!.push(...rest); + } else { + sections[current]?.push(first!, ...rest); + } + } + return { invariants: sections.INVARIANT ?? [], properties: sections.PROPERTY ?? [] }; +} + +/// Applies a planted bug: `find` must occur exactly once in `text`. +export function plant(text: string, find: string, replace: string, id: string): string { + const first = text.indexOf(find); + if (first < 0 || text.indexOf(find, first + 1) >= 0) { + throw new Error(`${id}: the text to replace must occur exactly once`); + } + return text.slice(0, first) + replace + text.slice(first + find.length); +} diff --git a/tests/tla/drivers/image_route/mutations.ts b/tests/tla/drivers/image_route/mutations.ts new file mode 100644 index 000000000..d0ec06c96 --- /dev/null +++ b/tests/tla/drivers/image_route/mutations.ts @@ -0,0 +1,81 @@ +// Proves no check is vacuous. Each planted bug in mutations.json breaks one +// guard, in the spec or in the code, and must make its named property fail. +// +// A spec bug is checked twice: TLC on the mutated model with only the named +// property, which must report exactly that property violated, and with all of +// MCImageRoute.cfg, which must fail. A code bug is planted in a copy of +// src/core/images/image_route.zig, the harness is rebuilt from the copy, and +// both directions run: the spec-to-code sweep must find the core leaving the +// model, and the code-to-spec traces must violate the named property alone and +// fail the full trace check. +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { codeToSpec } from "./code-to-spec.ts"; +import { CORE_SOURCE, MODEL_DIR, SPEC_DIR, buildHarness, checkTlc, configNames, plant, singleConfig, specFiles } from "./lib.ts"; +import { specToCode } from "./spec-to-code.ts"; + +type Mutation = { id: string; breaks: string; find: string; replace: string; expect: string }; +export const mutations = JSON.parse(readFileSync(join(MODEL_DIR, "mutations.json"), "utf8")) as { spec: Mutation[]; code: Mutation[] }; + +export type MutationResult = { + id: string; + breaks: string; + expect: string; + caught: boolean; + detail: Record; +}; + +export async function specMutations(dir: string): Promise { + const spec = readFileSync(join(SPEC_DIR, "ImageRoute.tla"), "utf8"); + const { invariants } = configNames(); + const results: MutationResult[] = []; + for (const mutation of mutations.spec) { + const mutated = plant(spec, mutation.find, mutation.replace, mutation.id); + const files = specFiles(mutated); + const single = await checkTlc(join(dir, mutation.id, "single"), "ImageRoute", "Single.cfg", { + ...files, + "Single.cfg": singleConfig(mutation.expect, "Spec", invariants), + }); + const full = await checkTlc(join(dir, mutation.id, "full"), "ImageRoute", "MCImageRoute.cfg", files); + results.push({ + id: mutation.id, + breaks: mutation.breaks, + expect: mutation.expect, + caught: single.verdict === "violated" && single.property === mutation.expect && full.verdict === "violated", + detail: { alone: single.property ?? single.verdict, full: full.property ?? full.verdict, states: single.distinctStates }, + }); + } + return results; +} + +export async function codeMutations(dir: string, graphPath: string, seed: number, randomTraces: number): Promise { + const core = readFileSync(CORE_SOURCE, "utf8"); + const traceCfg = readFileSync(join(SPEC_DIR, "TraceImageRoute.cfg"), "utf8"); + const { invariants } = configNames(traceCfg); + const results: MutationResult[] = []; + for (const mutation of mutations.code) { + const work = join(dir, mutation.id); + mkdirSync(work, { recursive: true }); + const source = join(work, "image_route.zig"); + writeFileSync(source, plant(core, mutation.find, mutation.replace, mutation.id)); + const bin = buildHarness(work, source); + const sweep = await specToCode(bin, graphPath); + const alone = await codeToSpec({ + bin, dir: join(work, "alone"), seed, randomTraces, maxLength: 30, + cfgName: "Single.cfg", cfgText: singleConfig(mutation.expect, "TraceSpec", invariants), + }); + const full = await codeToSpec({ bin, dir: join(work, "full"), seed, randomTraces, maxLength: 30 }); + results.push({ + id: mutation.id, + breaks: mutation.breaks, + expect: mutation.expect, + caught: sweep.failure !== null && alone.tlc.verdict === "violated" && alone.tlc.property === mutation.expect && full.tlc.verdict === "violated", + detail: { + specToCode: sweep.failure ? `left the model after ${sweep.failure.path.join(" > ")}` : "not detected", + alone: alone.tlc.property ?? alone.tlc.verdict, + full: full.tlc.property ?? full.tlc.verdict, + }, + }); + } + return results; +} diff --git a/tests/tla/drivers/image_route/spec-to-code.ts b/tests/tla/drivers/image_route/spec-to-code.ts new file mode 100644 index 000000000..d5c6d96e8 --- /dev/null +++ b/tests/tla/drivers/image_route/spec-to-code.ts @@ -0,0 +1,86 @@ +// Spec to code: replays every transition of ImageRoute's TLC state graph +// against the real core through the harness, and after every step checks +// that what the core shows (abstract(state) and the step's effects) equals +// the model's state. +// +// The model keeps no history beyond one turn, so its graph is small enough +// to sweep whole: for each edge, the core is reset to its zero state, driven +// along a shortest path from the initial state to the edge's source, and then +// across the edge. A Crash edge kills the harness process and starts a new +// one. Every edge is taken, so coverage is complete or the run fails. +import { loadGraph } from "../graph.mjs"; +import { Harness, canonical, eventFor, shownOf, type Act } from "./lib.ts"; + +type Graph = { states: Map>; next: Map>; init: string }; + +export type SpecToCodeResult = { + states: number; + edges: number; + steps: number; + failure: null | { path: string[]; expected: string; shown: string }; +}; + +function label(act: Act): string { + if (act.name === "TurnStarted" || act.name === "CapabilitiesResolved") { + const view = act.view.kind === "listed" ? `listed(vision=${act.view.vision},file_input=${act.view.file_input})` : act.view.kind; + return `${act.name}(${view})`; + } + if (act.name === "RequestReady") { + const flags = Object.entries(act.req).filter(([, on]) => on).map(([name]) => name); + return `RequestReady(${flags.join(",") || "no images"})`; + } + return act.name; +} + +/// Shortest paths from the initial state, as state ids from init to each state. +function shortestPaths(graph: Graph): Map { + const paths = new Map([[graph.init, [graph.init]]]); + const queue = [graph.init]; + while (queue.length > 0) { + const from = queue.shift()!; + for (const to of graph.next.get(from) ?? []) { + if (paths.has(to)) continue; + paths.set(to, [...paths.get(from)!, to]); + queue.push(to); + } + } + return paths; +} + +export async function specToCode(bin: string, graphPath: string): Promise { + const graph = (await loadGraph(graphPath)) as Graph; + const paths = shortestPaths(graph); + const zero = canonical(shownOf(graph.states.get(graph.init)!)); + const edges: [string, string][] = []; + for (const [from, targets] of graph.next) for (const to of targets) edges.push([from, to]); + edges.sort(([a1, b1], [a2, b2]) => (a1 + b1 < a2 + b2 ? -1 : 1)); + + const harness = await Harness.start(bin); + let steps = 0; + try { + for (const [from, to] of edges) { + // Back to the zero state: a turn that starts with an unreachable + // catalog holds exactly the core's initial state. + const reset = await harness.send({ event: "turn_started", view: { kind: "down", vision: false, file_input: false } }); + if (canonical(reset) !== zero) { + return { states: graph.states.size, edges: edges.length, steps, failure: { path: ["reset"], expected: zero, shown: canonical(reset) } }; + } + const route = [...paths.get(from)!.slice(1), to]; + const taken: string[] = []; + for (const id of route) { + const state = graph.states.get(id)!; + const act = state.act as Act; + taken.push(label(act)); + const shown = canonical(await harness.send(eventFor(act))); + steps += 1; + const expected = canonical(shownOf(state)); + if (shown !== expected) { + return { states: graph.states.size, edges: edges.length, steps, failure: { path: taken, expected, shown } }; + } + } + } + } finally { + await harness.close(); + } + return { states: graph.states.size, edges: edges.length, steps, failure: null }; +} diff --git a/tests/tla/drivers/image_route/traceability.ts b/tests/tla/drivers/image_route/traceability.ts new file mode 100644 index 000000000..686cd3566 --- /dev/null +++ b/tests/tla/drivers/image_route/traceability.ts @@ -0,0 +1,139 @@ +// The traceability check: design/DESIGN.md must account for every review +// question, every rule, and every law, and each enforcement entry must name +// something that exists. +// +// Enforcement entries are backticked tokens in the last column of the rule +// and law tables under "## Enforcement": +// inv:Name an invariant defined in ImageRoute.tla and listed in MCImageRoute.cfg +// prop:Name a property defined in ImageRoute.tla and listed in MCImageRoute.cfg +// trace:Name a check TraceImageRoute.cfg lists, run on traces of the real core +// mutation:ID a planted bug in mutations.json +// zig:path#name a `test "name"` block in that Zig file +// e2e:path#name, sdk:path#name a test whose source contains that name +// driver:name a driver stage: spec-to-code or code-to-spec +// review:text how a check that cannot be automated is enforced instead +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { REPO_ROOT } from "../lib/tlc.ts"; +import { MODEL_DIR, SPEC_DIR, configNames } from "./lib.ts"; +import { mutations } from "./mutations.ts"; + +export type Traceability = { problems: string[]; rules: number; laws: number; questions: number; entries: number }; + +function section(text: string, heading: string): string { + const start = text.indexOf(`\n## ${heading}\n`); + if (start < 0) return ""; + const rest = text.slice(start + heading.length + 5); + const end = rest.search(/\n## /); + return end < 0 ? rest : rest.slice(0, end); +} + +function rows(text: string, prefix: string): string[][] { + return text.split("\n") + .filter((line) => new RegExp(`^\\|\\s*${prefix}\\d+\\s*\\|`).test(line)) + .map((line) => line.split("|").slice(1, -1).map((cell) => cell.trim())); +} + +export function checkTraceability(designPath = join(MODEL_DIR, "design", "DESIGN.md")): Traceability { + const problems: string[] = []; + const design = readFileSync(designPath, "utf8"); + const spec = readFileSync(join(SPEC_DIR, "ImageRoute.tla"), "utf8"); + const traceSpec = readFileSync(join(SPEC_DIR, "TraceImageRoute.tla"), "utf8"); + const model = configNames(); + const trace = configNames(readFileSync(join(SPEC_DIR, "TraceImageRoute.cfg"), "utf8")); + const mutationIds = new Set([...mutations.spec, ...mutations.code].map((m) => m.id)); + const defined = (text: string, name: string) => new RegExp(`^${name} ==`, "m").test(text); + const referenced = new Set(); + let entries = 0; + + const checkToken = (owner: string, token: string) => { + entries += 1; + const colon = token.indexOf(":"); + const kind = token.slice(0, colon); + const value = token.slice(colon + 1).trim(); + const fileTest = () => { + const [path, name] = value.split("#"); + const file = join(REPO_ROOT, path ?? ""); + if (!path || !name || !existsSync(file)) return problems.push(`${owner}: ${token} names a file that does not exist`); + const source = readFileSync(file, "utf8"); + const found = kind === "zig" ? source.includes(`test "${name}"`) : source.includes(name); + if (!found) problems.push(`${owner}: ${token} names a test that does not exist`); + }; + switch (kind) { + case "inv": + referenced.add(value); + if (!defined(spec, value)) problems.push(`${owner}: ${token} is not defined in ImageRoute.tla`); + if (!model.invariants.includes(value)) problems.push(`${owner}: ${token} is not an INVARIANT in MCImageRoute.cfg`); + break; + case "prop": + referenced.add(value); + if (!defined(spec, value)) problems.push(`${owner}: ${token} is not defined in ImageRoute.tla`); + if (!model.properties.includes(value)) problems.push(`${owner}: ${token} is not a PROPERTY in MCImageRoute.cfg`); + break; + case "trace": + if (!defined(traceSpec, value)) problems.push(`${owner}: ${token} is not defined in TraceImageRoute.tla`); + if (![...trace.invariants, ...trace.properties].includes(value)) problems.push(`${owner}: ${token} is not checked by TraceImageRoute.cfg`); + break; + case "mutation": + if (!mutationIds.has(value)) problems.push(`${owner}: ${token} is not in mutations.json`); + break; + case "zig": + case "e2e": + case "sdk": + fileTest(); + break; + case "driver": + if (!["spec-to-code", "code-to-spec"].includes(value)) problems.push(`${owner}: ${token} is not a driver stage`); + break; + case "review": + if (value.length < 20) problems.push(`${owner}: ${token} must say how it is enforced`); + break; + default: + problems.push(`${owner}: ${token} has an unknown kind`); + } + }; + const tokens = (cell: string | undefined) => [...(cell ?? "").matchAll(/`([a-z]+:[^`]+)`/g)].map((m) => m[1]!); + + // Every review question is answered or listed as not applicable. + const answers = section(design, "Questions"); + const answered = new Set([...answers.matchAll(/^- Q(\d+)\b/gm)].map((m) => Number(m[1]))); + const notApplicable = /^Not applicable:(.*)$/m.exec(answers)?.[1] ?? ""; + const skipped = new Set([...notApplicable.matchAll(/Q(\d+)/g)].map((m) => Number(m[1]))); + for (let q = 1; q <= 39; q += 1) { + if (answered.has(q) && skipped.has(q)) problems.push(`Q${q} is both answered and listed as not applicable`); + if (!answered.has(q) && !skipped.has(q)) problems.push(`Q${q} is neither answered nor listed as not applicable`); + } + + const enforcement = section(design, "Enforcement"); + // Every rule has one row with a status; an applicable one names its checks. + const ruleRows = rows(enforcement, "R"); + for (let r = 1; r <= 11; r += 1) { + const matches = ruleRows.filter((row) => row[0] === `R${r}`); + if (matches.length !== 1) { problems.push(`R${r} needs exactly one row under Enforcement, found ${matches.length}`); continue; } + const [, status, reason, enforced] = matches[0]!; + if (!["pass", "fail", "n/a"].includes(status ?? "")) problems.push(`R${r} has status "${status}", not pass, fail, or n/a`); + if (!reason) problems.push(`R${r} gives no reason`); + const found = tokens(enforced); + if (status !== "n/a" && found.length === 0) problems.push(`R${r} applies but names no enforcement`); + for (const token of found) checkToken(`R${r}`, token); + } + + // Every law the design states has a row, enforced by the model or a test. + const lawIds = [...section(design, "Laws").matchAll(/^- \*\*(L\d+)\b/gm)].map((m) => m[1]!); + if (lawIds.length === 0) problems.push("the Laws section states no laws"); + const lawRows = rows(enforcement, "L"); + for (const id of lawIds) { + const matches = lawRows.filter((row) => row[0] === id); + if (matches.length !== 1) { problems.push(`${id} needs exactly one row under Enforcement, found ${matches.length}`); continue; } + const found = tokens(matches[0]!.at(-1)); + if (found.length === 0) problems.push(`${id} names no enforcement`); + for (const token of found) checkToken(id, token); + } + for (const row of lawRows) if (!lawIds.includes(row[0]!)) problems.push(`${row[0]} has an enforcement row but is not stated under Laws`); + + // Every check the model runs belongs to a rule or a law. + for (const name of [...model.invariants, ...model.properties]) { + if (name !== "TypeOK" && !referenced.has(name)) problems.push(`MCImageRoute.cfg checks ${name}, which no rule or law names`); + } + return { problems, rules: ruleRows.length, laws: lawIds.length, questions: answered.size + skipped.size, entries }; +} diff --git a/tests/tla/drivers/image_route/verify.ts b/tests/tla/drivers/image_route/verify.ts new file mode 100644 index 000000000..fbc1082b4 --- /dev/null +++ b/tests/tla/drivers/image_route/verify.ts @@ -0,0 +1,111 @@ +// `bun run verify` from tests/tla/image_route/: the one gate for the image +// route design. In order, and every step must pass: +// +// 1. the pinned TLC and the Java that runs it; +// 2. TLC on ImageRoute.tla with MCImageRoute.cfg, which also writes the +// state graph; +// 3. the harness, built from the real src/core/images/image_route.zig; +// 4. spec to code: every graph edge replayed against the real core; +// 5. code to spec: scripted and random traces with faults, checked by TLC; +// 6. every planted bug in the spec caught by its property; +// 7. every planted bug in the code caught in both directions; +// 8. traceability: DESIGN.md accounts for every question, rule, and law. +// +// Settings: FX_IMAGE_ROUTE_SEED (default 20261019), FX_IMAGE_ROUTE_TRACES +// (random traces, default 300), FX_TLA_OUT (where the report goes). +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { outDir, writeJson } from "../lib/tlc.ts"; +import { codeToSpec } from "./code-to-spec.ts"; +import { MODEL_DIR, PINNED_TLC_VERSION, buildHarness, checkTlc, pinnedToolchain, specFiles } from "./lib.ts"; +import { codeMutations, specMutations } from "./mutations.ts"; +import { specToCode } from "./spec-to-code.ts"; +import { checkTraceability } from "./traceability.ts"; + +const seed = Number(process.env.FX_IMAGE_ROUTE_SEED ?? "20261019"); +const randomTraces = Number(process.env.FX_IMAGE_ROUTE_TRACES ?? "300"); +const out = outDir("image-route"); +const report: Record = { started: new Date().toISOString(), seed, randomTraces, out }; +const failures: string[] = []; +const started = performance.now(); +const elapsed = (from: number) => `${((performance.now() - from) / 1000).toFixed(1)} s`; + +function stage(name: string, ok: boolean, summary: string) { + console.log(`${ok ? "PASS" : "FAIL"} ${name}: ${summary}`); + if (!ok) failures.push(name); +} + +try { + const toolchain = pinnedToolchain(); + report.toolchain = toolchain; + stage("toolchain", true, `tla2tools.jar sha256 ${toolchain.jarSha256.slice(0, 12)}, ${toolchain.javaVersion}`); + + let at = performance.now(); + const model = await checkTlc(join(out, "model"), "ImageRoute", "MCImageRoute.cfg", specFiles(), ["-dump", "dot,actionlabels", join(out, "model", "graph")]); + report.model = { verdict: model.verdict, property: model.property, distinctStates: model.distinctStates, depth: model.depth, durationMs: model.durationMs, tlcVersion: model.tlcVersion, generated: Number(/^(\d+) states generated, \d+ distinct states found/m.exec(model.output)?.[1] ?? NaN) }; + const versionOk = model.tlcVersion === PINNED_TLC_VERSION; + stage("model check", model.verdict === "pass" && versionOk, + `${model.verdict}${model.property ? ` (${model.property})` : ""}, ${model.distinctStates} distinct of ${(report.model as { generated: number }).generated} generated states, depth ${model.depth}, ${model.durationMs} ms, ${model.tlcVersion}`); + const graphPath = join(out, "model", "graph.dot"); + + at = performance.now(); + const bin = buildHarness(join(out, "harness")); + stage("harness", true, `built from src/core/images/image_route.zig in ${elapsed(at)}`); + + at = performance.now(); + const sweep = await specToCode(bin, graphPath); + report.specToCode = sweep; + stage("spec to code", sweep.failure === null, + sweep.failure + ? `the core left the model after ${sweep.failure.path.join(" > ")}: expected ${sweep.failure.expected}, shown ${sweep.failure.shown}` + : `${sweep.edges} of ${sweep.edges} edges over ${sweep.states} states, ${sweep.steps} steps, ${elapsed(at)}`); + + at = performance.now(); + const traces = await codeToSpec({ bin, dir: join(out, "code-to-spec"), seed, randomTraces, maxLength: 30 }); + report.codeToSpec = { traces: traces.traces, steps: traces.steps, tracesPath: traces.tracesPath, verdict: traces.tlc.verdict, property: traces.tlc.property, distinctStates: traces.tlc.distinctStates, durationMs: traces.tlc.durationMs }; + stage("code to spec", traces.tlc.verdict === "pass", + `${traces.traces} traces, ${traces.steps} steps, TLC ${traces.tlc.verdict}${traces.tlc.property ? ` (${traces.tlc.property})` : ""}, ${traces.tlc.distinctStates} distinct states, ${elapsed(at)}`); + + at = performance.now(); + const specBugs = await specMutations(join(out, "spec-mutations")); + report.specMutations = specBugs; + for (const bug of specBugs) console.log(` ${bug.caught ? "caught" : "MISSED"} ${bug.id} by ${bug.expect}: alone ${bug.detail.alone}, full ${bug.detail.full}`); + stage("spec mutations", specBugs.every((bug) => bug.caught), `${specBugs.filter((bug) => bug.caught).length} of ${specBugs.length} caught, ${elapsed(at)}`); + + at = performance.now(); + const codeBugs = await codeMutations(join(out, "code-mutations"), graphPath, seed, Math.min(randomTraces, 150)); + report.codeMutations = codeBugs; + for (const bug of codeBugs) console.log(` ${bug.caught ? "caught" : "MISSED"} ${bug.id} by ${bug.expect}: alone ${bug.detail.alone}, full ${bug.detail.full}; spec to code ${bug.detail.specToCode}`); + stage("code mutations", codeBugs.every((bug) => bug.caught), `${codeBugs.filter((bug) => bug.caught).length} of ${codeBugs.length} caught, ${elapsed(at)}`); + + const trace = checkTraceability(); + report.traceability = trace; + for (const problem of trace.problems) console.log(` ${problem}`); + // The check itself must reject a design that drops a rule, names a check + // that does not exist, or leaves a question unanswered. + const design = readFileSync(join(MODEL_DIR, "design", "DESIGN.md"), "utf8"); + const broken = [ + ["a missing rule row", design.replace(/^\| R5 \|.*\n/m, "")], + ["an invariant that does not exist", design.replace("`inv:OneSnapshot`, `trace:OneSnapshot`", "`inv:OneSnapshots`, `trace:OneSnapshot`")], + ["an unanswered question", design.replace(/^- Q24:.*\n/m, "")], + ] as const; + const rejected = broken.filter(([, text]) => { + const path = join(out, "traceability", "DESIGN.md"); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, text); + return text !== design && checkTraceability(path).problems.length > 0; + }); + report.traceabilitySelfCheck = rejected.map(([name]) => name); + stage("traceability", trace.problems.length === 0 && rejected.length === broken.length, + `${trace.questions} questions, ${trace.rules} rules, ${trace.laws} laws, ${trace.entries} enforcement entries, ${trace.problems.length} problems; rejects ${rejected.length} of ${broken.length} broken designs`); +} catch (error) { + failures.push("verify"); + console.error(`FAIL verify: ${(error as Error).message}`); + report.error = (error as Error).stack; +} + +report.failures = failures; +report.durationMs = Math.round(performance.now() - started); +writeJson(join(out, "report.json"), report); +console.log(`${failures.length === 0 ? "verify passed" : `verify failed: ${failures.join(", ")}`} in ${elapsed(started)}; report ${join(out, "report.json")}`); +process.exitCode = failures.length === 0 ? 0 : 1; diff --git a/tests/tla/drivers/session_manager/prompt-images.test.ts b/tests/tla/drivers/session_manager/prompt-images.test.ts new file mode 100644 index 000000000..afa9867c4 --- /dev/null +++ b/tests/tla/drivers/session_manager/prompt-images.test.ts @@ -0,0 +1,238 @@ +// Prompt images as blobs of a v2 session (D89) through the real binary, +// judged by TLC. Each scenario runs `fx ask` against a fake gateway, then +// writes what it observed, every turn's first request and saved user item +// and the session's blobs folder, as one line per step of +// tests/tla/session_manager/PromptImages.tla, and has TLC check the lines +// through trace/PromptImagesTrace.tla. A note is observed as naming an image +// only when the file it names is that image's blob in the request's own +// session, so TLC rejects any other note. Faults planted into a kept run +// must be rejected. A converted v1 session's import is observed by its user +// item. Forks, which only the terminal makes, are checked by the spec and by +// session_adapter.zig's D89 test. +import { expect, test } from "bun:test"; +import { chmodSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { artifacts, binary, testEnvironment, worktree } from "../lib/context"; +import { runTlc, sha256File, TLA_ROOT } from "../lib/tlc"; + +const { fakeGatewayTitleDefault, TITLE_GENERATION_MARKER } = await import(join(worktree, "tests", "e2e", "tmux-helpers.ts")); +const { solidPng } = await import(join(worktree, "tests", "e2e", "fixtures", "image-encoding.ts")); +const { copyV1Fixture } = await import(join(worktree, "tests", "e2e", "saved-sessions.ts")); + +const TIMEOUT = 60_000; +// Tagged for image input, so images go to the model inline. +const MODEL = "google/gemini-2.5-flash"; +const TRACE_SPECS = ["PromptImagesTrace.tla", "PromptImagesTrace.cfg"]; +const MODEL_SPECS = ["PromptImages.tla"]; + +type Line = Record; +const set = (values: Iterable) => ({ $set: [...new Set(values)].sort() }); + +function sseText(text: string) { + return new Response( + `data: ${JSON.stringify({ type: "text-delta", id: "a", delta: text })}\n\n` + + `data: ${JSON.stringify({ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } } })}\n\n` + + "data: [DONE]\n\n", + { headers: { "content-type": "text/event-stream" } }, + ); +} + +function fixture(name: string) { + const root = mkdtempSync(join(artifacts, `prompt-images-${name}-`)); + const home = join(root, "home"), workspace = join(root, "workspace"); + mkdirSync(join(home, ".fx"), { recursive: true, mode: 0o700 }); + mkdirSync(workspace, { mode: 0o700 }); + writeFileSync(join(home, ".fx", "settings.json"), JSON.stringify({ permission: {} })); + const sessionsRoot = join(home, ".fx", "sessions", "v2"); + const requests: string[] = []; + const server = Bun.serve({ + port: 0, + async fetch(req) { + if (new URL(req.url).pathname === "/coding-agent/v1/models") { + return Response.json({ data: [{ id: MODEL, type: "language", tags: ["vision", "file-input", "tool-use"] }] }); + } + if (req.method !== "POST") return new Response("not found", { status: 404 }); + const body = await req.text(); + if (body.includes(TITLE_GENERATION_MARKER)) return fakeGatewayTitleDefault(); + requests.push(body); + return sseText("seen"); + }, + }); + const env = testEnvironment({ + HOME: home, AI_GATEWAY_API_KEY: "synthetic-prompt-images-fixture", FX_SKIP_ONBOARDING: "1", + FX_GATEWAY_BASE_URL: `http://127.0.0.1:${server.port}`, + FX_GATEWAY_CHAT_URL: `http://127.0.0.1:${server.port}/v4/ai/language-model`, + FX_MODEL: MODEL, NO_COLOR: "1", + }); + // Each distinct image gets the next label, d1, d2, ..., in the order seen. + const labels = new Map(); + const label = (digest: string) => { + if (!labels.has(digest)) labels.set(digest, `d${labels.size + 1}`); + return labels.get(digest)!; + }; + const lines: Line[] = []; + return { + root, home, sessionsRoot, requests, lines, label, + image(name: string, width: number, height: number): string { + const path = join(workspace, name); + writeFileSync(path, solidPng(width, height)); + return path; + }, + /// One `fx ask` turn; its first request and its user item become lines. + async ask(session: "source" | "fork", images: string[], resume?: string): Promise { + const before = requests.length; + const args = ["ask", "--json", ...(resume ? ["--resume", resume] : [])]; + for (const image of images) args.push("--image", image); + args.push("Look at the images."); + const proc = Bun.spawn([binary, ...args], { cwd: workspace, env, stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, exitCode] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]); + expect(stderr).toBe(""); + expect(exitCode).toBe(0); + expect(requests.length).toBe(before + 1); + const id = JSON.parse(stdout).session_id as string; + const asked = images.map(path => label(sha256File(path))); + lines.push(observeRequest(requests[before]!, session, asked, join(sessionsRoot, id), label)); + lines.push(observeWrite(join(sessionsRoot, id), session, label)); + return id; + }, + blobs(session: "source" | "fork", id: string) { + lines.push({ event: "Blobs", s: session, blobs: set(blobLabels(join(sessionsRoot, id), label)) }); + }, + stop() { server.stop(true); }, + }; +} + +/// The request's images: those sent inline, and those whose note names the +/// image's own blob in `sessionDir`. A note naming any other file is +/// observed under a label that no digest has, which TLC cannot match. +function observeRequest(body: string, s: string, asked: string[], sessionDir: string, label: (d: string) => string): Line { + const sent: string[] = [], noted: string[] = []; + const prompt = JSON.parse(body).prompt as Array<{ content?: unknown }>; + for (const message of prompt) { + if (!Array.isArray(message.content)) continue; + for (const part of message.content as Array>) { + if (part.type === "file" && part.data?.type === "data") { + sent.push(label(new Bun.CryptoHasher("sha256").update(Buffer.from(part.data.data, "base64")).digest("hex"))); + } + if (part.type !== "text") continue; + for (const match of String(part.text).matchAll(/^\[Image #\d+ \([^)]*\) is saved at ("(?:[^"\\]|\\.)*")\.\]$/gm)) { + const path = JSON.parse(match[1]!) as string; + const digest = path.split("/").pop()!; + let named = "unsaved"; + try { + if (path === join(sessionDir, "blobs", digest) && sha256File(path) === digest) named = label(digest); + } catch {} + noted.push(named); + } + } + } + return { event: "Request", s, asked: set(asked), sent: set(sent), noted: set(noted) }; +} + +/// The session's user item `at`, the last by default: each image by how the +/// item keeps it, and the blobs the item lists. +function observeWrite(sessionDir: string, s: string, label: (d: string) => string, at = -1): Line { + const items = readFileSync(join(sessionDir, "log.jsonl"), "utf8").split("\n").filter(Boolean) + .map(line => JSON.parse(line)).filter(line => line.kind === "item" && line.type === "user"); + const item = items.at(at); + const refs: string[] = [], inline: string[] = []; + for (const image of item.data.images as Array>) { + if (typeof image.blob === "string") refs.push(label(image.blob)); + else if (typeof image.inline_data === "string") { + inline.push(label(new Bun.CryptoHasher("sha256").update(Buffer.from(image.inline_data, "base64")).digest("hex"))); + } + } + // These runs store no other blob, so the item lists images only. + const listed = ((item.blobs ?? []) as string[]).map(label); + return { event: "Write", s, refs: set(refs), inline: set(inline), listed: set(listed) }; +} + +function blobLabels(sessionDir: string, label: (d: string) => string): string[] { + return readdirSync(join(sessionDir, "blobs")).map(label); +} + +async function judge(name: string, lines: Line[]) { + const dir = join(artifacts, name); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "trace.ndjson"), lines.map(line => JSON.stringify(line)).join("\n") + "\n"); + const convert = Bun.spawnSync(["python3", join(TLA_ROOT, "session_manager", "trace", "to_tla.py"), join(dir, "trace.ndjson"), join(dir, "TraceData.tla")]); + expect(convert.exitCode).toBe(0); + for (const spec of MODEL_SPECS) writeFileSync(join(dir, spec), readFileSync(join(TLA_ROOT, "session_manager", spec))); + const run = await runTlc(dir, "PromptImagesTrace", "PromptImagesTrace.cfg", TRACE_SPECS, join(TLA_ROOT, "session_manager", "trace")); + const matched = Math.max(0, ...[...run.output.matchAll(/<<"matched_lines", (\d+)>>/g)].map(m => Number(m[1]))); + return { accepted: run.violated === "TraceIncomplete", matched, lines: lines.length, dir }; +} + +test("prompt images become blobs the notes name, across a resume (D89)", async () => { + const f = fixture("resume"); + try { + const a = f.image("a.png", 20, 10), b = f.image("b.png", 10, 20); + const id = await f.ask("source", [a]); + // The same image again and a new one: one blob each, both noted. + await f.ask("source", [a, b], id); + f.blobs("source", id); + const verdict = await judge("resume", f.lines); + expect(verdict).toMatchObject({ accepted: true, matched: f.lines.length }); + + // Planted faults: a note naming the image's source file, an item that + // keeps a saved image inline, and a blob the item does not list. + const noteSource = f.lines.map(line => line.event === "Request" ? { ...line, noted: set([...(line.noted as any).$set, "unsaved"]) } : line); + expect((await judge("planted-note", noteSource)).accepted).toBe(false); + const keptInline = f.lines.map((line, index) => index === 1 ? { ...line, refs: set([]), inline: (line as any).refs } : line); + expect((await judge("planted-inline", keptInline)).accepted).toBe(false); + const unlisted = f.lines.map((line, index) => index === 1 ? { ...line, listed: set([]) } : line); + expect((await judge("planted-unlisted", unlisted)).accepted).toBe(false); + } finally { + f.stop(); + } +}, TIMEOUT * 3); + +test("an image the session cannot save stays inline with no note, and a saved one keeps its note (D89)", async () => { + const f = fixture("refused"); + try { + const a = f.image("a.png", 20, 10), c = f.image("c.png", 12, 12); + const id = await f.ask("source", [a]); + // The store can no longer add blobs: `a` is stored already, `c` is not. + const blobs = join(f.sessionsRoot, id, "blobs"); + chmodSync(blobs, 0o500); + try { + await f.ask("source", [a, c], id); + } finally { + chmodSync(blobs, 0o700); + } + f.blobs("source", id); + const verdict = await judge("refused", f.lines); + expect(verdict).toMatchObject({ accepted: true, matched: f.lines.length }); + const second = f.lines[2] as any; + expect(second.noted.$set).toEqual(["d1"]); + } finally { + f.stop(); + } +}, TIMEOUT * 2); + +test("a converted v1 session's prompt images become blobs its next request names (D89)", async () => { + const f = fixture("converted"); + try { + // v0.0.14 saved one turn with twenty attachments of one image. + const [id] = copyV1Fixture(f.home, "many-images"); + const v1Images = join(f.home, ".fx", "sessions", id, "images"); + for (const name of readdirSync(v1Images)) f.label(sha256File(join(v1Images, name))); + // Resuming converts the session, then sends a new image with the old. + await f.ask("source", [f.image("b.png", 10, 20)], id); + // The import wrote its item before the request went out. + f.lines.unshift(observeWrite(join(f.sessionsRoot, id), "source", f.label, 0)); + f.blobs("source", id); + const verdict = await judge("converted", f.lines); + expect(verdict).toMatchObject({ accepted: true, matched: f.lines.length }); + // The spec allows an image the store refused to stay inline, so the + // writable store's import must have saved it. + expect(f.lines[0]).toMatchObject({ refs: { $set: ["d1"] }, inline: { $set: [] } }); + expect((f.lines[1] as any).noted.$set).toEqual(["d1", "d2"]); + + // A planted fault: the imported item does not list its blob. + const unlisted = f.lines.map((line, index) => index === 0 ? { ...line, listed: set([]) } : line); + expect((await judge("planted-import-unlisted", unlisted)).accepted).toBe(false); + } finally { + f.stop(); + } +}, TIMEOUT * 2); diff --git a/tests/tla/image_route/check.sh b/tests/tla/image_route/check.sh new file mode 100755 index 000000000..1845654bb --- /dev/null +++ b/tests/tla/image_route/check.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Checks the image route design: TLC on ImageRoute.tla, both driver +# directions against the real core, every planted bug, and DESIGN.md +# traceability. `bun run verify` in this folder is the gate; this script is +# the tests/tla//check.sh entry point that runs it. +set -euo pipefail +cd "$(dirname "$0")" +exec bun run verify diff --git a/tests/tla/image_route/design/CONTEXT.md b/tests/tla/image_route/design/CONTEXT.md new file mode 100644 index 000000000..14a5167c0 --- /dev/null +++ b/tests/tla/image_route/design/CONTEXT.md @@ -0,0 +1,98 @@ +# Context: how images reach the model + +Line numbers refer to `origin/main` at `b0a90fd3a` unless a line says "this change". The design in `DESIGN.md` moves some of this code. `git show b0a90fd3a:` shows it as cited. + +## The report + +A libfx user building an AI SDK harness adapter reported that an image in a tool result never reaches the agent unless they pass a custom `model`. The default model, `spacexai/grok-4.7`, is listed in fx's model catalog with `vision` but without `file-input`, and fx requires both. The report held on libfx 0.0.13 and again on 0.0.14. The reproduction is the AI SDK example `examples/ai-functions/src/harness-agent/fx/with-to-model-output.ts`, lines 53–54, which set the model. Removing them makes the image disappear. + +Both catalogs fx reads were fetched on 2026-10-19: + +| Catalog | Used by | Models | `spacexai/grok-4.7` tags | +| --- | --- | --- | --- | +| `https://ai-gateway.vercel.sh/coding-agent/v1/models` | capability lookups (`src/builtins/gateway.zig:46`, the E2E fake at `tests/e2e/gateway-stream-lifecycle.test.ts:1305`), libfx `listModels` (`sdk/fx-sdk.js:611`) | 263 | `reasoning`, `tool-use`, `implicit-caching`, `vision`, `structured-output` | +| `https://ai-gateway.vercel.sh/v1/models` | the public catalog, and the fallback models path when no web-search path is set (`src/core/app/app_agent_runtime.zig:250`) | 478 | the same | + +In `/v1/models`, 73 models list `vision` without `file-input`, including the default and `spacexai/grok-4.6`. None lists `file-input` without `vision`. + +The AI Gateway documents the two tags as separate capabilities. Vision covers images such as a PNG screenshot. File input covers documents such as a PDF ([Inputs and Tools](https://vercel.com/docs/ai-gateway/inputs-and-tools)). For the AI SDK format that fx speaks, image input is "a `file` part with an `image/*` `mediaType`" ([Vision and Image Input](https://vercel.com/docs/ai-gateway/inputs-and-tools/vision)). + +## Entry points + +| Entry point | Path to the decision | +| --- | --- | +| `fx`, `fx ask` | `processQueuedPromptLoop` (`src/core/agent/runtime/orchestrator.zig:6915`) builds every model request of a turn | +| libfx `createFxEngine` (native and WebAssembly) | The ACP server in the Zig core, then the same loop. A host tool returns `{ type: "libfx.tool-result", text, images }` (`sdk/tests/test-agent-images.mjs:1266`), which becomes `tool_result_memory.tool_images` | +| `read_file` on an image | The same loop, with the image in `tool_result_memory` (`tests/e2e/file-tool-paths.test.ts:349-440`) | + +The default model is `spacexai/grok-4.7` (`src/builtins/gateway.zig:43`). + +## Data flow today + +1. The gateway catalog's `tags` array is parsed into `has_vision` and `has_file_input` (`src/builtins/gateway.zig:2755-2762`, fields at `src/core/gateway/model_catalog.zig:326`). +2. `fromCatalogEntry` copies them to `supports_vision` and `supports_file_input` (`src/core/gateway/model_catalog_metadata.zig:17-18`). +3. `mergeCapabilities` sets `image_input_support` to `.native` only when both are true, and to `.non_native` otherwise (`src/core/config/model_capabilities.zig:100-103`). Without catalog metadata it stays `.unknown` (`src/core/config/model_capabilities.zig:36-40`, `:66`). **This is the bug.** +4. `CapabilityResolver` caches the catalog per process. Its states are `idle`, `ready`, and `failed` (`src/core/gateway/gateway_provider.zig:148-160`). `resolve` fetches only when idle (`:217-219`). When the catalog failed, or the model is missing from it, `resolve` returns the local fallback, whose support is unknown (`:221-228`, `:242-247`). The catalog is refreshed at most every 60 seconds (`src/core/gateway/provider_versions.zig:12`, `gateway_provider.zig:177-181`), and a failed refresh keeps the last good catalog (`gateway_provider.zig:196-197`). +5. At turn start, the orchestrator resolves capabilities when the turn has images or when vision routing needs them (`orchestrator.zig:5772-5783`, `requiresResolvedRequestCapabilities` at `:5606-5618`). +6. For each request in the turn loop: + - `request_capabilities` is a turn-local copy (`orchestrator.zig:6942`). + - `visionPolicy` picks the user-image route and the vision tool mode from `image_input_support` (`orchestrator.zig:6196-6213`, called at `:7372-7377`). + - User images are projected for that route. A request with user images and no route fails with `ModelImageCapabilityUnavailable` or `SubscriptionNativeImageUnavailable` (`:7402-7437`). + - Only then, if tool images are present and support is unknown, capabilities are resolved again, at most once per turn (`:7485-7493`). + - Native support materializes the tool images inline (`:7494-7506`). Otherwise `projectToolImageMessages` strips them and adds a notice to the tool message (`src/core/agent/runtime/gateway_step.zig:268-300`). + - When support is unknown and the catalog is known to be down, the user is told once per turn (`orchestrator.zig:7523-7530`). +7. On the wire, every retained tool image goes out as a `{"type":"file","mediaType":"image/…","data":…}` part in a user message after the tool run (`src/gateway/vercel_protocol.zig:679-716`). The tool-result part keeps only its text (`:718-742`). User images are file parts too (`:780-823`). + +## Who owns each piece of state + +| State | Owner | Lifetime | Refresh | +| --- | --- | --- | --- | +| Which models take images | The AI Gateway catalog | External | Read by HTTP | +| Cached catalog | `CapabilityResolver` (`gateway_provider.zig:154-160`) | One process | Every 60 s, or when access changes | +| Request capabilities | The turn loop (`orchestrator.zig:5774`, `:6942`) | One turn | Once at turn start, and once more for tool images (`:7489`) | +| Whether the turn re-resolved | `tool_image_capabilities_resolved` (`orchestrator.zig:6943`) | One turn | Set once | +| Whether the user was told | `tool_image_strip_notified` (`orchestrator.zig:7079`) | One turn | Set once | +| Pending user images | `pending_image_ids` (`orchestrator.zig:7028-7033`) | One turn | As the model views them | +| Tool image bytes | The session image store, through `tool_result_memory.tool_images` and `tool_image_handle` (`src/core/shared/types.zig:1320-1338`) | The session | Dropped images keep a name without bytes (`types.zig:1335-1337`). `withholdDroppedImages` turns them into notes (`src/core/images/image_attachments.zig:862-889`) | + +## Contracts and saved formats + +- `ImageInputSupport` (`model_capabilities.zig:36-40`) is in-memory only. `/model` info prints it next to `supports_vision` and `supports_file_input` (`src/core/app/app_commands.zig:3048-3050`). +- The catalog `tags` array is an external contract that fx reads and never writes. +- The routing decision, the withheld-image notice, and the materialized image bytes are built per request in the overlay arena and never saved (`orchestrator.zig:7494-7507`). Sessions save tool images by handle or name, as before. No saved format records image support. +- The wire format is the AI SDK prompt with `file` parts. It doesn't change. + +## Callers of `image_input_support` + +`orchestrator.zig:5615`, `:7373`, `:7425`, `:7485`, `:7494`, `:7507`, `:7524`, and `:9849` (a suggestion in a model error), `gateway_step.zig:271-286`, `app_commands.zig:3048`, and tests in `src/core/agent/runtime/tests/gateway_flow.zig`, `tool_flow.zig`, `support.zig`, `src/core/gateway/gateway_provider.zig`, `src/core/subagent/agent_adapter.zig`, and `src/gateway/chat_completions.zig`. + +## Constraints + +- Zig 0.17, standard library only. The core imports nothing but `std`. +- The decision runs once per model request, never at startup. The startup budget is 2 ms per CLI path (`AGENTS.md`, Benchmarks). +- Permission handling is unchanged. Images from permission-denied results never leave the process (`vercel_protocol.zig:691-704`). +- libfx has two surfaces, native and WebAssembly, that share this Zig code. Both must pass `sdk/tests/test-agent-images.mjs`. +- The model never changes within a turn: `gateway_model` is always `job.model` (`orchestrator.zig:7273`, `:7300`). + +## Existing code that makes the wrong design look easy + +1. The wire part is named `file` (`vercel_protocol.zig:708`), so the `file-input` tag reads like a requirement for sending it. It isn't. The tag means documents. +2. Three providers that have no `file-input` tag set `has_file_input = vision` to satisfy the rule: configured Chat Completions providers (`src/gateway/chat_completions.zig:235`), Codex (`src/gateway/openai_codex_models.zig:286`), and Grok subscriptions (`src/gateway/xai_grok_models.zig:364`). That kept the bug invisible everywhere except the gateway catalog. +3. Every image test fixture pairs `vision` with `file-input`: `sdk/tests/test-agent-images.mjs:34`, `tests/e2e/file-tool-paths.test.ts:365`, `tests/e2e/vision-route-fake-gateway.test.ts:227-232`, and about 40 more (`git grep -n file-input -- tests sdk`). No test ran a vision-only catalog entry, though 73 live models are vision-only. +4. `processQueuedPrompt routes images natively only when vision and file input are both supported` (`src/core/agent/runtime/tests/gateway_flow.zig:2508-2537`) set `image_input_support` by hand from its expectation. It stated the rule without testing where it comes from. +5. The rule predates `ImageInputSupport`. Commit `77e62a3ac` (2026-08-30, "Hide Vision from native image models") carried `supports_vision and supports_file_input` over from the initial commit (`git show 77e62a3ac^:src/core/agent/runtime/orchestrator.zig`, lines 3462 and 3505). +6. The tool-image re-resolve sits after the route is chosen (`orchestrator.zig:7372` against `:7485`), the easiest place to add it. When it changes support, one request takes its route and vision tool mode from the old value and its tool images from the new one. + +## Fakes the drivers use + +The only fakes are effect handlers at the edge of a shell: + +| Fake | Where | Stands in for | +| --- | --- | --- | +| The catalog reply | `tests/tla/drivers/image_route/code-to-spec.ts`, `spec-to-code.ts` | The result of the core's `resolve_capabilities` effect, and the capabilities a turn starts with, given as a catalog view the harness turns into support with the real `supportFromCatalog` | +| Recording `send`, `fail`, and `notify` | `tests/tla/drivers/image_route/harness.zig` | Sending the request, failing the turn, and printing the notice. The harness prints the effects instead of running them | +| A crash | `Harness.crash` in `tests/tla/drivers/image_route/lib.ts` | A process dying. The driver kills the harness process with `SIGKILL` and starts a new one | +| The fake gateway | `startFakeGateway` (`tests/e2e/tmux-helpers.ts`) in `tests/e2e/file-tool-paths.test.ts` | The catalog and the model, for the built `fx` binary | +| `mockGateway` | `sdk/tests/test-agent-images.mjs:79-101` | The catalog and the model, for libfx | + +The drivers call the real `src/core/images/image_route.zig` through the harness. They never reimplement it. diff --git a/tests/tla/image_route/design/DESIGN.md b/tests/tla/image_route/design/DESIGN.md new file mode 100644 index 000000000..30dc3c5b3 --- /dev/null +++ b/tests/tla/image_route/design/DESIGN.md @@ -0,0 +1,143 @@ +# Design: image route + +How fx decides, for each model request, where images go: inline, through the vision tool, or nowhere with a notice. `CONTEXT.md` describes the code on `b0a90fd3a`. Line numbers here refer to that commit too, except those marked "this change", which refer to this branch. + +`tests/tla/drivers/image_route/traceability.ts` reads this file. It requires every question below to be answered or listed as not applicable, every rule and law to have a row under Enforcement, and every entry there to name a check that exists. + +## Questions + +- Q1: Values cross. The catalog sends `tags` strings, the core gets a `Support` enum and five `Request` flags (`image_route.zig:94-105`, this change), and image bytes stay as session handles until the shell materializes them for the provider (`orchestrator.zig:7463-7476`, this change). Logs get the decision's tag names (`orchestrator.zig:7375-7388`, this change). +- Q2: A core event is under 16 bytes, once per request, plus at most one capability request per turn. Images are at most 8 per result and 5 MiB encoded each (`image_data.zig:6-8`). At 100x the images per session, `requestMaxDimension` shrinks each one (`image_data.zig:168`), and the request budget withholds the rest (`request_image_budget.zig`). +- Q3: The gateway catalog owns capability. `CapabilityResolver` caches it per process and refreshes every 60 s (`gateway_provider.zig:167-207`). The core holds one snapshot per turn and refreshes it at most once in the turn, only when images wait on unknown support. +- Q4: The largest inline value is one tool image, 5 MiB encoded at most, bounded by `withholdRequestToolImages` (`execution_memory.zig:800`) and the request budget. Notices are fixed text cut to `max_tool_result_bytes` (`gateway_step.zig:285-293`, this change). +- Q5: Yes. The model gets images only when it can read them, a notice only when they are withheld, and the core gets flags, never bytes. +- Q6: One turn loop per session builds requests. Parallel tools finish before the next request is built, and subagents run their own turn loops. No two writers share a core state. +- Q7: The turn loop is the core state's only writer (`orchestrator.zig:6888-6891`, this change), so nothing is lost. +- Q8: A reply nothing waits for, stale or duplicated, is ignored. A request delivered while a reply is owed replaces the waiting one, and the reply answers the newest. A reply after its turn ended meets a reset state. +- Q9: Yes, today. The route is chosen before tool images trigger a second catalog check (`orchestrator.zig:7372` against `:7485`), so one request can mix two support values. The design asks first and decides after the reply (`image_route.zig:166-190`, this change), and `OneSnapshot` checks it. +- Q10: The resolver's catalog, which its owner already guards against concurrent readers (`gateway_provider.zig:165-166`). It is unchanged. The core's state is a turn-local value. +- Q11: A request delivered twice runs nothing more (`DuplicateRequestIsQuiet`). Model request retries are the gateway step's, unchanged. +- Q12: The capability request is a GET and safe to repeat, but the core never repeats it within a turn. A lost reply ends with the turn, and the next turn asks again. +- Q13: `resolve_capabilities` is idempotent, keyed per turn by `resolved`. `notify_catalog_down` is not idempotent, keyed per turn by `notified`. `send` and `fail` happen once per request, and the core never re-emits them. +- Q14: The catalog fetch keeps the transport's deadline. A timeout leaves the resolver failed and support unknown, so the request goes out without images and with the "could not confirm" notice. The caller assumes nothing about vision. +- Q15: A slow, down, rate-limited, or malformed catalog leaves the resolver failed (`gateway.zig:2713-2715` rejects malformed JSON), which the core sees as unknown support. The request still goes out, without images. +- Q16: The core's state lives in memory and dies with the process. Tool images stay in the session store, and the next process decides again from its first turn. +- Q18: Unknown support with a down catalog: one notice per turn in the transcript (`orchestrator.zig:7493-7496`, this change), and a model-facing note that it may recover. No vision: the model is told to call the vision tool, or to ask for a vision model. User images with no route: `ModelImageCapabilityUnavailable` or `SubscriptionNativeImageUnavailable`, as before. Each recovers on a later turn once the catalog answers. +- Q19: Nothing. A withheld image stays in the store with a notice (`NoSilentWithhold`), session-dropped images become notes (`image_attachments.zig:883-887`), and the `vision_policy` trace line now records the tool image decision (`orchestrator.zig:7375-7388`, this change). +- Q21: None in the core. The resolver's refresh interval uses `io_mod.milliTimestamp` (`gateway_provider.zig:173`), unchanged. +- Q22: The core has one waiting request and at most two effects per step (`Bounded`). Images in history are bounded by the existing caps in Q2. +- Q24: One `step` call and one scan of the request's messages per model request (`requestHasToolImages`, `orchestrator.zig:6199-6208`, this change). Nothing runs at startup, per keystroke, or per token. +- Q25: The scan is linear in the request's history, like the projections beside it that already walk every message (`withholdDroppedImages`, `materializeToolImages`). +- Q26: One core state per turn, turns run one at a time per session, and each subagent has its own. +- Q27: No saved format or protocol changes. `ImageInputSupport` keeps its tags and moves its definition to `image_route.Support` (`model_capabilities.zig:37`, this change). Saved sessions resume unchanged, and their images now reach vision-only models. +- Q29: `/model` info prints image support (`app_commands.zig:3048-3050`) and now shows `native` for vision-only models. The CLI, TUI, ACP, and libfx share the code. Older releases keep the old rule. +- Q30: The gateway controls the tags over TLS. A repository cannot set capabilities, because project `.fx.json` accepts no profile-owned keys (`config_runtime.zig` `parseProjectSafeFields`). Tool output reaches the core only as a flag that images exist. +- Q31: Nowhere. Core events and the trace line hold only model ids and tag names. +- Q32: No. Images from permission-denied results still never leave the process (`vercel_protocol.zig:691-704`). +- Q33: Dropping `file_input` from one condition (`model_capabilities.zig:100` on `b0a90fd3a`) fixes the reported bug. +- Q34: The core extraction. Without it the reported bug is still fixed, but Q9's ordering fix has no function to check against a model. The design keeps it for that reason and for G6; see R10. +- Q35: It reuses the resolver, `projectToolImageMessages`, and the wire encoder. It replaces `visionPolicy` and the inline re-resolve, so no parallel path remains, and it drops both inherited flaws: the `file-input` rule and the late re-resolve. +- Q36: Nothing. No setting, override, or provider table. +- Q37: `tests/e2e/file-tool-paths.test.ts` through the built binary, `sdk/tests/test-agent-images.mjs` through libfx on both backends, the drivers on the real core, and a live run of the built binary against the gateway (`GOAL.md`, item 6). +- Q38: The `vision_policy` trace line logs `image_support`, `route`, `mode`, `pending_images`, and `tool_images` for every request, and `image_gate` diagnostics record each failure (`orchestrator.zig:7357-7366`, this change). +- Q39: Revert the change. No data needs migrating. Reverting `supportFromCatalog` alone (`image_route.zig:47-52`, this change) restores the old rule. + +Not applicable: Q17, Q20, Q23, Q28 (the core writes nothing that can be partial, holds no process, file, lock, or connection, consumes one synchronous event at a time, and changes no saved format, so nothing needs migrating). + +## Laws + +The laws `step` obeys. Each one is a named invariant or property in `spec/ImageRoute.tla`, checked by TLC over the whole model and on traces of the real core. + +- **L1 VisionImpliesNative**: a model the catalog lists with `vision` takes images inline, whatever its `file-input` tag says. One listed without vision does not, and an unlisted model or an unreachable catalog leaves support unknown. +- **L2 RouteModeCoherent**: the vision tool is offered exactly on the fallback route. +- **L3 NativeExcludesFallback**: native delivery and the vision tool never meet in one request. +- **L4 OneSnapshot**: a request's route, vision tool mode, and tool image handling all follow the one support value the state holds after the step (ordering). +- **L5 UnknownNeverInline**: no image goes to a model the catalog has not confirmed (ownership: the catalog decides). +- **L6 NoSilentWithhold**: tool images are attached, or withheld with a notice the model can act on, never neither. The vision tool is required exactly while user images wait for it (`RequiredWhenPending`). +- **L7 FailExactlyWhenUnroutable**: a request with user images and no route fails, with the reason that matches support. Nothing else fails. +- **L8 ResolveOncePerTurn**: the core asks for capabilities only when images wait on unknown support (`ResolveOnlyWhenNeeded`), and at most once per turn (idempotence). This also makes the shell's effect loop end. +- **L9 NotifyOncePerTurn**: the user hears about a down catalog only for tool images it withholds (`NotifyOnlyWhenDown`), and at most once per turn (idempotence). +- **L10 DuplicateRequestIsQuiet**: a request delivered while a reply is owed runs nothing, and a reply nothing waits for changes nothing (`StaleReplyIsQuiet`) (idempotence and ordering). +- **L11 Bounded**: at most one waiting request, and at most two effects per step, a notice and then one answer (bounds). +- **L12 Answered**: every request that waits for a reply is answered, unless its turn ends first (liveness). +- **L13 DecideOnlyWhenReady**: a decision comes only once no reply is owed. While one is owed, the turn has asked once and decided nothing (`PendingWaitsForReply`) (ordering). +- **L14 TurnResets**: a new turn, and a new process after a crash, start with no request waiting and no capability request or notice spent. + +## Enforcement + +### Rules + +| Rule | Status | Reason | Enforced by | +| --- | --- | --- | --- | +| R1 | pass | Core events carry flags only. Bytes stay as session handles until the shell materializes them, and base64 exists only in the AI SDK file part at the transport (`vercel_protocol.zig:706-712`). | `zig:src/core/images/image_route.zig#requests carry flags, not image bytes`, `review:the shell's materialization and the wire encoder are unchanged; a reviewer checks any new caller of image_route passes flags, which the Request type enforces at compile time` | +| R2 | fail | Inherited and accepted. The AI SDK prompt has no image reference form, so every request resends the images in its history, now also for vision-only models. The existing caps bound each request. | `review:per-request image bytes stay bounded by request_image_budget.zig and withholdRequestToolImages; sending once by reference would need provider-side storage, a non-goal in GOAL.md` | +| R3 | pass | Notices are fixed text cut to the result limit. Images are bounded by the existing caps, and the core never holds bytes. | `zig:src/core/agent/runtime/gateway_step.zig#projectToolImageMessages names the dead end when no fallback exists`, `inv:Bounded` | +| R4 | pass | The gateway catalog is the source, the resolver caches it, and the core keeps one snapshot per turn that only a reply to its own request replaces. | `inv:VisionImpliesNative`, `inv:OneSnapshot`, `prop:StaleReplyIsQuiet` | +| R5 | pass | One event per request, at most one capability request per turn, at most two effects per step. Image bytes per request are capped as in Q2. | `inv:Bounded`, `prop:ResolveOncePerTurn`, `inv:ResolveOnlyWhenNeeded` | +| R6 | pass | Only the capability request repeats safely, and the core still sends it once per turn. Sends, failures, and notices are never re-emitted. | `prop:DuplicateRequestIsQuiet`, `prop:NotifyOncePerTurn`, `prop:ResolveOncePerTurn` | +| R7 | pass | Crashes, a down or missing catalog, and lost, stale, and duplicated replies are actions in the model, and the code is checked on traces that inject them. | `inv:UnknownNeverInline`, `inv:TurnResets`, `prop:Answered`, `inv:FailExactlyWhenUnroutable`, `driver:code-to-spec` | +| R8 | n/a | No saved format or protocol changes (Q27). The routing decision is never saved. | | +| R9 | pass | It drops two inherited flaws: the `file-input` rule and the re-resolve after the route is chosen. | `mutation:S1`, `mutation:C1`, `mutation:S2`, `mutation:C7`, `inv:OneSnapshot` | +| R10 | fail | Accepted. The reported bug needs one line. The pure core, its harness, and the drivers exist so the ordering fix (Q9) is checkable against a model, as G6 requires. | `review:the core adds no setting, cache, or provider table; Q34 and Final design name what could be removed`, `driver:spec-to-code` | +| R11 | n/a | No mutable resource is opened by more than one process. The core's state is a value local to one turn. | | + +### Laws + +| Law | Statement | Enforced by | +| --- | --- | --- | +| L1 | Vision means native | `inv:VisionImpliesNative`, `trace:VisionImpliesNative`, `mutation:S1`, `mutation:C1`, `zig:src/core/images/image_route.zig#a catalog that lists vision makes image input native with or without file input`, `zig:src/core/config/model_capabilities.zig#image input support distinguishes unknown native and non native capability`, `zig:src/core/agent/runtime/tests/gateway_flow.zig#processQueuedPrompt routes images natively whenever the catalog lists vision`, `e2e:tests/e2e/file-tool-paths.test.ts#read_file attaches a workspace image inline for a vision model without file input`, `sdk:sdk/tests/test-agent-images.mjs#A host tool's image reaches a model the catalog lists with vision but not` | +| L2 | Vision tool only on the fallback route | `inv:RouteModeCoherent`, `trace:RouteModeCoherent`, `mutation:S18`, `mutation:C17`, `zig:src/core/images/image_route.zig#every decision reads one support value` | +| L3 | Native excludes the vision tool | `inv:NativeExcludesFallback`, `trace:NativeExcludesFallback`, `mutation:S19`, `mutation:C18`, `sdk:sdk/tests/test-agent-images.mjs#a native image model is not offered the vision tool` | +| L4 | One snapshot per request | `inv:OneSnapshot`, `trace:OneSnapshot`, `mutation:S2`, `mutation:C7` | +| L5 | Unknown never inline | `inv:UnknownNeverInline`, `trace:UnknownNeverInline`, `mutation:S5`, `mutation:C4`, `zig:src/core/agent/runtime/gateway_step.zig#projectToolImageMessages tells the truth when the catalog is unavailable` | +| L6 | No silent withhold | `inv:NoSilentWithhold`, `inv:RequiredWhenPending`, `trace:NoSilentWithhold`, `trace:RequiredWhenPending`, `mutation:S17`, `mutation:C16`, `mutation:S10`, `mutation:C10`, `zig:src/core/agent/runtime/orchestrator.zig#requestHasToolImages ignores images the session dropped`, `e2e:tests/e2e/file-tool-paths.test.ts#read_file image is withheld with guidance for a text-only model` | +| L7 | Fail exactly when unroutable | `inv:FailExactlyWhenUnroutable`, `trace:FailExactlyWhenUnroutable`, `mutation:S8`, `mutation:C9`, `zig:src/core/images/image_route.zig#user images without a route fail the request and never send` | +| L8 | One capability request per turn, only for images | `inv:ResolveOnlyWhenNeeded`, `prop:ResolveOncePerTurn`, `trace:ResolveOnlyWhenNeeded`, `trace:ResolveOncePerTurn`, `mutation:S3`, `mutation:C2`, `mutation:S15`, `mutation:C14`, `zig:src/core/images/image_route.zig#a turn asks for capabilities at most once and answers the request it held` | +| L9 | One catalog-down notice per turn, only when down | `inv:NotifyOnlyWhenDown`, `prop:NotifyOncePerTurn`, `trace:NotifyOnlyWhenDown`, `trace:NotifyOncePerTurn`, `mutation:S4`, `mutation:C3`, `mutation:S16`, `mutation:C15`, `zig:src/core/images/image_route.zig#the catalog-down notice reaches the user once per turn` | +| L10 | Duplicates and stale replies are quiet | `prop:DuplicateRequestIsQuiet`, `prop:StaleReplyIsQuiet`, `trace:DuplicateRequestIsQuiet`, `trace:StaleReplyIsQuiet`, `mutation:S6`, `mutation:C5`, `mutation:S7`, `mutation:C6` | +| L11 | Bounded | `inv:Bounded`, `trace:Bounded`, `mutation:S12`, `mutation:C11`, `zig:src/core/images/image_route.zig#abstract maps the pending request onto a sequence of at most one` | +| L12 | Every waiting request is answered | `prop:Answered`, `mutation:S11`, `review:a finite trace cannot show liveness; TLC proves Answered over the whole model under fairness of replies, and ConformsToSpec makes each recorded reply clear its request` | +| L13 | Decide only when no reply is owed | `inv:DecideOnlyWhenReady`, `inv:PendingWaitsForReply`, `trace:DecideOnlyWhenReady`, `trace:PendingWaitsForReply`, `mutation:S13`, `mutation:C12`, `mutation:S14`, `mutation:C13` | +| L14 | Turns and processes start clean | `inv:TurnResets`, `trace:TurnResets`, `mutation:S9`, `mutation:C8` | + +## Final design + +### The fix + +`supportFromCatalog` (`src/core/images/image_route.zig:47-52`, this change) returns `native` when the catalog lists `vision` and `non_native` when it lists the model without it, and `unknown` for a missing model or a down catalog. `file-input` plays no part. `mergeCapabilities` calls it (`src/core/config/model_capabilities.zig:97-100`, this change), so the CLI, TUI, ACP, and libfx all change at once. `ImageInputSupport` is now an alias of `image_route.Support` (`model_capabilities.zig:37`, this change). + +### The core + +`src/core/images/image_route.zig` (this change) imports only `std`. + +- **State** (`:107-116`): `support`, the turn's one capability snapshot; `resolved` and `notified`, the turn's spent capability request and notice; `pending`, the request waiting for a reply, if any. +- **Events** (`:118-127`): `turn_started(Support)`, `request_ready(Request)`, and `capabilities_resolved(Support)`. Time does not enter. I/O results arrive as `capabilities_resolved`, and failures arrive as the support value they produce. +- **Effects** (`:135-146`), a tagged union of values: `resolve_capabilities`, `notify_catalog_down`, `send(Decision)`, and `fail(FailReason)`. At most two per step, returned in a fixed buffer with no allocation (`:148-156`). +- **`step`** (`:158-164`) dispatches to three pure functions: `onRequest` (`:166-181`), `onResolved` (`:183-190`), and `decide` (`:192-205`). `decision` (`:207-225`) maps one support value and one request to a route, a vision tool mode, and a tool image handling. Absence is `?Request`, and errors are the `fail` effect. Nothing panics, and mutation is limited to `var next` copies inside one function. +- **`abstract`** (`:235-251`) maps a state onto the spec's `support`, `resolved`, `notified`, and `pending`. + +### The shell + +The turn loop in `processQueuedPromptLoop` is the only code that runs effects. + +- It starts the core's state with `turn_started` (`orchestrator.zig:6888-6891`, this change). +- For each request, it builds the `Request` flags, including whether a tool image survives `withholdDroppedImages` (`requestHasToolImages`, `:6199-6208`). +- It then runs the effect loop: a capability request becomes `deps.resolve_model_capabilities` and its reply event, a notice is recorded, and `send` or `fail` is taken (`:7325-7366`). +- The decision drives the user-image projection, the vision tool mode, tool image materialization, and the notice (`:7367-7496`). +- `projectToolImageMessages` takes the core's `ToolImages` value instead of recomputing it from support and a flag (`gateway_step.zig:264-295`, this change). + +`visionPolicy`, `ImageRoute`, `VisionPolicy`, and the per-turn flags `tool_image_capabilities_resolved` and `tool_image_strip_notified` are gone, along with their tests. The core's tests and the model cover the same laws. + +### Mechanisms added, and the answers that required them + +| Mechanism | Required by | +| --- | --- | +| Support from `vision` alone | Q1 and R9: the inherited rule reads a document tag as an image tag | +| Asking before deciding, with the request held in `pending` | Q9: deciding first splits one request across two snapshots | +| `pending` replaced by a newer request, and stale replies ignored | Q8 and Q11 | +| A pure core with `abstract` and a harness | Q37 and G6: the orchestrator cannot be checked against a model, and a pure step can | + +### Rules this design breaks + +- **R2, send only what is needed.** Every request still resends the images in its history, now also to the 73 vision-only models. The AI SDK prompt offers no reference form, and the existing caps bound each request. Sending by reference would need provider-side storage, a non-goal in `GOAL.md`. +- **R10, build only what the problem needs.** The reported bug needs only `supportFromCatalog`'s one condition. The core extraction (about 250 lines with tests), the harness, and the drivers exist for the verification this workspace is for, and they make the ordering fix (Q9) checkable. Without that requirement, the smallest design is the one-line fix plus moving the orchestrator's re-resolve ahead of `visionPolicy`. diff --git a/tests/tla/image_route/design/GOAL.md b/tests/tla/image_route/design/GOAL.md new file mode 100644 index 000000000..3226b0db4 --- /dev/null +++ b/tests/tla/image_route/design/GOAL.md @@ -0,0 +1,44 @@ +# Goal + +## Problem + +fx never sends an image, from a tool or from the user, to a model that the catalog lists with `vision` but without `file-input`. The image stays in the session, and the model gets a notice that it should call the vision tool instead. fx's own default model, `spacexai/grok-4.7`, is one of the 73 models the gateway catalog lists this way (`CONTEXT.md`, The report). A libfx user hit this when an AI SDK tool returned an image to the default model: the agent never saw the image unless they set a different model. + +The cause is one rule: image input counts as native only when the catalog lists both `vision` and `file-input` (`src/core/config/model_capabilities.zig:100-103` on `b0a90fd3a`). The AI Gateway uses `file-input` for documents such as PDFs. fx sends images as AI SDK file parts with an image media type, and those need only `vision`. + +The same code has a second flaw. When tool images appear while support is unknown, the turn asks the catalog again, but only after the request's route and vision tool mode are already chosen (`orchestrator.zig:7372` against `:7485`). That one request can mix two capability values. + +## Success criteria + +Each criterion is a check that passes on this change and failed, or could fail, before it. + +| # | Criterion | Check | +| --- | --- | --- | +| G1 | A `read_file` image reaches a model the catalog lists with the default model's exact tags, inline, as an `image/png` file part. | `tests/e2e/file-tool-paths.test.ts`, "read_file attaches a workspace image inline for a vision model without file input". It fails on `b0a90fd3a` | +| G2 | A libfx host tool's image reaches that model inline, on the native and WebAssembly backends, and the model is not offered the vision tool. | `sdk/tests/test-agent-images.mjs native` and `wasm`. The new case fails on `b0a90fd3a` | +| G3 | A model listed without `vision` still gets the vision tool notice, and unknown support still sends no image. | The text-only E2E case, plus `UnknownNeverInline` in `ImageRoute.tla` and on code traces | +| G4 | Every request's route, vision tool mode, and tool image handling come from one capability value. | `OneSnapshot` in `ImageRoute.tla` and on code traces, and planted bugs S2 and C7 | +| G5 | No extra catalog traffic: at most one capability request per turn from the image decision, and none for a request without images. | `ResolveOncePerTurn` and `ResolveOnlyWhenNeeded` | +| G6 | The decision is one pure function, checked against its model in both directions. | `bun run verify` in `tests/tla/image_route` | +| G7 | Nothing slows startup or adds network I/O on a path without images. | The `Startup Latency` CI job, and `ResolveOnlyWhenNeeded` | + +## Non-goals + +- Supporting documents (PDFs) as tool results. The `file-input` tag would govern that, and fx does not send documents today. +- Changing the gateway catalog or asking for different tags. +- Removing the `has_file_input = vision` stand-ins in the Codex, Grok subscription, and Chat Completions providers (`CONTEXT.md`, item 2). They are now harmless, and changing them changes only what `/model` info prints. +- Sending images once and referring to them afterwards (rule R2). The AI SDK prompt has no reference form, so every request resends the images in its history, as it already does for models with `file-input`. +- A setting or override for image support. + +## What 100% done means + +Every item below passes for the exact commit that ships. + +1. `bun run verify` in `tests/tla/image_route` exits 0: the model check, spec to code over every edge, code to spec, every planted bug caught, and traceability. +2. `zig fmt --check src/` exits 0. +3. `zig build test` exits 0. +4. `FX_SOUND=0 bun test file-tool-paths.test.ts -t 'read_file (attaches|image is withheld)'` in `tests/e2e` passes all three cases. +5. `node sdk/tests/test-agent-images.mjs native` and `node --experimental-wasm-jspi sdk/tests/test-agent-images.mjs wasm` pass. +6. A live run of `./zig-out/bin/fx` with `FX_MODEL=spacexai/grok-4.7`, reading a PNG with `read_file`, answers with what the image shows, with clean stderr. +7. CI passes for the exact commit: `Build & Test`, `E2E (deterministic)`, `Shellcheck`, `Startup Latency`, and `macOS arm64`. This needs a pushed branch, which happens only when asked. +8. `tests/tla/README.md` lists `image_route/`. The release note belongs to the next release PR, which the maintainer writes. diff --git a/tests/tla/image_route/design/VISION.md b/tests/tla/image_route/design/VISION.md new file mode 100644 index 000000000..0ad290a94 --- /dev/null +++ b/tests/tla/image_route/design/VISION.md @@ -0,0 +1,32 @@ +# Vision + +## The end state + +fx decides how images reach the model from one catalog fact: whether the model has vision. A model with vision gets every image inline, from the user or from a tool, through the one wire form the gateway documents for images. A model without vision gets the vision tool, or a plain explanation when no vision tool exists. A model fx cannot confirm gets no images, a notice that says so, and one more catalog check in the turn. + +That decision is one pure function, `image_route.step(state, event)`, in `src/core/images/image_route.zig`. It reads no clock, environment, or global and performs no I/O. The orchestrator's turn loop is its shell: it runs the effects `step` returns and feeds their results back as events. A TLA+ model states the laws the function keeps, and Bun drivers check the real function against the model, in both directions, on every change. + +## What the system is + +- **One snapshot per request.** Every part of a request's image handling reads the same capability value, even when that value changes mid-turn. +- **Honest about what it withholds.** An image that does not reach the model always leaves a notice the model can act on. The user hears about it once per turn when the catalog itself is down. +- **Bounded.** One waiting request, at most two effects per step, and at most one catalog request per turn from the image decision. +- **Shared by every surface.** The CLI, the TUI, ACP, and both libfx backends run the same core. + +## What it refuses to be + +- **A guess.** fx sends no image to a model the catalog does not confirm, even one whose name suggests vision. Unknown means withheld. +- **A second catalog.** fx keeps no table of which models take images, and offers no setting to override the gateway. The catalog is the source of truth, and a wrong tag gets fixed there. +- **A retry loop.** The core never repeats a send, a failure, or a notice, and never asks the catalog twice in a turn. +- **Provider-aware.** The core knows support values, not provider names. Provider quirks stay in the provider code that maps a catalog to support. + +## Tradeoffs it makes on purpose + +| Choice | Over | Because | +| --- | --- | --- | +| Trust the `vision` tag | Also requiring `file-input`, or probing a model with a test image | The gateway defines image input as `vision`. If a vision-tagged route rejects an image, the request fails visibly instead of silently losing the image | +| Resend images with every request | Uploading once and sending references | The AI SDK prompt has no reference form. Existing caps bound every request (`request_image_budget.zig`, `image_data.requestMaxDimension`) | +| One catalog request per turn | Re-checking on every request | Bounded latency and traffic matter more than seeing a recovery a few seconds sooner. The next turn checks again | +| Ask before deciding | Deciding first and re-checking for tool images | Deciding first is what split one request across two values | +| A pure core with a thin shell | Leaving the decision inline in the orchestrator | The orchestrator cannot be model checked. A pure step with `abstract()` can be checked edge by edge against the model | +| Tell the user once per turn | Every request, or never | Every request is noise. Never hides a down catalog the user can do something about | diff --git a/tests/tla/image_route/mutations.json b/tests/tla/image_route/mutations.json new file mode 100644 index 000000000..8d3caa749 --- /dev/null +++ b/tests/tla/image_route/mutations.json @@ -0,0 +1,266 @@ +{ + "about": "Planted bugs. Each one breaks one guard, and the named property must fail. Spec mutations edit spec/ImageRoute.tla and run TLC on MCImageRoute.cfg. Code mutations edit a copy of src/core/images/image_route.zig, rebuild the harness from it, and run both driver directions. Each find string must occur exactly once.", + "spec": [ + { + "id": "S1", + "breaks": "Image support requires file-input as well as vision: the reported bug.", + "find": "(IF v.vision THEN \"native\" ELSE \"non_native\")", + "replace": "(IF v.vision /\\ v.file_input THEN \"native\" ELSE \"non_native\")", + "expect": "VisionImpliesNative" + }, + { + "id": "S2", + "breaks": "A reply's decision reads the support from before the reply.", + "find": "/\\ out' = DecideOut(s, Head(pending), notified)", + "replace": "/\\ out' = DecideOut(support, Head(pending), notified)", + "expect": "OneSnapshot" + }, + { + "id": "S3", + "breaks": "A turn asks for capabilities again after it already asked.", + "find": "NeedsResolve(r) == support = \"unknown\" /\\ ~resolved /\\ (r.user_images \\/ r.tool_images)", + "replace": "NeedsResolve(r) == support = \"unknown\" /\\ (r.user_images \\/ r.tool_images)", + "expect": "ResolveOncePerTurn" + }, + { + "id": "S4", + "breaks": "The catalog-down notice repeats within a turn.", + "find": "/\\ r.catalog_down /\\ ~n", + "replace": "/\\ r.catalog_down", + "expect": "NotifyOncePerTurn" + }, + { + "id": "S5", + "breaks": "Tool images go inline while support is unknown.", + "find": "[] s = \"unknown\" -> \"withheld_unknown\"", + "replace": "[] s = \"unknown\" -> \"attached\"", + "expect": "UnknownNeverInline" + }, + { + "id": "S6", + "breaks": "A request delivered twice asks for capabilities twice.", + "find": "THEN pending' = <> /\\ out' = <<>> /\\ UNCHANGED <>", + "replace": "THEN pending' = <> /\\ out' = <<[kind |-> \"resolve\"]>> /\\ UNCHANGED <>", + "expect": "DuplicateRequestIsQuiet" + }, + { + "id": "S7", + "breaks": "A stale reply overwrites the turn's support.", + "find": "THEN out' = <<>> /\\ UNCHANGED <>", + "replace": "THEN out' = <<>> /\\ support' = Derive(v) /\\ UNCHANGED <>", + "expect": "StaleReplyIsQuiet" + }, + { + "id": "S8", + "breaks": "User images without a route are sent anyway.", + "find": "Unroutable(s, r) == r.user_images /\\ Decide(s, r).route = \"unavailable\"", + "replace": "Unroutable(s, r) == FALSE", + "expect": "FailExactlyWhenUnroutable" + }, + { + "id": "S9", + "breaks": "A new turn keeps the previous turn's notice flag.", + "find": "/\\ notified' = FALSE \\* a new turn may tell the user again", + "replace": "/\\ notified' = notified", + "expect": "TurnResets" + }, + { + "id": "S10", + "breaks": "The vision tool is optional even while user images wait.", + "find": "ELSE IF r.pending_images THEN \"required\" ELSE \"optional\"", + "replace": "ELSE \"optional\"", + "expect": "RequiredWhenPending" + }, + { + "id": "S11", + "breaks": "A reply leaves its request waiting.", + "find": "/\\ support' = s\n /\\ pending' = <<>>", + "replace": "/\\ support' = s\n /\\ pending' = pending", + "expect": "Answered" + }, + { + "id": "S12", + "breaks": "The catalog-down notice comes after the answer.", + "find": "ELSE IF Notifies(s, r, n) THEN <<[kind |-> \"notify\"], Decide(s, r)>>", + "replace": "ELSE IF Notifies(s, r, n) THEN < \"notify\"]>>", + "expect": "Bounded" + }, + { + "id": "S13", + "breaks": "A request delivered while a reply is owed is decided at once.", + "find": "THEN pending' = <> /\\ out' = <<>> /\\ UNCHANGED <>", + "replace": "THEN pending' = <> /\\ out' = DecideOut(support, r, notified) /\\ UNCHANGED <>", + "expect": "DecideOnlyWhenReady" + }, + { + "id": "S14", + "breaks": "Asking for capabilities does not mark the turn as having asked.", + "find": "THEN /\\ resolved' = TRUE", + "replace": "THEN /\\ resolved' = resolved", + "expect": "PendingWaitsForReply" + }, + { + "id": "S15", + "breaks": "The core asks for capabilities for a request without images.", + "find": "NeedsResolve(r) == support = \"unknown\" /\\ ~resolved /\\ (r.user_images \\/ r.tool_images)", + "replace": "NeedsResolve(r) == support = \"unknown\" /\\ ~resolved", + "expect": "ResolveOnlyWhenNeeded" + }, + { + "id": "S16", + "breaks": "The catalog-down notice appears when the catalog is up.", + "find": "/\\ r.catalog_down /\\ ~n", + "replace": "/\\ ~n", + "expect": "NotifyOnlyWhenDown" + }, + { + "id": "S17", + "breaks": "Tool images vanish without a notice while support is unknown.", + "find": "[] s = \"unknown\" -> \"withheld_unknown\"", + "replace": "[] s = \"unknown\" -> \"none\"", + "expect": "NoSilentWithhold" + }, + { + "id": "S18", + "breaks": "The vision tool is offered off the fallback route.", + "find": "mode == IF route # \"fallback\" THEN \"unavailable\"", + "replace": "mode == IF route # \"fallback\" THEN \"optional\"", + "expect": "RouteModeCoherent" + }, + { + "id": "S19", + "breaks": "A native image model is routed through the vision tool.", + "find": "LET route == CASE s = \"native\" -> \"native\"", + "replace": "LET route == CASE s = \"native\" -> (IF r.fallback THEN \"fallback\" ELSE \"native\")", + "expect": "NativeExcludesFallback" + } + ], + "code": [ + { + "id": "C1", + "breaks": "Image support requires file-input as well as vision: the reported bug.", + "find": ".listed => |tags| if (tags.vision) .native else .non_native,", + "replace": ".listed => |tags| if (tags.vision and tags.file_input) .native else .non_native,", + "expect": "VisionImpliesNative" + }, + { + "id": "C2", + "breaks": "A turn asks for capabilities again after it already asked.", + "find": "if (state.support == .unknown and !state.resolved and (request.user_images or request.tool_images)) {", + "replace": "if (state.support == .unknown and (request.user_images or request.tool_images)) {", + "expect": "ResolveOncePerTurn" + }, + { + "id": "C3", + "breaks": "The catalog-down notice repeats within a turn.", + "find": "if (chosen.tool_images == .withheld_unknown and request.catalog_down and !state.notified) {", + "replace": "if (chosen.tool_images == .withheld_unknown and request.catalog_down) {", + "expect": "NotifyOncePerTurn" + }, + { + "id": "C4", + "breaks": "Tool images go inline while support is unknown.", + "find": ".unknown => .withheld_unknown,", + "replace": ".unknown => .attached,", + "expect": "UnknownNeverInline" + }, + { + "id": "C5", + "breaks": "A request delivered twice asks for capabilities twice.", + "find": "next.pending = request;\n return .{ .state = next };", + "replace": "next.pending = request;\n return withEffects(next, &.{.resolve_capabilities});", + "expect": "DuplicateRequestIsQuiet" + }, + { + "id": "C6", + "breaks": "A stale reply overwrites the turn's support.", + "find": "const request = state.pending orelse return .{ .state = state };", + "replace": "const request = state.pending orelse return .{ .state = .{ .support = support, .resolved = state.resolved, .notified = state.notified } };", + "expect": "StaleReplyIsQuiet" + }, + { + "id": "C7", + "breaks": "A reply's decision reads the support from before the reply: the old orchestrator order.", + "find": "next.support = support;\n next.pending = null;\n return decide(next, request);", + "replace": "next.pending = null;\n var stale = decide(next, request);\n stale.state.support = support;\n return stale;", + "expect": "OneSnapshot" + }, + { + "id": "C8", + "breaks": "A new turn keeps the previous turn's notice flag.", + "find": ".turn_started => |support| .{ .state = .{ .support = support } },", + "replace": ".turn_started => |support| .{ .state = .{ .support = support, .notified = state.notified } },", + "expect": "TurnResets" + }, + { + "id": "C9", + "breaks": "User images without a route are sent anyway.", + "find": "if (request.user_images and chosen.route == .unavailable) {", + "replace": "if (request.user_images and chosen.route == .unavailable and false) {", + "expect": "FailExactlyWhenUnroutable" + }, + { + "id": "C10", + "breaks": "The vision tool is optional even while user images wait.", + "find": "else if (request.pending_images)\n .required", + "replace": "else if (false)\n .required", + "expect": "RequiredWhenPending" + }, + { + "id": "C11", + "breaks": "The catalog-down notice comes after the answer.", + "find": "return withEffects(next, &.{ .notify_catalog_down, .{ .send = chosen } });", + "replace": "return withEffects(next, &.{ .{ .send = chosen }, .notify_catalog_down });", + "expect": "Bounded" + }, + { + "id": "C12", + "breaks": "A request delivered while a reply is owed is decided at once.", + "find": "next.pending = request;\n return .{ .state = next };", + "replace": "next.pending = request;\n return decide(next, request);", + "expect": "DecideOnlyWhenReady" + }, + { + "id": "C13", + "breaks": "Asking for capabilities does not mark the turn as having asked.", + "find": "next.resolved = true;\n next.pending = request;", + "replace": "next.pending = request;", + "expect": "PendingWaitsForReply" + }, + { + "id": "C14", + "breaks": "The core asks for capabilities for a request without images.", + "find": "if (state.support == .unknown and !state.resolved and (request.user_images or request.tool_images)) {", + "replace": "if (state.support == .unknown and !state.resolved) {", + "expect": "ResolveOnlyWhenNeeded" + }, + { + "id": "C15", + "breaks": "The catalog-down notice appears when the catalog is up.", + "find": "if (chosen.tool_images == .withheld_unknown and request.catalog_down and !state.notified) {", + "replace": "if (chosen.tool_images == .withheld_unknown and !state.notified) {", + "expect": "NotifyOnlyWhenDown" + }, + { + "id": "C16", + "breaks": "Tool images vanish without a notice while support is unknown.", + "find": ".unknown => .withheld_unknown,", + "replace": ".unknown => .none,", + "expect": "NoSilentWithhold" + }, + { + "id": "C17", + "breaks": "The vision tool is offered off the fallback route.", + "find": "const mode: VisionMode = if (route != .fallback)\n .unavailable", + "replace": "const mode: VisionMode = if (route != .fallback)\n .optional", + "expect": "RouteModeCoherent" + }, + { + "id": "C18", + "breaks": "A native image model is routed through the vision tool.", + "find": ".native => .native,", + "replace": ".native => if (request.fallback) .fallback else .native,", + "expect": "NativeExcludesFallback" + } + ] +} diff --git a/tests/tla/image_route/package.json b/tests/tla/image_route/package.json new file mode 100644 index 000000000..a3a94d5fe --- /dev/null +++ b/tests/tla/image_route/package.json @@ -0,0 +1,8 @@ +{ + "name": "fx-tla-image-route", + "private": true, + "type": "module", + "scripts": { + "verify": "bun ../drivers/image_route/verify.ts" + } +} diff --git a/tests/tla/image_route/spec/ImageRoute.tla b/tests/tla/image_route/spec/ImageRoute.tla new file mode 100644 index 000000000..a37e749db --- /dev/null +++ b/tests/tla/image_route/spec/ImageRoute.tla @@ -0,0 +1,256 @@ +----------------------------- MODULE ImageRoute ----------------------------- +(***************************************************************************) +(* How images reach the model on one request: the pure core *) +(* `step(state, event)` in src/core/images/image_route.zig. *) +(* *) +(* Each core event is one action with the same name: TurnStarted, *) +(* RequestReady, and CapabilitiesResolved. The environment acts through *) +(* the values those events carry and through Crash: *) +(* - the catalog can be listed, missing the model, or down (a refused, *) +(* slow, or garbage catalog reads as down), in any order; *) +(* - a reply can be lost (the next turn starts first), duplicated, or *) +(* stale (CapabilitiesResolved with nothing waiting); *) +(* - a request can be delivered twice (RequestReady while a reply is *) +(* owed); *) +(* - the process can crash at any step, which drops all core state. *) +(* A second process is not modeled: core state lives in one turn of one *) +(* process and nothing else can read or write it. *) +(* *) +(* support, resolved, notified, and pending are `abstract(state)` in the *) +(* code. out is the effects the step returned, and act is the event the *) +(* step took, recorded so the laws can name it. *) +(***************************************************************************) +EXTENDS Naturals, Sequences + +Supports == {"unknown", "non_native", "native"} +Routes == {"native", "fallback", "unavailable"} +Modes == {"unavailable", "optional", "required"} +ToolImages == {"none", "attached", "withheld_fallback", "withheld_dead_end", "withheld_unknown"} +Reasons == {"capability_unknown", "no_native_image"} + +Views == [kind: {"listed"}, vision: BOOLEAN, file_input: BOOLEAN] + \cup [kind: {"missing", "down"}, vision: {FALSE}, file_input: {FALSE}] + +Requests == {r \in [user_images: BOOLEAN, pending_images: BOOLEAN, tool_images: BOOLEAN, + fallback: BOOLEAN, catalog_down: BOOLEAN] : + r.pending_images => r.user_images} + +Sends == [kind: {"send"}, route: Routes, mode: Modes, tool_images: ToolImages] +Effects == [kind: {"resolve", "notify"}] \cup Sends \cup [kind: {"fail"}, reason: Reasons] + +\* At most one request; the core's `?Request`. +Pending == {<<>>} \cup {<> : r \in Requests} + +Acts == [name: {"Init", "Crash"}] + \cup [name: {"TurnStarted"}, view: Views] + \cup [name: {"RequestReady"}, req: Requests] + \cup [name: {"CapabilitiesResolved"}, view: Views, answered: Pending] + +VARIABLES support, resolved, notified, pending, out, act + +vars == <> + +\* `supportFromCatalog`. +Derive(v) == IF v.kind = "listed" THEN (IF v.vision THEN "native" ELSE "non_native") ELSE "unknown" + +\* `decision`. +Decide(s, r) == + LET route == CASE s = "native" -> "native" + [] s = "unknown" -> "unavailable" + [] OTHER -> IF r.fallback THEN "fallback" ELSE "unavailable" + mode == IF route # "fallback" THEN "unavailable" + ELSE IF r.pending_images THEN "required" ELSE "optional" + tool == IF ~r.tool_images THEN "none" + ELSE CASE s = "native" -> "attached" + [] s = "unknown" -> "withheld_unknown" + [] OTHER -> IF route = "fallback" THEN "withheld_fallback" ELSE "withheld_dead_end" + IN [kind |-> "send", route |-> route, mode |-> mode, tool_images |-> tool] + +Unroutable(s, r) == r.user_images /\ Decide(s, r).route = "unavailable" +Notifies(s, r, n) == ~Unroutable(s, r) /\ Decide(s, r).tool_images = "withheld_unknown" /\ r.catalog_down /\ ~n + +\* `decide`: the effects of deciding r under s, and the next notified flag. +DecideOut(s, r, n) == + IF Unroutable(s, r) + THEN <<[kind |-> "fail", reason |-> IF s = "unknown" THEN "capability_unknown" ELSE "no_native_image"]>> + ELSE IF Notifies(s, r, n) THEN <<[kind |-> "notify"], Decide(s, r)>> + ELSE <> + +DecideNotified(s, r, n) == n \/ Notifies(s, r, n) + +NeedsResolve(r) == support = "unknown" /\ ~resolved /\ (r.user_images \/ r.tool_images) + +Init == + /\ support = "unknown" + /\ resolved = FALSE + /\ notified = FALSE + /\ pending = <<>> + /\ out = <<>> + /\ act = [name |-> "Init"] + +TurnStarted(v) == + /\ act' = [name |-> "TurnStarted", view |-> v] + /\ support' = Derive(v) + /\ resolved' = FALSE \* a new turn may ask again + /\ notified' = FALSE \* a new turn may tell the user again + /\ pending' = <<>> + /\ out' = <<>> + +RequestReady(r) == + /\ act' = [name |-> "RequestReady", req |-> r] + /\ IF pending # <<>> + THEN pending' = <> /\ out' = <<>> /\ UNCHANGED <> + ELSE IF NeedsResolve(r) + THEN /\ resolved' = TRUE + /\ pending' = <> + /\ out' = <<[kind |-> "resolve"]>> + /\ UNCHANGED <> + ELSE /\ out' = DecideOut(support, r, notified) + /\ notified' = DecideNotified(support, r, notified) + /\ UNCHANGED <> + +CapabilitiesResolved(v) == + /\ act' = [name |-> "CapabilitiesResolved", view |-> v, answered |-> pending] + /\ IF pending = <<>> + THEN out' = <<>> /\ UNCHANGED <> + ELSE LET s == Derive(v) IN + /\ support' = s + /\ pending' = <<>> + /\ out' = DecideOut(s, Head(pending), notified) + /\ notified' = DecideNotified(s, Head(pending), notified) + /\ UNCHANGED resolved + +\* The process dies; the next process starts from the core's zero state. +Crash == + /\ act' = [name |-> "Crash"] + /\ support' = "unknown" + /\ resolved' = FALSE + /\ notified' = FALSE + /\ pending' = <<>> + /\ out' = <<>> + +Reply == pending # <<>> /\ \E v \in Views : CapabilitiesResolved(v) + +Next == + \/ \E v \in Views : TurnStarted(v) + \/ \E r \in Requests : RequestReady(r) + \/ \E v \in Views : CapabilitiesResolved(v) + \/ Crash + +\* A reply the shell waits for arrives unless the turn ends first. +Spec == Init /\ [][Next]_vars /\ WF_vars(Reply) + +----------------------------------------------------------------------------- +(* Laws. DESIGN.md names each one and where it is enforced. *) + +HasKind(k) == \E i \in 1..Len(out) : out[i].kind = k +SendsOut == {out[i] : i \in {j \in 1..Len(out) : out[j].kind = "send"}} +FailsOut == {out[i] : i \in {j \in 1..Len(out) : out[j].kind = "fail"}} +\* The request this step decided, as a sequence of at most one. +Decided == IF act.name = "RequestReady" THEN <> + ELSE IF act.name = "CapabilitiesResolved" THEN act.answered + ELSE <<>> + +TypeOK == + /\ support \in Supports + /\ resolved \in BOOLEAN + /\ notified \in BOOLEAN + /\ pending \in Pending + /\ Len(out) <= 2 /\ \A i \in 1..Len(out) : out[i] \in Effects + /\ act \in Acts + +\* L11: one waiting request, and at most a notice then one answer. +Bounded == + /\ Len(pending) <= 1 + /\ Len(out) <= 2 + /\ Len(out) = 2 => (out[1].kind = "notify" /\ out[2].kind = "send") + +\* L1: a model the catalog lists with vision takes images inline, whatever +\* its file-input tag says; one listed without vision does not; an unlisted +\* model or an unreachable catalog leaves support unknown. +VisionImpliesNative == + (act.name = "TurnStarted" \/ (act.name = "CapabilitiesResolved" /\ act.answered # <<>>)) => + /\ (act.view.kind = "listed" /\ act.view.vision) => support = "native" + /\ (act.view.kind = "listed" /\ ~act.view.vision) => support = "non_native" + /\ act.view.kind # "listed" => support = "unknown" + +\* L5: no image goes to a model the catalog has not confirmed. +UnknownNeverInline == \A e \in SendsOut : e.tool_images = "attached" => support = "native" + +\* L4: route, mode, and tool images all follow the one support value the +\* state holds after the step. +OneSnapshot == + \A e \in SendsOut : + /\ (e.route = "native") <=> (support = "native") + /\ e.tool_images = "withheld_unknown" => support = "unknown" + /\ e.tool_images = "withheld_fallback" => (support = "non_native" /\ e.route = "fallback") + /\ e.tool_images = "withheld_dead_end" => (support = "non_native" /\ e.route = "unavailable") + +\* L2: the vision tool is offered exactly on the fallback route. +RouteModeCoherent == \A e \in SendsOut : (e.route = "fallback") <=> (e.mode # "unavailable") + +\* L3: native delivery and the vision tool never meet. +NativeExcludesFallback == \A e \in SendsOut : support = "native" => (e.route = "native" /\ e.mode = "unavailable") + +\* L6: tool images are attached or withheld with a notice; never neither. +NoSilentWithhold == + \A e \in SendsOut : Decided # <<>> /\ (Head(Decided).tool_images <=> e.tool_images # "none") + +\* L6: the vision tool is required exactly while user images wait. +RequiredWhenPending == + \A e \in SendsOut : e.route = "fallback" => ((e.mode = "required") <=> Head(Decided).pending_images) + +\* L7: user images without a route fail the request; nothing else does. +FailExactlyWhenUnroutable == + /\ \A e \in FailsOut : + /\ Decided # <<>> + /\ Head(Decided).user_images + /\ (support = "unknown" \/ (support = "non_native" /\ ~Head(Decided).fallback)) + /\ e.reason = IF support = "unknown" THEN "capability_unknown" ELSE "no_native_image" + /\ \A e \in SendsOut : Head(Decided).user_images => e.route # "unavailable" + +\* L8: the core asks for capabilities only when images wait on unknown support. +ResolveOnlyWhenNeeded == + HasKind("resolve") => + /\ act.name = "RequestReady" + /\ support = "unknown" + /\ (act.req.user_images \/ act.req.tool_images) + /\ pending = <> + +\* L9: the user hears about a down catalog only for tool images it withholds. +NotifyOnlyWhenDown == + HasKind("notify") => + /\ \E e \in SendsOut : e.tool_images = "withheld_unknown" + /\ Head(Decided).catalog_down + +\* L13: a decision comes only once no reply is owed. +DecideOnlyWhenReady == + (HasKind("send") \/ HasKind("fail")) => + /\ pending = <<>> + /\ (act.name = "RequestReady" \/ (act.name = "CapabilitiesResolved" /\ act.answered # <<>>)) + +\* L14: a new turn, or a new process, starts clean. +TurnResets == + act.name \in {"TurnStarted", "Crash"} => (~resolved /\ ~notified /\ pending = <<>> /\ out = <<>>) + +\* L13: while a reply is owed, the core has asked once and decided nothing. +PendingWaitsForReply == + pending # <<>> => (resolved /\ support = "unknown" /\ (out = <<>> \/ out = <<[kind |-> "resolve"]>>)) + +\* L8: at most one capability request per turn. +ResolveOncePerTurn == [][(\E i \in 1..Len(out') : out'[i].kind = "resolve") => ~resolved]_vars + +\* L9: at most one catalog-down notice per turn. +NotifyOncePerTurn == [][(\E i \in 1..Len(out') : out'[i].kind = "notify") => ~notified]_vars + +\* L10: a request delivered while a reply is owed runs nothing. +DuplicateRequestIsQuiet == [][(act'.name = "RequestReady" /\ pending # <<>>) => out' = <<>>]_vars + +\* L10: a reply nothing waits for changes nothing. +StaleReplyIsQuiet == + [][(act'.name = "CapabilitiesResolved" /\ pending = <<>>) => + (out' = <<>> /\ UNCHANGED <>)]_vars + +\* L12: every request that waits is answered, unless its turn ends. +Answered == (pending # <<>>) ~> (pending = <<>>) +============================================================================= diff --git a/tests/tla/image_route/spec/MCImageRoute.cfg b/tests/tla/image_route/spec/MCImageRoute.cfg new file mode 100644 index 000000000..94bb4f0cb --- /dev/null +++ b/tests/tla/image_route/spec/MCImageRoute.cfg @@ -0,0 +1,27 @@ +\* Every law of ImageRoute.tla over its whole state space. The model has no +\* constants: every set it ranges over is finite by construction, so TLC +\* checks all of it. More specific invariants come first, since TLC names the +\* first one a state breaks. +SPECIFICATION Spec +INVARIANT + TypeOK + Bounded + VisionImpliesNative + UnknownNeverInline + OneSnapshot + RouteModeCoherent + NativeExcludesFallback + NoSilentWithhold + RequiredWhenPending + FailExactlyWhenUnroutable + ResolveOnlyWhenNeeded + NotifyOnlyWhenDown + DecideOnlyWhenReady + TurnResets + PendingWaitsForReply +PROPERTY + ResolveOncePerTurn + NotifyOncePerTurn + DuplicateRequestIsQuiet + StaleReplyIsQuiet + Answered diff --git a/tests/tla/image_route/spec/TraceImageRoute.cfg b/tests/tla/image_route/spec/TraceImageRoute.cfg new file mode 100644 index 000000000..4712d15bd --- /dev/null +++ b/tests/tla/image_route/spec/TraceImageRoute.cfg @@ -0,0 +1,30 @@ +\* Every safety law of ImageRoute, on traces of the real core. Answered is a +\* liveness property and cannot be judged on a finite trace: here +\* ConformsToSpec makes each reply clear the request it answers, and +\* MCImageRoute.cfg proves Answered over the whole model. TraceData.tla is +\* generated next to this file for each run. +SPECIFICATION TraceSpec +CHECK_DEADLOCK FALSE +INVARIANT + StartsAtInit + TypeOK + Bounded + VisionImpliesNative + UnknownNeverInline + OneSnapshot + RouteModeCoherent + NativeExcludesFallback + NoSilentWithhold + RequiredWhenPending + FailExactlyWhenUnroutable + ResolveOnlyWhenNeeded + NotifyOnlyWhenDown + DecideOnlyWhenReady + TurnResets + PendingWaitsForReply +PROPERTY + ResolveOncePerTurn + NotifyOncePerTurn + DuplicateRequestIsQuiet + StaleReplyIsQuiet + ConformsToSpec diff --git a/tests/tla/image_route/spec/TraceImageRoute.tla b/tests/tla/image_route/spec/TraceImageRoute.tla new file mode 100644 index 000000000..b4fd9891a --- /dev/null +++ b/tests/tla/image_route/spec/TraceImageRoute.tla @@ -0,0 +1,76 @@ +--------------------------- MODULE TraceImageRoute --------------------------- +(***************************************************************************) +(* Checks traces the real core produced against ImageRoute. *) +(* *) +(* tests/tla/drivers/image_route/code-to-spec.ts runs event sequences, *) +(* with crashes, duplicates, stale replies, and catalog outages, through *) +(* the harness around src/core/images/image_route.zig and writes each run *) +(* as one sequence of states into TraceData.tla. Each state is what the *) +(* core showed (abstract(state) and the effects of the step) plus the *) +(* event the driver sent. *) +(* *) +(* TLC walks every trace. ConformsToSpec holds when every recorded step is *) +(* a step of ImageRoute's Next, so each trace is a behavior of the model; *) +(* the law names below re-export the model's laws, checked on what the *) +(* code did. *) +(***************************************************************************) +EXTENDS Naturals, Sequences, TraceData + +VARIABLES support, resolved, notified, pending, out, act, trace, at + +IR == INSTANCE ImageRoute + +vars == <> + +Load(s) == + /\ support = s.support + /\ resolved = s.resolved + /\ notified = s.notified + /\ pending = s.pending + /\ out = s.out + /\ act = s.act + +LoadNext(s) == + /\ support' = s.support + /\ resolved' = s.resolved + /\ notified' = s.notified + /\ pending' = s.pending + /\ out' = s.out + /\ act' = s.act + +TraceInit == trace \in 1..Len(Traces) /\ at = 1 /\ Load(Traces[trace][1]) + +TraceNext == + /\ at < Len(Traces[trace]) + /\ at' = at + 1 + /\ trace' = trace + /\ LoadNext(Traces[trace][at + 1]) + +TraceSpec == TraceInit /\ [][TraceNext]_<> + +\* Every trace starts where the model starts. +StartsAtInit == at = 1 => IR!Init + +\* Every recorded step is a step of the model. +ConformsToSpec == [][IR!Next]_vars + +TypeOK == IR!TypeOK +Bounded == IR!Bounded +VisionImpliesNative == IR!VisionImpliesNative +UnknownNeverInline == IR!UnknownNeverInline +OneSnapshot == IR!OneSnapshot +RouteModeCoherent == IR!RouteModeCoherent +NativeExcludesFallback == IR!NativeExcludesFallback +NoSilentWithhold == IR!NoSilentWithhold +RequiredWhenPending == IR!RequiredWhenPending +FailExactlyWhenUnroutable == IR!FailExactlyWhenUnroutable +ResolveOnlyWhenNeeded == IR!ResolveOnlyWhenNeeded +NotifyOnlyWhenDown == IR!NotifyOnlyWhenDown +DecideOnlyWhenReady == IR!DecideOnlyWhenReady +TurnResets == IR!TurnResets +PendingWaitsForReply == IR!PendingWaitsForReply +ResolveOncePerTurn == IR!ResolveOncePerTurn +NotifyOncePerTurn == IR!NotifyOncePerTurn +DuplicateRequestIsQuiet == IR!DuplicateRequestIsQuiet +StaleReplyIsQuiet == IR!StaleReplyIsQuiet +============================================================================= diff --git a/tests/tla/session_manager/PromptImages.cfg b/tests/tla/session_manager/PromptImages.cfg new file mode 100644 index 000000000..3cda0de6e --- /dev/null +++ b/tests/tla/session_manager/PromptImages.cfg @@ -0,0 +1,15 @@ +\* Two images, so one can be saved while the other fails, and either can be +\* attached again in a later turn of the source or the fork. Two items per +\* session and three requests reach every planted bug in mutants.py. +CONSTANTS + Digests = {d1, d2} + MaxRequests = 3 + MaxItems = 2 +SPECIFICATION Spec +SYMMETRY Symmetry +INVARIANTS + TypeOK + NoteNamesBlob + RefHasBlob + ItemKeepsImages + RequestSendsAll diff --git a/tests/tla/session_manager/PromptImages.tla b/tests/tla/session_manager/PromptImages.tla new file mode 100644 index 000000000..82c996554 --- /dev/null +++ b/tests/tla/session_manager/PromptImages.tla @@ -0,0 +1,190 @@ +---------------------------- MODULE PromptImages ---------------------------- +(* A v2 session's prompt images as blobs of the session (D89, amending D44). + + A turn's images are saved as blobs before its first request is built + and before its user item is written. The request sends every image's + bytes, and its note names the saved file of each image that has one. The + user item names each saved image by its digest and lists the blobs the + turn stored, so a fork, which hard-links only listed blobs, can read + them. An image whose save fails stays inside the item and gets no note. + A conversion imports v1's turns the same way, but with no request. + + An image is its digest: attaching the same image twice attaches one + digest, and its blob is stored once. Blobs are never deleted, so a note + that was true when its request was sent stays true. + + Checked by tests/tla/session_manager/check.sh; mutants.py plants the bugs + each invariant exists to catch, and + tests/tla/drivers/session_manager/prompt-images.test.ts has TLC check + real runs against this spec through trace/PromptImagesTrace.tla. *) +EXTENDS Naturals, Sequences, FiniteSets, TLC + +CONSTANTS Digests, MaxRequests, MaxItems + +\* Images are interchangeable: every property holds for each alike. +Symmetry == Permutations(Digests) + +Sessions == {"source", "fork"} +Forms == {"ref", "inline"} +\* `asked` is what the user attached; `images` how the item keeps each one. +Item == [asked : SUBSET Digests, images : [Digests -> Forms \cup {"none"}], + listed : SUBSET Digests] + +VARIABLES + up, \* the fx process is running + forked, \* the fork exists + blobs, \* [Sessions -> SUBSET Digests]: files in each blobs folder + log, \* [Sessions -> Seq(Item)]: each session's user items + pending, \* [Sessions -> SUBSET Digests]: stored, not yet listed (memory) + turn, \* the turn being prepared, or none + requests \* Seq([s, asked, sent, noted]): what each request carried + +vars == <> + +\* `requested` and `written` say whether the turn's first request was built +\* and its user item written. The code writes the item first, before the +\* request goes out; the spec allows either order. +NoTurn == [phase |-> "none", s |-> "source", asked |-> {}, + form |-> [d \in Digests |-> "none"], + requested |-> FALSE, written |-> FALSE] + +TypeOK == + /\ up \in BOOLEAN /\ forked \in BOOLEAN + /\ blobs \in [Sessions -> SUBSET Digests] + /\ \A s \in Sessions : log[s] \in Seq(Item) + /\ pending \in [Sessions -> SUBSET Digests] + /\ turn.phase \in {"none", "open"} + /\ turn.s \in Sessions /\ turn.asked \subseteq Digests + /\ turn.form \in [Digests -> Forms \cup {"none", "unsaved"}] + /\ turn.requested \in BOOLEAN /\ turn.written \in BOOLEAN + +Init == + /\ up = TRUE /\ forked = FALSE + /\ blobs = [s \in Sessions |-> {}] + /\ log = [s \in Sessions |-> <<>>] + /\ pending = [s \in Sessions |-> {}] + /\ turn = NoTurn + /\ requests = <<>> + +Open(s) == s = "source" \/ forked + +\* Images an earlier turn of `s` saved, by how its item keeps them. +Earlier(s, f) == {d \in Digests : \E k \in 1..Len(log[s]) : log[s][k].images[d] = f} + +\* A prompt with images starts its turn. +Begin(s, asked) == + /\ up /\ Open(s) /\ turn.phase = "none" /\ asked # {} + /\ Len(log[s]) < MaxItems /\ Len(requests) < MaxRequests + /\ turn' = [phase |-> "open", s |-> s, asked |-> asked, + form |-> [d \in Digests |-> IF d \in asked THEN "unsaved" ELSE "none"], + requested |-> FALSE, written |-> FALSE] + /\ UNCHANGED <> + +\* A conversion imports each v1 turn before the session's first request and +\* before any fork. v1 sent the turn's request long ago, so the turn saves +\* its images, or keeps them inline, and writes its item with no request. +Import(asked) == + /\ up /\ ~forked /\ requests = <<>> /\ turn.phase = "none" /\ asked # {} + /\ Len(log["source"]) < MaxItems + /\ turn' = [phase |-> "open", s |-> "source", asked |-> asked, + form |-> [d \in Digests |-> IF d \in asked THEN "unsaved" ELSE "none"], + requested |-> TRUE, written |-> FALSE] + /\ UNCHANGED <> + +\* Every image is saved as a blob or kept inline. +Saved == \A d \in turn.asked : turn.form[d] # "unsaved" + +\* The image's bytes become a blob named by their digest; the same bytes +\* stored again cost one stat. The blob waits to be listed. +Save(d) == + /\ up /\ turn.phase = "open" /\ turn.form[d] = "unsaved" + /\ blobs' = [blobs EXCEPT ![turn.s] = @ \cup {d}] + /\ pending' = [pending EXCEPT ![turn.s] = @ \cup {d}] + /\ turn' = [turn EXCEPT !.form[d] = "ref"] + /\ UNCHANGED <> + +\* The store refuses (disk full, permissions): the image stays inline. +SaveFails(d) == + /\ up /\ turn.phase = "open" /\ turn.form[d] = "unsaved" + /\ turn' = [turn EXCEPT !.form[d] = "inline"] + /\ UNCHANGED <> + +\* The first request is built once every image is saved or kept inline. It +\* sends every image of the session, and notes those with a blob. +Request == + /\ up /\ turn.phase = "open" /\ ~turn.requested + /\ Saved + /\ requests' = Append(requests, + [s |-> turn.s, + asked |-> turn.asked, + sent |-> {d \in turn.asked : turn.form[d] \in Forms} + \cup Earlier(turn.s, "ref") \cup Earlier(turn.s, "inline"), + noted |-> {d \in turn.asked : turn.form[d] = "ref"} \cup Earlier(turn.s, "ref")]) + /\ turn' = [turn EXCEPT !.requested = TRUE] + /\ UNCHANGED <> + +\* The user item: each image by the form its save gave it, listing every +\* blob the turn stored. +Write == + /\ up /\ turn.phase = "open" /\ ~turn.written + /\ Saved + /\ log' = [log EXCEPT ![turn.s] = Append(@, + [asked |-> turn.asked, + images |-> [d \in Digests |-> IF d \in turn.asked THEN turn.form[d] ELSE "none"], + listed |-> pending[turn.s]])] + /\ pending' = [pending EXCEPT ![turn.s] = {}] + /\ turn' = [turn EXCEPT !.written = TRUE] + /\ UNCHANGED <> + +\* The turn ends once its request was sent and its item written. +Finish == + /\ up /\ turn.phase = "open" /\ turn.requested /\ turn.written + /\ turn' = NoTurn + /\ UNCHANGED <> + +\* A crash loses memory: the turn and the unlisted blobs' names. The blob +\* files stay, unreferenced. +Crash == + /\ up /\ up' = FALSE + /\ turn' = NoTurn /\ pending' = [s \in Sessions |-> {}] + /\ UNCHANGED <> + +Restart == /\ ~up /\ up' = TRUE /\ UNCHANGED <> + +\* A fork between turns copies the source's items and hard-links the blobs +\* they list (D44). +Fork == + /\ up /\ ~forked /\ turn.phase = "none" /\ Len(log["source"]) > 0 + /\ forked' = TRUE + /\ log' = [log EXCEPT !["fork"] = log["source"]] + /\ blobs' = [blobs EXCEPT !["fork"] = + UNION {log["source"][k].listed : k \in 1..Len(log["source"])}] + /\ UNCHANGED <> + +Next == + \/ \E s \in Sessions, a \in SUBSET Digests : Begin(s, a) + \/ \E a \in SUBSET Digests : Import(a) + \/ \E d \in Digests : Save(d) \/ SaveFails(d) + \/ Request \/ Write \/ Finish \/ Crash \/ Restart \/ Fork + +Spec == Init /\ [][Next]_vars + +\* Every note names a file that holds that image's bytes in the request's +\* own session. Blobs are never removed, so it stays true after the send. +NoteNamesBlob == \A k \in 1..Len(requests) : requests[k].noted \subseteq blobs[requests[k].s] + +\* Every image an item names by digest has its blob in that session, +\* including a fork's copy of the item. +RefHasBlob == + \A s \in Sessions : \A k \in 1..Len(log[s]) : \A d \in Digests : + log[s][k].images[d] = "ref" => d \in blobs[s] + +\* An item keeps every image the user attached, as a blob or inline. +ItemKeepsImages == + \A s \in Sessions : \A k \in 1..Len(log[s]) : + {d \in Digests : log[s][k].images[d] \in Forms} = log[s][k].asked + +\* Every request sends every image the user attached to its turn. +RequestSendsAll == \A k \in 1..Len(requests) : requests[k].asked \subseteq requests[k].sent + +============================================================================= diff --git a/tests/tla/session_manager/check.sh b/tests/tla/session_manager/check.sh index 975caa8ec..97e239cf0 100755 --- a/tests/tla/session_manager/check.sh +++ b/tests/tla/session_manager/check.sh @@ -37,7 +37,7 @@ if [ -z "$java" ]; then exit 2 fi -all="SessionLog TurnLifecycle WriterLock ResumeSnapshot Fork Subagents Catalog Lifecycle Wiring MoveSideFiles V1Conversion" +all="SessionLog TurnLifecycle WriterLock ResumeSnapshot Fork Subagents Catalog Lifecycle Wiring MoveSideFiles V1Conversion PromptImages" specs="${*:-$all}" # A new private folder per run for TLC state, so nothing planted in the temp # folder can redirect it. TLC prints to the terminal, so it is removed. diff --git a/tests/tla/session_manager/mutants.py b/tests/tla/session_manager/mutants.py index 82392233e..df38cd618 100755 --- a/tests/tla/session_manager/mutants.py +++ b/tests/tla/session_manager/mutants.py @@ -137,6 +137,52 @@ MUTANTS = [ ("Lifecycle", "ack before fsync of the directory", 'Advance(s, "renamed", "done")', 'Advance(s, "synced", "done")', "AckedIsVisible"), + ("PromptImages", "the note names an image the store refused", + 'noted |-> {d \\in turn.asked : turn.form[d] = "ref"} \\cup Earlier(turn.s, "ref")', + 'noted |-> turn.asked \\cup Earlier(turn.s, "ref")', + "NoteNamesBlob"), + ("PromptImages", "the request is built before its images are saved", + ' /\\ Saved\n' + ' /\\ requests\' = Append(requests,\n' + ' [s |-> turn.s,\n' + ' asked |-> turn.asked,\n' + ' sent |-> {d \\in turn.asked : turn.form[d] \\in Forms}\n' + ' \\cup Earlier(turn.s, "ref") \\cup Earlier(turn.s, "inline"),\n' + ' noted |-> {d \\in turn.asked : turn.form[d] = "ref"} \\cup Earlier(turn.s, "ref")])', + ' /\\ requests\' = Append(requests,\n' + ' [s |-> turn.s,\n' + ' asked |-> turn.asked,\n' + ' sent |-> turn.asked \\cup Earlier(turn.s, "ref") \\cup Earlier(turn.s, "inline"),\n' + ' noted |-> {d \\in turn.asked : turn.form[d] \\in {"ref", "unsaved"}} \\cup Earlier(turn.s, "ref")])', + "NoteNamesBlob"), + ("PromptImages", "the item names blobs it does not list", + 'listed |-> pending[turn.s]])]', 'listed |-> {}])]', + "RefHasBlob"), + ("PromptImages", "bytes already stored are not listed again", + " /\\ pending' = [pending EXCEPT ![turn.s] = @ \\cup {d}]", + " /\\ pending' = [pending EXCEPT ![turn.s] = IF d \\in blobs[turn.s] THEN @ ELSE @ \\cup {d}]", + "RefHasBlob"), + ("PromptImages", "a fork hard-links no blobs", + ' /\\ blobs\' = [blobs EXCEPT !["fork"] =\n' + ' UNION {log["source"][k].listed : k \\in 1..Len(log["source"])}]', + ' /\\ blobs\' = [blobs EXCEPT !["fork"] = {}]', + "RefHasBlob"), + ("PromptImages", "an import names an image it never saved", + 'form |-> [d \\in Digests |-> IF d \\in asked THEN "unsaved" ELSE "none"],\n' + ' requested |-> TRUE', + 'form |-> [d \\in Digests |-> IF d \\in asked THEN "ref" ELSE "none"],\n' + ' requested |-> TRUE', + "RefHasBlob"), + # A failed save losing the image, once per place that keeps it, so each + # mutant fails only its own invariant. + ("PromptImages", "a request leaves out an image kept inline", + 'sent |-> {d \\in turn.asked : turn.form[d] \\in Forms}', + 'sent |-> {d \\in turn.asked : turn.form[d] = "ref"}', + "RequestSendsAll"), + ("PromptImages", "an item leaves out an image kept inline", + 'images |-> [d \\in Digests |-> IF d \\in turn.asked THEN turn.form[d] ELSE "none"]', + 'images |-> [d \\in Digests |-> IF d \\in turn.asked /\\ turn.form[d] = "ref" THEN "ref" ELSE "none"]', + "ItemKeepsImages"), ] @@ -162,8 +208,12 @@ def main(): print("needs a Java runtime and tla2tools.jar (see check.sh)", file=sys.stderr) return 2 survivors = 0 + # Spec names on the command line run only their mutants. + wanted = set(sys.argv[1:]) with tempfile.TemporaryDirectory(prefix="session-manager-mutants-") as tmp: for index, (spec, desc, old, new, expected) in enumerate(MUTANTS): + if wanted and spec not in wanted: + continue work = pathlib.Path(tmp) / f"{index:02d}-{spec}" work.mkdir() text = (HERE / f"{spec}.tla").read_text() diff --git a/tests/tla/session_manager/system-design.md b/tests/tla/session_manager/system-design.md index 24d7bf1ec..74e1d99b1 100644 --- a/tests/tla/session_manager/system-design.md +++ b/tests/tla/session_manager/system-design.md @@ -1002,7 +1002,8 @@ Settled while moving off the side folder (2026-10-01): session is open. Prompt images are captured before the session is on disk, where no blob can be written, so they are saved inside the user's item, as libfx saves them; a large item is a blob - already, and D79 also keeps each image as a blob. fx's own size limits stay where they are. **Why:** one folder per session, + already. D89 saves them as blobs when their turn starts. fx's own size + limits stay where they are. **Why:** one folder per session, forks that hard-link instead of copying, no orphan side folders, and content addressing that makes a stored body unforgeable. **Alternative:** keep today's handle names with an index from name to blob, which grows @@ -1393,24 +1394,16 @@ Settled while removing v1 (2026-10-17): `InvalidSessionFormat`, the file and the reason the converter's check gives, and points at `fx doctor`. **Why:** the user can act on it. **Alternative:** a generic message that the session is corrupt. -- **D79 Decision (amends D44):** prompt images are also kept as blobs. - When a turn's user item is written, by a stream, a commit or a - conversion's import, each inline prompt image is put as a blob of the - session and listed on that item, and the item saves its - `snapshot_path` as that blob's path; the bytes stay inside the item. - Each request names that blob for every prompt image it sends, as +- **D79 Decision (amends D44; D89 amends it):** each request names the + saved file of every prompt image it sends, as `[Image #N (type) is saved at "".]` ahead of the prompt, and the notice for an image too large to send, or left out to keep a request within its size limit, names the same file, in the running process and - after a resume alike: the blob is the file named by - the image's hash in the session's blob folder, and the turn writes it - with the user item before any of its tools runs, so the model can open - the file it is told about. Without a saved session the notice names the - copy fx captured in its temporary folder, which lasts while fx runs, and - the note is left out. A recalled image has the same bytes, so it names - the same blob. **Why:** - the model can read the original to make a smaller copy, or open the - image it sees with tools, as v1's `images/` folder let it. + after a resume alike. That file is the image's blob (D89). An image + with no saved file, one the store did not save or one in a session + that is not saved, is named by no note, and its notices name no file. + **Why:** the model can read the original to make a smaller copy, or + open the image it sees with tools, as v1's `images/` folder let it. **Alternative:** tell the model no local file is available. - **D80 Decision:** when fx is killed while a response is being retried, the reopened session shows the turn's finished pieces, such as a tool @@ -1449,7 +1442,7 @@ Settled while removing v1 (2026-10-17): image bytes saved with the prompt, checked against their digest, so a damaged or deleted captured file no longer refuses the recall. The recalled image keeps those bytes inline and writes no file, so its turn - saves it as a blob as it does any prompt image (D79); a copy in the + saves it as a blob as it does any prompt image (D89); a copy in the temporary folder would be gone once fx exits. The composer tests pin that the recall succeeds and names the same blob. **Why:** the saved bytes are the prompt's image; the captured file is a temporary copy. @@ -1490,6 +1483,52 @@ Settled while removing v1 (2026-10-17): **Alternative:** convert descendants at every depth, for sessions no build opens. +Settled for prompt images (2026-10-19, after a model searched the disk +for an image pasted into its prompt): +- **D89 Decision (amends D44 and D79):** each prompt image becomes its own + blob, named by its digest, when its turn starts and before the turn's + first request is built (`AgentRuntimeDeps.save_prompt_images`). D44's + reason for keeping prompt images inside the user's item holds when an + image is captured, before the session is on disk, but not when its turn + starts. + - The `user` item names the image by `blob`, without its base64 bytes or + the temporary snapshot path of the process that captured it. The blob + is listed on the turn's next item, so a fork hard-links it and + `verify` checks it. + - A conversion's import saves each converted prompt image the same way, + and a recalled image keeps its bytes (D84), so its turn saves it too. + - In memory a saved image is only its handle too: the blob's path and + its digest. Whatever needs the bytes, a request, Vision or a recall, + reads them from the blob and checks them against the digest, as v1 + read an image's snapshot file. A resumed session loads no image bytes, + and the turn's own copy drops them once the blob is saved. The + request still sends the bytes inline. Its note and every notice for an + image left out of it name the blob's path (D79), so the model can open + the image with its tools. + - Every save lists the blob again, so the next turn that sends an image + repairs a listing a crash lost. The session remembers the digests it + saved, so every copy of the turn a host commits encodes the same way + and the commit matches what was streamed. + - An image the store does not save keeps its bytes inline, as before, + and no note or notice names a file for it. A blob an item names that + is gone damages the session when it opens (D39). One whose bytes were + damaged is found by `verify` and by any reader, which leaves the image + out as it does a damaged snapshot. Items written before D89 read as + they did. + - `PromptImages.tla` checks that a note names only a saved, listed blob, + that every request sends every image, and that a fork holds every blob + its items name, for a conversion's import as for a turn. `mutants.py` + plants eight bugs against it. + + **Why:** D79 kept each image twice, as its blob and as base64 inside the + item, a third larger than its bytes, and wrote both only when the user + item was written, after the first request named the blob. Keeping the + bytes in memory too would hold every image of a long session for as long + as it is open, and copy them again for each turn's Vision catalog. + **Alternative:** save at capture, when no session is on disk (D44); + keep D79's two copies; load every blob when the session opens; or name + the temporary snapshot, which is gone when its process exits. + ## Design Risks And Failure Behavior - **Stale index.** A crash between publish and the index append hides a diff --git a/tests/tla/session_manager/trace/PromptImagesTrace.cfg b/tests/tla/session_manager/trace/PromptImagesTrace.cfg new file mode 100644 index 000000000..b79b2c4c8 --- /dev/null +++ b/tests/tla/session_manager/trace/PromptImagesTrace.cfg @@ -0,0 +1,10 @@ +\* A run's images are labeled d1, d2, d3 in the order the driver first sees +\* them; the bounds leave room for the driver's turns. +CONSTANTS + Digests = {"d1", "d2", "d3"} + MaxRequests = 6 + MaxItems = 6 +INIT TraceInit +NEXT TraceNext +CONSTRAINT Progress +INVARIANT TraceIncomplete diff --git a/tests/tla/session_manager/trace/PromptImagesTrace.tla b/tests/tla/session_manager/trace/PromptImagesTrace.tla new file mode 100644 index 000000000..b1e6f4ccd --- /dev/null +++ b/tests/tla/session_manager/trace/PromptImagesTrace.tla @@ -0,0 +1,67 @@ +------------------------ MODULE PromptImagesTrace ------------------------ +(***************************************************************************) +(* Checks a real fx run against PromptImages.tla (D89). *) +(* *) +(* tests/tla/drivers/session_manager/prompt-images.test.ts runs the built *) +(* binary and writes one line per thing it observed, each a step of the *) +(* spec, with images named by labels for their digests: *) +(* *) +(* Request s, asked, sent, noted: a turn's first request, from the *) +(* gateway. `noted` holds an image only when its note names *) +(* that image's blob in the request's own session; a note *) +(* naming anything else gets a label outside Digests. *) +(* Write s, refs, inline, listed: a turn's user item, from the log, *) +(* including one a conversion imported. *) +(* Blobs s, blobs: the files in a session's blobs folder. *) +(* *) +(* What nothing observes (each turn's or import's start, each save or *) +(* failed save, each process exit) is left to TLC between lines; the *) +(* driver makes no fork, so none happens. The run is accepted when some *) +(* behavior of the spec makes every observation in order: the invariant *) +(* TraceIncomplete fails exactly then. *) +(***************************************************************************) +EXTENDS PromptImages, TraceData, TLC + +VARIABLE l + +traceVars == <> + +ASSUME TLCSet(1, 0) + +Hidden == + \/ \E s \in Sessions, a \in SUBSET Digests : Begin(s, a) + \/ \E a \in SUBSET Digests : Import(a) + \/ \E d \in Digests : Save(d) \/ SaveFails(d) + \/ Finish \/ Crash \/ Restart + +Form(e, d) == IF d \in e.refs THEN "ref" ELSE IF d \in e.inline THEN "inline" ELSE "none" + +Observed(e) == + \/ /\ e.event = "Request" + /\ Request + /\ requests'[Len(requests')] = [s |-> e.s, asked |-> e.asked, sent |-> e.sent, noted |-> e.noted] + \/ /\ e.event = "Write" + /\ Write + /\ log'[e.s][Len(log'[e.s])] = + [asked |-> e.refs \cup e.inline, + images |-> [d \in Digests |-> Form(e, d)], + listed |-> e.listed] + \/ /\ e.event = "Blobs" + /\ blobs[e.s] = e.blobs + /\ UNCHANGED vars + +TraceInit == Init /\ l = 1 + +TraceNext == + /\ l <= Len(Trace) + /\ \/ Hidden /\ UNCHANGED l + \/ Observed(Trace[l]) /\ l' = l + 1 + +\* Prints the furthest line matched, which names the rejected one. +Progress == + IF l > TLCGet(1) + THEN TLCSet(1, l) /\ PrintT(<<"matched_lines", l - 1>>) + ELSE TRUE + +TraceIncomplete == l <= Len(Trace) +============================================================================= diff --git a/tests/tla/session_manager/trace/README.md b/tests/tla/session_manager/trace/README.md index 392bbf25c..ce60e74c6 100644 --- a/tests/tla/session_manager/trace/README.md +++ b/tests/tla/session_manager/trace/README.md @@ -12,8 +12,11 @@ we execute: unit tests, the standalone driver, and the `kill -9` and machine-crash matrix. In fx, the manager's trace hook is compiled out: `build.zig` sets the -`hooks` option to false, and the standalone driver and crash matrix are not -part of this repository. What runs here is `selftest.sh`, each checker's +`hooks` option to false for every artifact it ships. `zig build test` also +runs the manager's own tests with `hooks` on, and they write their traces to +`zig-out/session-manager-traces/`, but nothing checks those traces against +the specs yet. The standalone driver and crash matrix are not part of this +repository. What runs here is `selftest.sh`, each checker's `--selftest` mode, `check_saved_logs.py` on logs fx saved, and `check_wiring_traces.py` on traces from a run with `FX_TRACE_SCOPES=wiring`. @@ -56,6 +59,7 @@ emits one line "log_len": 2, ...} (Trace == <<...>>) EXTENDS | `WiringTrace.tla` | Its wrapper. Accepted when every step is allowed in turn; the usage steps change nothing a traced step reads, so they are left out | | `check_conversion.py --request R --report D` | Checks a recorded v1-to-v2 conversion run against `V1Conversion.tla` (D59): each step's result on disk and through fx's own readers and listing. Exit 0 ACCEPTED only when TLC matched every observation, 1 REJECTED, 2 INCONCLUSIVE, 3 TOOL_ERROR. `--selftest` requires its good and planted runs to get their verdicts | | `V1ConversionTrace.tla` | Its wrapper. Each observation's step kind limits the model actions that may run before it; a request step makes exactly one request, and only a writer an earlier hold step left running may write during it | +| `PromptImagesTrace.tla`, `.cfg` | Checks a run of `drivers/session_manager/prompt-images.test.ts` against `PromptImages.tla` (D89): each turn's first request, its user item, and the session's blobs folder, with every save, failed save and process exit between them left to TLC. A note counts only when it names the image's own blob | | `selftest.sh` | Proves the tooling both accepts and rejects (below) | ## Self-test