mirror of
https://github.com/vercel-labs/fx.git
synced 2026-10-28 10:15:17 +08:00
Pranit
7e2f67d607
The E2E files, TLA+ drivers and the long-turn memory benchmark that read v1's session files now read the v2 log. Fixes for what those tests found: - Resuming a subagent child directly fails with v1's message. - fx session recover copies a v1 conversation whose permissions.json is malformed or empty, without its rules, and says so. - fx doctor counts the v1 sessions not converted yet and names each one it cannot convert. - Salvage leaves out a side file that is missing or whose size or digest changed. - A v1 log whose checkpoint splits a tool call from its result still converts. - --resume last and fx session last fail with NoReadableSessions when they skipped an unreadable v1 session. - A converted session keeps a last turn that never ended, marked interrupted. - fx session --id on a damaged v1 session names the file and the reason. - Prompt images are also kept as blobs, so the oversized-image notice names a file that still exists after a resume. - A session reopened after fx was killed mid-retry shows the finished tool calls and results, not the partial reply. - An update relaunch from a session with nothing saved opens a fresh session. - Converting a schema-2 session captures images it saved only by path, as v1's resume did. - Recalling a prompt from history uses its saved image bytes, so a damaged captured file no longer refuses the recall. - fx -c says the remembered session could not be read when v1's bookmark is unreadable. - The /resume picker orders v2 and unconverted v1 sessions newest first, like fx sessions. - Wiring.tla models one store, without the backend switch or the usage ledger.
721 lines
37 KiB
TypeScript
721 lines
37 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { execFileSync } from "node:child_process";
|
|
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import { onlySessionId, savedText, sessionDir } from "./saved-sessions";
|
|
|
|
// The shared tmux helper imports eval helpers; do not load repository dotenv files.
|
|
process.env.FX_E2E_DISABLE_DOTENV = "1";
|
|
const {
|
|
FAKE_GATEWAY_MODEL, TmuxSession, fakeGatewayFinalText, fakeGatewayToolCall,
|
|
fakeShellRun, heldFakeGatewayFinalText, startDynamicFakeGateway,
|
|
hasEmptyComposer, tmuxAvailable,
|
|
} = await import("./tmux-helpers");
|
|
|
|
const binary = resolve(import.meta.dir, "../../zig-out/bin/fx");
|
|
const HEAD = "HISTORY_HEAD_29b7";
|
|
const TAIL = "HISTORY_TAIL_16d3";
|
|
// A fact entry, which the compacted conversation keeps whatever the turn
|
|
// numbers.
|
|
const HANDOFF = `Facts:\nF1: INTERNAL_HANDOFF_4e12: preserve ${HEAD} and ${TAIL}; follow the latest user request.`;
|
|
const FOLLOWUP = "FOLLOWUP_OK_732c";
|
|
const REOPEN = "REOPEN_OK_492a";
|
|
const ACTIVITY = /Compacting \((?:\d+h)?(?:\d+m)?\d+s\)/;
|
|
const COMPACTION_OUTPUT = /Compacting|compaction|Context compacted|No context to compact|Synthetic summary rejection|INTERNAL_HANDOFF_4e12/i;
|
|
|
|
type Trigger = "manual" | "auto" | "overflow" | "ordinary";
|
|
type Outcome = "success" | "cancel" | "empty" | "provider-error";
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
}
|
|
|
|
/// A session log's conversation: what a close or a reopen leaves as it was,
|
|
/// since the store adds only its own `closed` and `snapshot` lines then.
|
|
function conversationLines(text: string): string[] {
|
|
return text.split("\n").filter(Boolean).filter((line) => !["closed", "snapshot"].includes(JSON.parse(line).kind));
|
|
}
|
|
|
|
/// The compactions a session log holds.
|
|
function checkpoints(bytes: Buffer): number {
|
|
return bytes.toString().trim().split("\n").filter(Boolean)
|
|
.map((line) => JSON.parse(line)).filter((line) => line.kind === "compacted").length;
|
|
}
|
|
|
|
async function until(predicate: () => boolean, label: string, timeout = 20_000) {
|
|
const deadline = Date.now() + timeout;
|
|
while (!predicate()) {
|
|
if (Date.now() >= deadline) throw new Error(`timed out waiting for ${label}`);
|
|
await Bun.sleep(20);
|
|
}
|
|
}
|
|
|
|
async function fixture(trigger: Trigger, outcome: Outcome = "success", longResume = false) {
|
|
// Ctrl+O includes the recording path; keep it free of forbidden notice words.
|
|
const root = mkdtempSync(join(tmpdir(), "fx-activity-"));
|
|
const home = join(root, "home");
|
|
const workspace = join(root, "workspace");
|
|
mkdirSync(join(home, ".fx"), { recursive: true });
|
|
mkdirSync(workspace);
|
|
writeFileSync(join(workspace, "seed-notes.txt"), "seed notes\n");
|
|
writeFileSync(join(home, ".fx/settings.json"), JSON.stringify({
|
|
model: FAKE_GATEWAY_MODEL, auto_upgrade: false, startup_scrollback: false,
|
|
}));
|
|
// Failure attempts need an older exchange outside the retained suffix, but
|
|
// must leave the next ordinary request below the automatic pressure threshold.
|
|
const seedTurns = trigger === "manual" && outcome !== "success" ? 3 : 1;
|
|
const lines = seedTurns > 1 ? 400 : trigger === "ordinary" ? 2 : trigger === "auto" || longResume ? 18_000 : 1600;
|
|
const seedReply = (turn: number) => `${turn === 1 ? HEAD : `HISTORY_MIDDLE_${turn}`}\n${"history alpha beta gamma delta sample line\n".repeat(lines)}SEED_DONE_${turn}${turn === seedTurns ? `\n${TAIL}` : ""}`;
|
|
const summaryHold = heldFakeGatewayFinalText();
|
|
const ordinaryHold = heldFakeGatewayFinalText();
|
|
let phase: "seed" | "attempt" | "followup" | "reopen" = "seed";
|
|
let summaries = 0;
|
|
let ordinary = 0;
|
|
let overflowSent = false;
|
|
let terminal: InstanceType<typeof TmuxSession> | undefined;
|
|
const requests: object[] = [];
|
|
const gateway = startDynamicFakeGateway((raw) => {
|
|
const request = JSON.parse(raw);
|
|
const summary = request.toolChoice?.type === "none" && request.tools?.length === 0;
|
|
requests.push({ phase, summary, at: Date.now(), bytes: raw.length,
|
|
head: raw.includes(HEAD), tail: raw.includes(TAIL), handoff: raw.includes("INTERNAL_HANDOFF_4e12"),
|
|
followup: raw.includes("Follow the latest request.") || raw.includes("Recover with another request."),
|
|
reopen: raw.includes("Continue after reopening.") || raw.includes("Check the reopened context."),
|
|
});
|
|
if (summary) {
|
|
summaries++;
|
|
if (phase === "attempt" && outcome === "provider-error") {
|
|
return Response.json({ error: { message: "Synthetic summary rejection for Bearer abcdef0123456789xyz" } }, { status: 400 });
|
|
}
|
|
if (phase === "attempt" && summaries === 1) return summaryHold.response;
|
|
// Each chunk/retry needs a fresh Response, not a consumed held body.
|
|
return fakeGatewayFinalText(phase === "attempt" && outcome === "empty" ? "" : HANDOFF);
|
|
}
|
|
// Each seed turn reads a note before replying, so compaction has work to
|
|
// summarize. That extra request is not counted as ordinary.
|
|
const seedRead = `seed-read-${ordinary + 1}`;
|
|
if (phase === "seed" && !raw.includes(`"${seedRead}"`)) return fakeGatewayToolCall(seedRead, "read_file", { path: "seed-notes.txt" });
|
|
ordinary++;
|
|
if (phase === "seed") return fakeGatewayFinalText(seedReply(ordinary));
|
|
if (phase === "attempt") {
|
|
if (trigger === "overflow" && !overflowSent) {
|
|
overflowSent = true;
|
|
return Response.json({ error: {
|
|
type: "invalid_request_error", code: "context_length_exceeded",
|
|
message: "This model's maximum context length is 128000 tokens. The request exceeds the context window.",
|
|
} }, { status: 400 });
|
|
}
|
|
return ordinaryHold.response;
|
|
}
|
|
return fakeGatewayFinalText(phase === "reopen" ? REOPEN : FOLLOWUP);
|
|
}, { models: [{ id: FAKE_GATEWAY_MODEL, type: "language", tags: ["tool-use"], context_window: 128000, max_tokens: 8192 }] });
|
|
const env = {
|
|
PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: home, TMPDIR: root,
|
|
TERM: "xterm-256color", AI_GATEWAY_API_KEY: "synthetic-compaction-key",
|
|
FX_DISABLE_KEYCHAIN: "1", FX_E2E_DISABLE_DOTENV: "1", FX_SKIP_ONBOARDING: "1",
|
|
FX_SOUND: "0", FX_AUTO_UPGRADE: "0", FX_MODEL: FAKE_GATEWAY_MODEL,
|
|
FX_GATEWAY_BASE_URL: gateway.baseUrl, FX_GATEWAY_CHAT_URL: gateway.chatUrl,
|
|
FX_E2E_GATEWAY_CHAT_URL: gateway.chatUrl,
|
|
FX_E2E_GATEWAY_MODELS_URL: `${gateway.baseUrl}/coding-agent/v1/models`,
|
|
};
|
|
let sessionId = "";
|
|
let eventsPath = "";
|
|
let initial = Buffer.alloc(0);
|
|
let launchIndex = 0;
|
|
const stderrPaths: string[] = [];
|
|
const tapes: string[] = [];
|
|
async function cli(args: string[], expectedStderr = "") {
|
|
const child = Bun.spawn([binary, ...args], { cwd: workspace, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
|
const timer = setTimeout(() => child.kill(), 30_000);
|
|
try {
|
|
const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
|
expect(code).toBe(0);
|
|
expect(stderr).toBe(expectedStderr);
|
|
return JSON.parse(stdout);
|
|
} finally { clearTimeout(timer); }
|
|
}
|
|
async function launch(withoutCredential = false, withTraceLog = true) {
|
|
const tape = join(root, `terminal-${++launchIndex}.fxtape`);
|
|
const stderr = join(root, `terminal-${launchIndex}.stderr`);
|
|
tapes.push(tape);
|
|
stderrPaths.push(stderr);
|
|
const terminalEnv: Record<string, string> = { ...env, FX_RECORD: tape, FX_DEBUG_RECORD_SILENT_BANNER: "1",
|
|
FX_TRACE_LOG: join(root, `terminal-${launchIndex}.trace`),
|
|
FX_TRACE_SCOPES: "input,worker,session,scroll,agent,gateway,compaction",
|
|
};
|
|
if (withoutCredential) delete terminalEnv.AI_GATEWAY_API_KEY;
|
|
if (!withTraceLog) {
|
|
delete terminalEnv.FX_TRACE_LOG;
|
|
delete terminalEnv.FX_TRACE_SCOPES;
|
|
}
|
|
// Do not inherit provider overrides, credentials, shell startup or dotenv state.
|
|
const command = `/usr/bin/env -i ${Object.entries(terminalEnv).map(([key, value]) => shellQuote(`${key}=${value}`)).join(" ")} ${shellQuote(binary)} --resume ${shellQuote(sessionId)}`;
|
|
terminal = await TmuxSession.create({
|
|
cmd: command, cwd: workspace, env: { HOME: home, FX_SOUND: "0" }, isolated: true,
|
|
stderrPath: stderr, width: 90, height: 32, minimumHistoryLines: 25_000,
|
|
startupWaitMs: 0,
|
|
});
|
|
// A composer can be painted while resume history is still being restored.
|
|
// Wait for a stable frame, then acknowledge input without submitting a turn
|
|
// or retiring the retained resume transcript before the tested /compact.
|
|
await terminal.waitForStableComposer(20_000);
|
|
await terminal.sendLiteral("startup-input-handshake");
|
|
await terminal.waitForText("startup-input-handshake", 5000);
|
|
await terminal.sendKeys("C-u");
|
|
await terminal.waitForComposer(5000);
|
|
return terminal;
|
|
}
|
|
async function waitForResumeHandoff() {
|
|
if (lines < 18_000) return;
|
|
const tracePath = join(root, `terminal-${launchIndex}.trace`);
|
|
// Wait only after submission: the composer accepts input during restoration,
|
|
// but replies need the immutable history source to hand off to the live one.
|
|
// 18k rows / 64 per frame needs about 282 frames. Debug observations reached
|
|
// 77 ms median / 134 ms max per frame; 60s bounds the drain, not the reply.
|
|
await until(() => readFileSync(tracePath, "utf8").includes("resume historical flow committed"),
|
|
`resume history-source handoff in ${tracePath}`, 60_000);
|
|
}
|
|
function durable() {
|
|
const bytes = readFileSync(eventsPath);
|
|
expect(bytes.subarray(0, initial.length).equals(initial)).toBe(true);
|
|
return checkpoints(bytes);
|
|
}
|
|
async function close() {
|
|
if (!terminal) return;
|
|
expect(terminal.paneStatus()).toEqual({ dead: false, status: null });
|
|
await terminal.sendText("/quit");
|
|
expect(await terminal.waitForSessionEnd(5000)).toBe(true);
|
|
await terminal.kill();
|
|
terminal = undefined;
|
|
for (const path of stderrPaths) expect(readFileSync(path, "utf8")).toBe("");
|
|
}
|
|
async function cleanup(passed: boolean) {
|
|
if (!passed) {
|
|
writeFileSync(join(root, "requests.json"), JSON.stringify({
|
|
trigger, outcome, phase, seedTurns, summaries, ordinary, overflowSent, requests,
|
|
paneStatus: terminal?.paneStatus(),
|
|
}, null, 2));
|
|
if (terminal) writeFileSync(join(root, "failure.scrollback.ansi"), await terminal.captureFullScrollbackEscapes());
|
|
console.error(`compaction evidence retained: ${root}; phase=${phase}, summaries=${summaries}, ordinary=${ordinary}`);
|
|
}
|
|
summaryHold.dispose();
|
|
ordinaryHold.dispose();
|
|
await terminal?.kill();
|
|
gateway.stop();
|
|
if (passed) rmSync(root, { recursive: true, force: true });
|
|
}
|
|
try {
|
|
for (let turn = 1; turn <= seedTurns; turn++) {
|
|
const reply = await cli(["ask", "--json", "--auto", ...(sessionId ? ["--resume", sessionId] : []), `Seed ordinary historical turn ${turn}.`], "Reading seed-notes.txt\n");
|
|
if (sessionId) expect(reply.session_id).toBe(sessionId);
|
|
else sessionId = reply.session_id;
|
|
expect(sessionId).toBeTruthy();
|
|
expect(ordinary).toBe(turn);
|
|
expect(summaries).toBe(0);
|
|
}
|
|
eventsPath = join(sessionDir(home, sessionId), "log.jsonl");
|
|
initial = readFileSync(eventsPath);
|
|
// A large read's body is one of the session's blobs (D44).
|
|
expect(savedText(home, sessionId)).toContain(HEAD);
|
|
expect(savedText(home, sessionId)).toContain(TAIL);
|
|
expect(checkpoints(initial)).toBe(0);
|
|
expect(summaries).toBe(0);
|
|
expect(ordinary).toBe(seedTurns);
|
|
phase = "attempt";
|
|
return {
|
|
launch, close, cleanup, durable, cli, tapes, root, seedTurns, summaryHold, ordinaryHold, waitForResumeHandoff,
|
|
savedEvents: () => readFileSync(eventsPath, "utf8"),
|
|
counts: () => ({ summaries, ordinary, overflowSent }),
|
|
phase: (value: typeof phase) => { phase = value; },
|
|
lastRequest: () => gateway.requests.at(-1)!.body,
|
|
requestsContaining: (marker: string) => gateway.requests.map((request) => request.body).filter((body) => body.includes(marker)),
|
|
};
|
|
} catch (error) { await cleanup(false); throw error; }
|
|
}
|
|
|
|
async function assertSilent(terminal: InstanceType<typeof TmuxSession>, root: string, label: string) {
|
|
const inline = await terminal.captureFullScrollbackEscapes();
|
|
writeFileSync(join(root, `${label}.scrollback.ansi`), inline);
|
|
expect(inline.length).toBeGreaterThan(0);
|
|
expect(inline).not.toMatch(COMPACTION_OUTPUT);
|
|
await terminal.sendKeys("C-o");
|
|
await terminal.waitForText("full detail", 5000);
|
|
await terminal.sendKeys("End");
|
|
const full = await terminal.waitForPane((pane) => pane.includes("full detail"), 5000);
|
|
writeFileSync(join(root, `${label}.full-transcript.txt`), full);
|
|
expect(full).not.toMatch(COMPACTION_OUTPUT);
|
|
await terminal.sendKeys("C-o");
|
|
await terminal.waitForComposer(5000);
|
|
}
|
|
|
|
async function assertLive(terminal: InstanceType<typeof TmuxSession>) {
|
|
const first = await terminal.waitForText(ACTIVITY, 10_000);
|
|
const clock = first.match(ACTIVITY)![0];
|
|
const markers = new Set<boolean>();
|
|
const later = await terminal.waitForPane((pane) => {
|
|
const rows = pane.split("\n").filter((line) => ACTIVITY.test(line));
|
|
if (rows.length === 0) return false;
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]).not.toMatch(/[↑↓]|tokens|chunk|%/i);
|
|
markers.add(rows[0].includes("•"));
|
|
return markers.size === 2 && !rows[0].includes(clock);
|
|
}, 5000);
|
|
expect(later.match(/Compacting/g)).toHaveLength(1);
|
|
}
|
|
|
|
describe.skipIf(!tmuxAvailable())("tui: compaction activity", () => {
|
|
for (const trigger of ["manual", "auto", "overflow"] as const) {
|
|
test(`${trigger}: held summary is transient, commits once released, and resumes ordinary work`, async () => {
|
|
const f = await fixture(trigger, "success", trigger === "manual");
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch();
|
|
// First interaction after a long resume; do not retire the resume source with a prompt.
|
|
await terminal.sendText(trigger === "manual" ? "/compact" : "Continue the current turn.");
|
|
await until(() => f.counts().summaries === 1, "first summary request");
|
|
await assertLive(terminal);
|
|
expect(f.durable()).toBe(0);
|
|
expect(f.counts().ordinary).toBe(f.seedTurns + (trigger === "overflow" ? 1 : 0));
|
|
if (trigger === "manual") {
|
|
await terminal.sendKeys("C-o");
|
|
await terminal.waitForText("full detail", 5000);
|
|
expect(await terminal.capturePane()).not.toMatch(COMPACTION_OUTPUT);
|
|
await terminal.sendKeys("C-o");
|
|
await terminal.waitForText(ACTIVITY, 5000);
|
|
await terminal.resizeWindow(52, 24);
|
|
await terminal.waitForText(ACTIVITY, 5000);
|
|
await terminal.resizeWindow(90, 32);
|
|
}
|
|
f.summaryHold.release(HANDOFF);
|
|
await until(() => f.durable() > 0, "acknowledged checkpoint");
|
|
if (trigger === "manual") {
|
|
await terminal.waitForPane((pane) => !ACTIVITY.test(pane) && hasEmptyComposer(pane), 10_000);
|
|
expect(f.counts().ordinary).toBe(f.seedTurns);
|
|
// The reply holds only facts, so fx asks once more for the notes
|
|
// of the turn that did work.
|
|
expect(f.counts().summaries).toBe(2);
|
|
} else {
|
|
await until(() => f.counts().ordinary === f.seedTurns + (trigger === "overflow" ? 2 : 1), "ordinary request after compaction");
|
|
const pane = await terminal.waitForText(/Thinking \(/, 10_000);
|
|
expect(pane).not.toMatch(ACTIVITY);
|
|
expect(f.lastRequest()).toContain("INTERNAL_HANDOFF_4e12");
|
|
f.ordinaryHold.release("CURRENT_TURN_OK_f713");
|
|
await f.waitForResumeHandoff();
|
|
await terminal.waitForPane((pane) => pane.includes("CURRENT_TURN_OK_f713") && hasEmptyComposer(pane), 10_000);
|
|
}
|
|
expect(f.counts().overflowSent).toBe(trigger === "overflow");
|
|
expect(f.durable()).toBe(1);
|
|
const before = f.counts();
|
|
f.phase("followup");
|
|
await terminal.sendText("Follow the latest request.");
|
|
await terminal.waitForPane((pane) => pane.includes(FOLLOWUP) && hasEmptyComposer(pane), 10_000);
|
|
expect(f.counts()).toEqual({ ...before, ordinary: before.ordinary + 1 });
|
|
expect(f.lastRequest()).toContain("INTERNAL_HANDOFF_4e12");
|
|
await assertSilent(terminal, f.root, "completed");
|
|
await f.close();
|
|
f.phase("reopen");
|
|
const reopened = await f.launch();
|
|
await reopened.sendText("Continue after reopening.");
|
|
await f.waitForResumeHandoff();
|
|
await reopened.waitForPane((pane) => pane.includes(REOPEN) && hasEmptyComposer(pane), 10_000);
|
|
expect(f.counts()).toEqual({ ...before, ordinary: before.ordinary + 2 });
|
|
expect(f.durable()).toBe(1);
|
|
expect(f.lastRequest()).toContain("INTERNAL_HANDOFF_4e12");
|
|
await assertSilent(reopened, f.root, "reopened");
|
|
await f.close();
|
|
for (const tape of f.tapes) {
|
|
const replay = await f.cli(["replay", tape, "--json"]);
|
|
expect(replay.frame_count).toBeGreaterThan(0);
|
|
expect(replay.stdout_bytes).toBeGreaterThan(0);
|
|
}
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 120_000);
|
|
}
|
|
|
|
for (const trigger of ["manual", "auto"] as const) {
|
|
test(`${trigger}: a message submitted during a held compaction waits and runs after with compacted context`, async () => {
|
|
const f = await fixture(trigger, "success", trigger === "manual");
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch();
|
|
await terminal.sendText(trigger === "manual" ? "/compact" : "Continue the current turn.");
|
|
await until(() => f.counts().summaries === 1, "summary request in flight");
|
|
await terminal.waitForText(ACTIVITY, 10_000);
|
|
const steer = trigger === "manual" ? "STEER_DURING_COMPACT_7f2a" : "STEER_DURING_AUTOCOMPACT_3b9c";
|
|
await terminal.sendText(steer);
|
|
// While the summary is held, the message must neither start its own
|
|
// turn nor cancel the compaction: no new gateway request arrives and
|
|
// the activity row stays visible. The resume backlog can delay fresh
|
|
// transcript cards, so only footer activity is asserted here.
|
|
await Bun.sleep(1500);
|
|
expect(f.counts()).toEqual({ summaries: 1, ordinary: f.seedTurns, overflowSent: false });
|
|
await terminal.waitForText(ACTIVITY, 5000);
|
|
// Flip phases before releasing so every later ordinary request gets a
|
|
// fresh FOLLOWUP response; the steer turn can start within milliseconds
|
|
// of the checkpoint, before a later flip would land.
|
|
f.phase("followup");
|
|
f.summaryHold.release(HANDOFF);
|
|
await until(() => f.durable() > 0, "acknowledged checkpoint");
|
|
await f.waitForResumeHandoff();
|
|
await until(() => f.requestsContaining(steer).length > 0, "steer request after compaction");
|
|
expect(f.requestsContaining(steer)[0]).toContain("INTERNAL_HANDOFF_4e12");
|
|
await terminal.waitForPane((pane) => pane.includes(FOLLOWUP) && hasEmptyComposer(pane), 20_000);
|
|
await f.close();
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 90_000);
|
|
}
|
|
|
|
test("overflow: a message submitted during held in-turn compaction rides the rebuilt request", async () => {
|
|
const f = await fixture("overflow", "success");
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch();
|
|
await terminal.sendText("Continue the current turn.");
|
|
await until(() => f.counts().summaries === 1, "summary request in flight");
|
|
await terminal.waitForText(ACTIVITY, 10_000);
|
|
const steer = "STEER_DURING_OVERFLOW_5c1d";
|
|
await terminal.sendText(steer);
|
|
// While the summary is held, the message must neither start its own
|
|
// turn nor cancel the compaction.
|
|
await Bun.sleep(1500);
|
|
expect(f.counts()).toEqual({ summaries: 1, ordinary: f.seedTurns + 1, overflowSent: true });
|
|
await terminal.waitForText(ACTIVITY, 5000);
|
|
f.summaryHold.release(HANDOFF);
|
|
await until(() => f.durable() > 0, "acknowledged checkpoint");
|
|
// The rebuilt continuation is held by ordinaryHold; the steering must
|
|
// already ride it, as same-turn guidance, before any reply arrives.
|
|
await until(() => f.requestsContaining(steer).length > 0, "steering in the rebuilt request");
|
|
const steered = f.requestsContaining(steer);
|
|
expect(steered).toHaveLength(1);
|
|
expect(steered[0]).toContain("<user_steering>");
|
|
expect(steered[0]).toContain("INTERNAL_HANDOFF_4e12");
|
|
f.ordinaryHold.release("CURRENT_TURN_OK_OVERFLOW_STEER");
|
|
await terminal.waitForPane((pane) => pane.includes("CURRENT_TURN_OK_OVERFLOW_STEER") && hasEmptyComposer(pane), 20_000);
|
|
await f.close();
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 90_000);
|
|
|
|
for (const outcome of ["cancel", "empty", "provider-error"] as const) {
|
|
test(`manual ${outcome}: scoped feedback preserves history and permits later input and reopen`, async () => {
|
|
const f = await fixture("manual", outcome);
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch();
|
|
await terminal.sendText("/compact");
|
|
await until(() => f.counts().summaries > 0, "summary boundary");
|
|
if (outcome !== "provider-error") {
|
|
await assertLive(terminal);
|
|
expect(f.durable()).toBe(0);
|
|
if (outcome === "cancel") {
|
|
await terminal.sendInterruptEscapePair(10_000);
|
|
} else f.summaryHold.release("");
|
|
}
|
|
const feedback = outcome === "cancel"
|
|
? "Compaction cancelled. Try /compact again when ready."
|
|
: "Compaction failed. Try /compact again.";
|
|
await terminal.waitForPane((pane) => !ACTIVITY.test(pane) && pane.includes(feedback) && hasEmptyComposer(pane), 15_000);
|
|
expect(f.durable()).toBe(0);
|
|
expect(f.counts().ordinary).toBe(f.seedTurns);
|
|
// A failed or empty summary is retried once on the fallback model.
|
|
expect(f.counts().summaries).toBe(outcome === "cancel" ? 1 : 2);
|
|
const afterFailure = f.counts();
|
|
if (outcome === "cancel") f.summaryHold.dispose();
|
|
await terminal.sendKeys("Escape");
|
|
await terminal.waitForPane((pane) => !/compact/i.test(pane) && hasEmptyComposer(pane), 5000);
|
|
await assertSilent(terminal, f.root, "failed");
|
|
f.phase("followup");
|
|
await terminal.sendText("Recover with another request.");
|
|
await terminal.waitForPane((pane) => pane.includes(FOLLOWUP) && hasEmptyComposer(pane), 10_000);
|
|
expect(f.counts()).toEqual({ ...afterFailure, ordinary: f.seedTurns + 1 });
|
|
expect(f.lastRequest()).toContain(HEAD);
|
|
expect(f.lastRequest()).toContain(TAIL);
|
|
expect(f.lastRequest()).not.toContain("INTERNAL_HANDOFF_4e12");
|
|
expect(f.durable()).toBe(0);
|
|
await f.close();
|
|
f.phase("reopen");
|
|
const reopened = await f.launch();
|
|
await reopened.sendText("Check the reopened context.");
|
|
await reopened.waitForPane((pane) => pane.includes(REOPEN) && hasEmptyComposer(pane), 10_000);
|
|
expect(f.counts()).toEqual({ ...afterFailure, ordinary: f.seedTurns + 2 });
|
|
expect(f.lastRequest()).toContain(HEAD);
|
|
expect(f.lastRequest()).toContain(TAIL);
|
|
expect(f.lastRequest()).not.toContain("INTERNAL_HANDOFF_4e12");
|
|
expect(f.durable()).toBe(0);
|
|
await assertSilent(reopened, f.root, "failed-reopen");
|
|
await f.close();
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 90_000);
|
|
}
|
|
|
|
for (const trigger of ["auto", "ordinary"] as const) {
|
|
test(`${trigger}: cancellation provenance preserves the correct transcript after reopen`, async () => {
|
|
const f = await fixture(trigger);
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch();
|
|
await terminal.sendText("Cancel this held response.");
|
|
await until(() => trigger === "auto" ? f.counts().summaries === 1 : f.counts().ordinary === 2, "held cancellation boundary");
|
|
await terminal.waitForText(trigger === "auto" ? ACTIVITY : /Thinking \(/, 10_000);
|
|
await terminal.sendInterruptEscapePair(10_000);
|
|
if (trigger === "auto") {
|
|
await terminal.waitForText("Compaction cancelled.", 10_000);
|
|
// An empty composer already exists while feedback is visible. Wait
|
|
// for Escape's visible acknowledgement, not a fixed sendKeys delay,
|
|
// so the slash in /quit cannot become part of a Meta key sequence.
|
|
terminal.sendKeysImmediate(["Escape"]);
|
|
} else {
|
|
await terminal.waitForText("Cancelled", 10_000);
|
|
}
|
|
await terminal.waitForPane((pane) => hasEmptyComposer(pane) &&
|
|
!/Compaction cancelled|Compacting|Thinking/.test(pane), 5000);
|
|
const interruptions = () => f.savedEvents().trim().split("\n")
|
|
.map((line) => JSON.parse(line)).filter((line) => line.kind === "item" && line.type === "interruption");
|
|
await until(() => interruptions().length === 1, "durable cancellation", 5000);
|
|
expect(interruptions()[0].data.reason).toBe("cancelled");
|
|
// An ordinary cancellation leaves its origin out, which reads as "turn".
|
|
expect((interruptions()[0].data.cancellation_origin ?? "turn") === "compaction").toBe(trigger === "auto");
|
|
const cancelledCounts = { summaries: trigger === "auto" ? 1 : 0,
|
|
ordinary: f.seedTurns + (trigger === "ordinary" ? 1 : 0), overflowSent: false };
|
|
expect(f.counts()).toEqual(cancelledCounts);
|
|
expect(f.durable()).toBe(0);
|
|
const cancelledEvents = f.savedEvents();
|
|
await f.close();
|
|
expect(f.counts()).toEqual(cancelledCounts);
|
|
expect(f.savedEvents().startsWith(cancelledEvents)).toBe(true);
|
|
expect(conversationLines(f.savedEvents())).toEqual(conversationLines(cancelledEvents));
|
|
|
|
const reopened = await f.launch();
|
|
await reopened.sendKeys("C-o");
|
|
await reopened.waitForText("full detail", 5000);
|
|
await reopened.sendKeys("End");
|
|
const full = await reopened.waitForPane((pane) => pane.includes("Cancel this held response."), 5000);
|
|
writeFileSync(join(f.root, "cancellation-reopened.txt"), full);
|
|
expect(full.includes("Cancelled")).toBe(trigger === "ordinary");
|
|
expect(full).not.toMatch(COMPACTION_OUTPUT);
|
|
await reopened.sendKeys("C-o");
|
|
await reopened.waitForComposer(5000);
|
|
expect(await reopened.captureFullScrollbackEscapes()).not.toMatch(COMPACTION_OUTPUT);
|
|
await f.close();
|
|
expect(f.counts()).toEqual(cancelledCounts);
|
|
expect(f.savedEvents().startsWith(cancelledEvents)).toBe(true);
|
|
expect(conversationLines(f.savedEvents())).toEqual(conversationLines(cancelledEvents));
|
|
expect(f.durable()).toBe(0);
|
|
for (const tape of f.tapes) {
|
|
const replay = await f.cli(["replay", tape, "--json"]);
|
|
expect(replay.frame_count).toBeGreaterThan(0);
|
|
expect(replay.stdout_bytes).toBeGreaterThan(0);
|
|
}
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 60_000);
|
|
}
|
|
|
|
test("manual failure then success: /trace records compaction decisions and failures without FX_TRACE", async () => {
|
|
const f = await fixture("manual", "provider-error");
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch(false, false);
|
|
await terminal.sendText("/compact");
|
|
await terminal.waitForText("Compaction failed. Try /compact again.", 15_000);
|
|
expect(f.durable()).toBe(0);
|
|
f.phase("followup");
|
|
await terminal.sendText("/compact");
|
|
await until(() => f.durable() === 1, "checkpoint after recovery compact");
|
|
await terminal.waitForPane((pane) => !ACTIVITY.test(pane) && hasEmptyComposer(pane), 10_000);
|
|
|
|
const before = new Set(readdirSync(f.root).filter((name) => name.startsWith("fx-trace-") && name.endsWith(".md")));
|
|
let report = "";
|
|
await terminal.sendText("/trace");
|
|
await until(() => {
|
|
const fresh = readdirSync(f.root).filter((name) => name.startsWith("fx-trace-") && name.endsWith(".md") && !before.has(name));
|
|
if (fresh.length === 0) return false;
|
|
const content = readFileSync(join(f.root, fresh[fresh.length - 1]), "utf8");
|
|
if (!content.includes("## Transcript Timeline")) return false;
|
|
report = content;
|
|
return true;
|
|
}, "trace report", 15_000);
|
|
|
|
// No FX_TRACE_LOG was configured, so the opt-in trace tail is absent while
|
|
// the always-on compaction section still carries the full story.
|
|
expect(report).not.toContain("## Trace Tail");
|
|
expect(report).toContain("## Context Compaction\n");
|
|
expect(report).toMatch(/last=\d+ failed=[1-9]\d* \(always recorded; does not require FX_TRACE\)/);
|
|
expect(report).toContain("event=summary_transport_failed");
|
|
// Provider error text is secret-masked before it reaches the report.
|
|
expect(report).toContain("[redacted]");
|
|
expect(report).not.toContain("abcdef0123456789xyz");
|
|
expect(report).toContain("event=transaction_failed");
|
|
expect(report).toContain("event=provider_start");
|
|
expect(report).toContain("event=provider_completed");
|
|
expect(report).toContain("event=committed");
|
|
const problems = report.split("## Problems")[1]?.split("##")[0] ?? "";
|
|
expect(problems).toContain("- context compaction");
|
|
await f.close();
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 120_000);
|
|
|
|
test("missing credentials keep compaction feedback local without hiding ordinary auth errors", async () => {
|
|
const f = await fixture("manual");
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch(true);
|
|
const authMessage = "fx needs access to Vercel AI Gateway";
|
|
async function authNotices(label: string) {
|
|
await terminal.sendKeys("C-o");
|
|
await terminal.waitForText("full detail", 5000);
|
|
await terminal.sendKeys("End");
|
|
const pane = await terminal.waitForPane((text) => text.includes("full detail") && text.includes(authMessage), 5000);
|
|
writeFileSync(join(f.root, `${label}.txt`), pane);
|
|
await terminal.sendKeys("C-o");
|
|
await terminal.waitForComposer(5000);
|
|
return pane.split(authMessage).length - 1;
|
|
}
|
|
const initialNotices = await authNotices("auth-before");
|
|
expect(initialNotices).toBeGreaterThan(0);
|
|
await terminal.sendText("/compact");
|
|
await terminal.waitForText("Compaction was not started.", 5000);
|
|
expect(f.counts().summaries).toBe(0);
|
|
expect(f.counts().ordinary).toBe(f.seedTurns);
|
|
expect(f.durable()).toBe(0);
|
|
await terminal.sendKeys("Escape");
|
|
expect(await authNotices("auth-after-compaction")).toBe(initialNotices);
|
|
|
|
await terminal.sendText("Ordinary request without a credential.");
|
|
await terminal.waitForText(authMessage, 5000);
|
|
expect(await authNotices("auth-after-ordinary")).toBe(initialNotices + 1);
|
|
expect(f.counts().ordinary).toBe(f.seedTurns);
|
|
await f.close();
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 60_000);
|
|
|
|
test("ordinary held request remains Thinking without compaction or extra requests", async () => {
|
|
const f = await fixture("ordinary");
|
|
let passed = false;
|
|
try {
|
|
const terminal = await f.launch();
|
|
await terminal.sendText("Ordinary control request.");
|
|
await until(() => f.counts().ordinary === 2, "ordinary held request");
|
|
await terminal.waitForText(/Thinking \(/, 5000);
|
|
expect(f.counts().summaries).toBe(0);
|
|
expect(f.durable()).toBe(0);
|
|
f.ordinaryHold.release("ORDINARY_CONTROL_OK_727a");
|
|
await terminal.waitForPane((pane) => pane.includes("ORDINARY_CONTROL_OK_727a") && hasEmptyComposer(pane), 10_000);
|
|
expect(f.counts().ordinary).toBe(2);
|
|
await assertSilent(terminal, f.root, "control");
|
|
await f.close();
|
|
passed = true;
|
|
} finally { await f.cleanup(passed); }
|
|
}, 60_000);
|
|
|
|
test("auto: steering sent during a turn stays user text through compaction", async () => {
|
|
const root = mkdtempSync(join(tmpdir(), "fx-compaction-steering-"));
|
|
const home = join(root, "home");
|
|
const workspace = join(root, "workspace");
|
|
const stderrPath = join(root, "stderr");
|
|
mkdirSync(join(home, ".fx"), { recursive: true });
|
|
mkdirSync(workspace);
|
|
writeFileSync(join(home, ".fx/settings.json"), JSON.stringify({
|
|
model: FAKE_GATEWAY_MODEL, auto_upgrade: false, startup_scrollback: false,
|
|
}));
|
|
for (let n = 1; n <= 30; n++) {
|
|
writeFileSync(join(workspace, `probe-${n}.txt`), `STEER_PROBE_${n}_8d4 context pressure line\n`.repeat(250));
|
|
}
|
|
const task = "STEER_TASK_8d4: read the probe files and report progress.";
|
|
// Longer than the excerpt compaction used for generated notices.
|
|
const steering = "STEER_RULE_8d4: prefix every progress line with S1>. " +
|
|
"Keep all of this correction. ".repeat(80) + "STEER_END_8d4";
|
|
const ordinaryBodies: string[] = [];
|
|
const summaryBodies: string[] = [];
|
|
const gateway = startDynamicFakeGateway((raw) => {
|
|
const request = JSON.parse(raw);
|
|
if (request.toolChoice?.type === "none" && request.tools?.length === 0) {
|
|
summaryBodies.push(raw);
|
|
return fakeGatewayFinalText("STEER_SUMMARY_8d4: the probe files are being read.");
|
|
}
|
|
ordinaryBodies.push(raw);
|
|
const n = ordinaryBodies.length;
|
|
if (summaryBodies.length > 0) return fakeGatewayFinalText("STEER_DONE_8d4");
|
|
if (n === 1) return fakeShellRun("steer-window-8d4", "printf STEER_WINDOW_8d4; sleep 3");
|
|
if (n <= 31) return fakeGatewayToolCall(`steer-probe-${n - 1}-8d4`, "read_file", { path: `probe-${n - 1}.txt` });
|
|
return fakeGatewayFinalText("STEER_NOT_COMPACTED_8d4");
|
|
}, { models: [{ id: FAKE_GATEWAY_MODEL, type: "language", tags: ["tool-use"], context_window: 60000, max_tokens: 4096 }] });
|
|
const env: Record<string, string> = {
|
|
PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: home, TMPDIR: root,
|
|
TERM: "xterm-256color", AI_GATEWAY_API_KEY: "fake-compaction-key",
|
|
FX_DISABLE_KEYCHAIN: "1", FX_SKIP_ONBOARDING: "1", FX_E2E_DISABLE_DOTENV: "1",
|
|
FX_SOUND: "0", FX_AUTO_UPGRADE: "0", FX_PERMISSION_MODE: "full-access",
|
|
FX_MODEL: FAKE_GATEWAY_MODEL, FX_GATEWAY_BASE_URL: gateway.baseUrl,
|
|
FX_GATEWAY_CHAT_URL: gateway.chatUrl, FX_E2E_GATEWAY_CHAT_URL: gateway.chatUrl,
|
|
FX_E2E_GATEWAY_MODELS_URL: `${gateway.baseUrl}/coding-agent/v1/models`,
|
|
};
|
|
const command = `/usr/bin/env -i ${Object.entries(env)
|
|
.map(([key, value]) => shellQuote(`${key}=${value}`)).join(" ")} ${shellQuote(binary)}`;
|
|
let terminal: InstanceType<typeof TmuxSession> | undefined;
|
|
let passed = false;
|
|
try {
|
|
terminal = await TmuxSession.create({
|
|
cmd: command, cwd: workspace, env, isolated: true, remainOnExit: true,
|
|
stderrPath, width: 120, height: 40, startupWaitMs: 0,
|
|
});
|
|
await terminal.waitForStableComposer(15_000);
|
|
await terminal.sendText(task);
|
|
await terminal.waitForText("STEER_WINDOW_8d4", 15_000);
|
|
await terminal.sendText(steering);
|
|
await until(() => summaryBodies.length === 1, "automatic compaction", 60_000);
|
|
await terminal.waitForText("STEER_DONE_8d4", 20_000);
|
|
|
|
const steered = ordinaryBodies.filter((body) => body.includes("STEER_RULE_8d4"));
|
|
expect(steered.length).toBeGreaterThan(1);
|
|
expect(steered[0]).toContain("<user_steering>");
|
|
|
|
const source = JSON.parse(summaryBodies[0]!).prompt
|
|
.filter((message: { role: string }) => message.role === "user")
|
|
.map((message: { content: { text?: string }[] }) => message.content.map((part) => part.text ?? "").join(""))
|
|
.join("\n");
|
|
const at = source.indexOf("STEER_RULE_8d4");
|
|
expect(at).toBeGreaterThan(-1);
|
|
// Steering is the user's own message, labeled as added during the turn.
|
|
const header = source.slice(source.lastIndexOf("[", at), at);
|
|
expect(header).toBe("[User, added while the assistant worked]\n");
|
|
expect(source).toContain(steering);
|
|
// The task stays the live request, so the summarizer sees it only as context.
|
|
expect(source).toContain(`[Turn in progress]\n[User, this message stays in the conversation after the summary]\n${task}`);
|
|
|
|
const log = join(sessionDir(home, onlySessionId(home)), "log.jsonl");
|
|
await until(() => readFileSync(log, "utf8").includes("\"turn_committed\""), "saved turn", 10_000);
|
|
const handoff = readFileSync(log, "utf8").trim().split("\n")
|
|
.map((line) => JSON.parse(line)).find((line) => line.kind === "compacted")?.data.summary ?? "";
|
|
expect(handoff.startsWith("fx-compactor-v1\n")).toBe(true);
|
|
const state = JSON.parse(handoff.slice("fx-compactor-v1\n".length));
|
|
// Only the turn in progress was compacted; its steering is kept exactly.
|
|
expect(state.turns).toEqual([]);
|
|
expect(state.open.users).toEqual([steering]);
|
|
expect(state.tool_count).toBeGreaterThan(0);
|
|
expect(state.open.first_tool).toBe(1);
|
|
|
|
const afterCompaction = ordinaryBodies.at(-1)!;
|
|
expect(afterCompaction).toContain("<compacted_conversation>");
|
|
expect(afterCompaction).toContain("STEER_END_8d4");
|
|
expect(afterCompaction.split("STEER_TASK_8d4").length - 1).toBe(1);
|
|
expect(readFileSync(stderrPath, "utf8")).toBe("");
|
|
passed = true;
|
|
} finally {
|
|
if (!passed) {
|
|
writeFileSync(join(root, "failure.json"), JSON.stringify({ ordinary: ordinaryBodies.length, summaries: summaryBodies.length }, null, 2));
|
|
if (terminal) writeFileSync(join(root, "failure.scrollback.txt"), await terminal.captureFullScrollback());
|
|
console.error(`compaction steering evidence retained: ${root}`);
|
|
}
|
|
await terminal?.kill();
|
|
gateway.stop();
|
|
if (passed) rmSync(root, { recursive: true, force: true });
|
|
}
|
|
}, 90_000);
|
|
});
|