mirror of
https://github.com/vercel-labs/fx.git
synced 2026-10-28 10:15:17 +08:00
Pranit
c2c6c1da9d
A call to a tool fx doesn't have is answered with "Unsupported tool: NAME" and the turn continues, but that branch of the tool loop never called the rejection hook. Every other refused call does, and each host finishes its own record of a call through that hook. ACP clients kept showing the call as pending after the turn ended, fx ask --json left it out of tool_calls, and the /trace report did not count it. Record it as rejected, like the validation and availability failures beside it. ACP now sends a final failed update with the result the model received, fx ask --json lists the call with status error, and /trace counts it as rejected. Fixes #1340.
676 lines
41 KiB
TypeScript
676 lines
41 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { FX_BIN, runFx } from "../evals/eval-helpers";
|
|
import { completion, toolCompletion, createConfiguredProviderFixture as fixture } from "./fixtures/chat-completions";
|
|
|
|
async function withReasoning(response: Response, ...deltas: Record<string, unknown>[]) {
|
|
const prefix = deltas.map(delta => `data: ${JSON.stringify({ choices: [{ index: 0, delta }] })}\n\n`).join("");
|
|
return new Response(prefix + await response.text(), { headers: response.headers });
|
|
}
|
|
|
|
// Tool calls whose arguments are sent verbatim, so they can be malformed.
|
|
function rawToolCalls(model: string, calls: Array<{ id: string; name: string; arguments: string }>) {
|
|
const chunks = [
|
|
{ id: "chat-tool", model, choices: [{ index: 0, delta: { tool_calls: calls.map((call, index) => ({ index, id: call.id, type: "function", function: { name: call.name, arguments: call.arguments } })) }, finish_reason: null }] },
|
|
{ id: "chat-tool", model, choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] },
|
|
];
|
|
return new Response(chunks.map(value => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", { headers: { "content-type": "text/event-stream" } });
|
|
}
|
|
|
|
describe("configured providers", () => {
|
|
test.each(["reasoning", "reasoning_content"])("replays %s through a tool result and saved-session continuation", async field => {
|
|
let calls = 0;
|
|
const f = fixture(body => {
|
|
calls++;
|
|
if (calls === 1) return withReasoning(toolCompletion(body.model, "read_file", { path: "note.txt" }), { [field]: "Inspect note first." });
|
|
return withReasoning(completion(body.model, calls === 2 ? "read complete" : "resumed reply"), { [field]: "Finish answer." });
|
|
});
|
|
try {
|
|
writeFileSync(join(f.workspace, "note.txt"), "fixture contents");
|
|
const first = await runFx(["ask", "--json", "Read note.txt"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (first.code !== 0) throw new Error(first.stdout + first.stderr);
|
|
const saved = JSON.parse(first.stdout);
|
|
expect(saved.output).toBe("read complete");
|
|
expect(f.requests).toHaveLength(2);
|
|
const toolTurn = f.requests[1].body.messages.find((message: any) => message.role === "assistant" && message.tool_calls?.length);
|
|
expect(toolTurn[field]).toBe("Inspect note first.");
|
|
expect(toolTurn._tool_call_ids).toBeUndefined();
|
|
expect(f.requests[1].body.messages.at(-1).content).toContain("fixture contents");
|
|
const resumed = await runFx(["ask", "--json", "--resume", saved.session_id, "Continue the saved task"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (resumed.code !== 0) throw new Error(resumed.stdout + resumed.stderr);
|
|
expect(JSON.parse(resumed.stdout).output).toBe("resumed reply");
|
|
expect(f.requests).toHaveLength(3);
|
|
const replayed = f.requests[2].body.messages.filter((message: any) => message.role === "assistant" && message[field] !== undefined);
|
|
expect(replayed.map((message: any) => message[field])).toEqual(["Inspect note first.", "Finish answer."]);
|
|
expect(f.requests.every(request => request.authorization === null)).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 45000);
|
|
|
|
test("reasoning-only completion after silent tools reaches the continuation request", async () => {
|
|
let calls = 0;
|
|
const f = fixture(body => {
|
|
calls++;
|
|
if (calls <= 2) return toolCompletion(body.model, "read_file", { path: `note-${calls}.txt` }, `call-${calls}`);
|
|
if (calls === 3) return withReasoning(completion(body.model, ""), { reasoning_content: "Both files have been read." });
|
|
return completion(body.model, "The two files were read.");
|
|
});
|
|
try {
|
|
writeFileSync(join(f.workspace, "note-1.txt"), "first contents");
|
|
writeFileSync(join(f.workspace, "note-2.txt"), "second contents");
|
|
const result = await runFx(["ask", "--json", "--no-save", "Read both notes"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
expect(JSON.parse(result.stdout).output).toBe("The two files were read.");
|
|
expect(f.requests).toHaveLength(4);
|
|
const messages = f.requests[3].body.messages;
|
|
expect(messages.at(-1).content).toBe("Summarize what you just did.");
|
|
expect(messages.at(-2).reasoning_content).toBe("Both files have been read.");
|
|
expect(messages.at(-2).tool_calls).toBeUndefined();
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
|
|
test("preserves opaque reasoning and tool IDs on resume but never forwards them to another connection", async () => {
|
|
const firstDetail = { type: "reasoning.text", text: "Inspect the file.", signature: "opaque-signature", format: "anthropic-claude-v1", index: 0 };
|
|
const secondDetail = { type: "reasoning.encrypted", data: "opaque-encrypted-data", id: "reasoning-1", index: 1 };
|
|
const callId = "call:opaque/1";
|
|
let calls = 0;
|
|
const f = fixture(body => {
|
|
calls++;
|
|
if (calls === 1) return withReasoning(toolCompletion(body.model, "read_file", { path: "note.txt" }, callId), { reasoning_details: [firstDetail] }, { reasoning_details: [secondDetail] });
|
|
return completion(body.model, calls === 2 ? "read complete" : "continued");
|
|
});
|
|
try {
|
|
writeFileSync(join(f.workspace, "note.txt"), "fixture contents");
|
|
const first = await runFx(["ask", "--json", "Read note.txt"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (first.code !== 0) throw new Error(first.stdout + first.stderr);
|
|
const saved = JSON.parse(first.stdout);
|
|
expect(saved.output).toBe("read complete");
|
|
expect(f.requests).toHaveLength(2);
|
|
const toolTurn = f.requests[1].body.messages.find((message: any) => message.role === "assistant" && message.tool_calls?.length);
|
|
expect(toolTurn.reasoning_details).toEqual([firstDetail, secondDetail]);
|
|
expect(toolTurn.tool_calls[0].id).toBe(callId);
|
|
expect(toolTurn._tool_call_ids).toBeUndefined();
|
|
expect(f.requests[1].body.messages.at(-1).tool_call_id).toBe(callId);
|
|
const resumed = await runFx(["ask", "--json", "--resume", saved.session_id, "Continue the saved task"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (resumed.code !== 0) throw new Error(resumed.stdout + resumed.stderr);
|
|
expect(f.requests).toHaveLength(3);
|
|
const restored = f.requests[2].body.messages.find((message: any) => message.reasoning_details);
|
|
expect(restored.reasoning_details).toEqual([firstDetail, secondDetail]);
|
|
expect(restored.tool_calls[0].id).toBe(callId);
|
|
const switched = await runFx(["ask", "--json", "--resume", saved.session_id, "Continue on the other connection"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote" }, timeoutMs: 20000 });
|
|
if (switched.code !== 0) throw new Error(switched.stdout + switched.stderr);
|
|
expect(f.requests).toHaveLength(4);
|
|
expect(f.requests[3].authorization).toBe(`Bearer ${f.env.FX_TEST_PROVIDER_TOKEN}`);
|
|
expect(f.requests[3].body.messages.every((message: any) => message.reasoning_details === undefined && message.reasoning === undefined && message.reasoning_content === undefined)).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 65000);
|
|
|
|
test("status identifies the configured connection and endpoint without probing it", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const result = await runFx(["status", "--json"], { cwd: f.workspace, env: f.env });
|
|
expect(result.code).toBe(0);
|
|
const status = JSON.parse(result.stdout);
|
|
expect(status.model_source).toBe("local");
|
|
expect(status.provider_endpoint).toBe(f.settings.providers.local.base_url);
|
|
expect(status.connected_providers).toContain("local");
|
|
expect(f.requests).toHaveLength(0);
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("anonymous local requests use Chat Completions without Gateway credentials", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const result = await runFx(["ask", "--json", "--no-save", "say hello"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
expect(result.stderr).toBe("");
|
|
if (result.code !== 0) throw new Error(`fx ask failed: ${result.stdout} ${result.stderr}; paths=${f.requests.map(r => r.path).join(",")}`);
|
|
expect(result.code).toBe(0);
|
|
expect(JSON.parse(result.stdout).output).toBe("local reply");
|
|
expect(f.requests).toHaveLength(1);
|
|
expect(f.requests[0].authorization).toBeNull();
|
|
expect(f.requests[0].path).toBe("/v1/chat/completions");
|
|
expect(f.requests[0].body.model).toBe("local-model");
|
|
expect(f.requests[0].body.messages.some((message: any) => message.role === "user")).toBe(true);
|
|
expect(f.requests[0].body.prompt).toBeUndefined();
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
|
|
test("terminal usage may repeat an empty matching finished choice", async () => {
|
|
const f = fixture(async body => {
|
|
const response = completion(body.model);
|
|
const wire = (await response.text()).replace('"choices":[]', '"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]');
|
|
return new Response(wire, { headers: response.headers });
|
|
});
|
|
try {
|
|
const result = await runFx(["ask", "--json", "--no-save", "say hello"], { cwd: f.workspace, env: f.env });
|
|
expect(result.code).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
const parsed = JSON.parse(result.stdout);
|
|
expect(parsed.final_output).toBe("local reply");
|
|
expect(parsed.usage).toMatchObject({ input_tokens: 12, output_tokens: 3 });
|
|
expect(f.requests).toHaveLength(1);
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("CLI selection persists a configured name and uses only its credential", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const selection = await runFx(["provider", "remote"], { cwd: f.workspace, env: f.env });
|
|
expect(selection.code).toBe(0);
|
|
expect(JSON.parse(readFileSync(f.settingsPath, "utf8")).provider).toBe("remote");
|
|
const result = await runFx(["ask", "--json", "--no-save", "say hello"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (result.code !== 0) throw new Error(`fx ask failed: ${result.stdout} ${result.stderr}; paths=${f.requests.map(r => r.path).join(",")}`);
|
|
expect(result.code).toBe(0);
|
|
expect(f.requests).toHaveLength(1);
|
|
expect(f.requests[0].authorization).toBe("Bearer own-provider-token");
|
|
expect(f.requests[0].body.model).toBe("remote-model");
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
|
|
test("CLI selection creates an owned model key from workspace-only preferences", async () => {
|
|
const f = fixture();
|
|
try {
|
|
delete (f.settings.models as any).remote;
|
|
(f.settings as any).workspaces = { [f.workspace]: { models: { remote: "workspace-model" } } };
|
|
f.save();
|
|
const selection = await runFx(["provider", "remote"], { cwd: f.workspace, env: f.env });
|
|
if (selection.code !== 0) throw new Error(selection.stdout + selection.stderr);
|
|
const saved = JSON.parse(readFileSync(f.settingsPath, "utf8"));
|
|
expect(saved.provider).toBe("remote");
|
|
expect(saved.models).toEqual({ local: "local-model", remote: "workspace-model" });
|
|
expect(saved.workspaces[f.workspace].models.remote).toBe("workspace-model");
|
|
expect(saved.providers).toEqual(f.settings.providers);
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("catalog listing and file model selection do not require a catalog login", async () => {
|
|
const f = fixture();
|
|
try {
|
|
f.settings.provider = "remote";
|
|
f.save();
|
|
const env = { ...f.env, FX_TEST_PROVIDER_TOKEN: undefined };
|
|
const catalog = await runFx(["models"], { cwd: f.workspace, env });
|
|
if (catalog.code !== 0) throw new Error(catalog.stdout + catalog.stderr);
|
|
expect(catalog.stdout).toContain("remote-model");
|
|
f.settings.models.remote = "opaque/custom:v1";
|
|
f.save();
|
|
const selection = await runFx(["status", "--json"], { cwd: f.workspace, env });
|
|
if (selection.code !== 0) throw new Error(selection.stdout + selection.stderr);
|
|
expect(JSON.parse(selection.stdout).model).toBe("opaque/custom:v1");
|
|
expect(f.requests).toHaveLength(0);
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("executes a tool and returns its result to the same connection", async () => {
|
|
const f = fixture(body => body.messages.some((message: any) => message.role === "tool")
|
|
? completion(body.model, "read succeeded")
|
|
: toolCompletion(body.model, "read_file", { path: "note.txt" }));
|
|
try {
|
|
writeFileSync(join(f.workspace, "note.txt"), "fixture contents");
|
|
const result = await runFx(["ask", "--json", "--no-save", "Read note.txt"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
expect(JSON.parse(result.stdout).output).toBe("read succeeded");
|
|
expect(f.requests).toHaveLength(2);
|
|
const returned = f.requests[1].body.messages.find((message: any) => message.role === "tool");
|
|
expect(returned.tool_call_id).toBe("call-local");
|
|
expect(returned.content).toContain("fixture contents");
|
|
expect(f.requests.every(request => request.authorization === null)).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
|
|
test("unknown tools and malformed arguments are answered and the turn continues", async () => {
|
|
const f = fixture(body => body.messages.some((message: any) => message.role === "tool")
|
|
? completion(body.model, "recovered")
|
|
: rawToolCalls(body.model, [
|
|
{ id: "call-read", name: "read_file", arguments: JSON.stringify({ path: "note.txt" }) },
|
|
{ id: "call-missing", name: "missing_tool", arguments: "{}" },
|
|
{ id: "call-malformed", name: "read_file", arguments: "{\"path\":\"note.txt\"" },
|
|
]));
|
|
try {
|
|
writeFileSync(join(f.workspace, "note.txt"), "fixture contents");
|
|
const result = await runFx(["ask", "--json", "--no-save", "Read note.txt"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
const output = JSON.parse(result.stdout);
|
|
expect(output.output).toBe("recovered");
|
|
expect(output.tool_calls.map((call: any) => [call.name, call.status]).sort()).toEqual([
|
|
["missing_tool", "error"],
|
|
["read_file", "error"],
|
|
["read_file", "success"],
|
|
]);
|
|
expect(f.requests).toHaveLength(2);
|
|
const messages = f.requests[1].body.messages;
|
|
const replayed = messages.find((message: any) => message.role === "assistant" && message.tool_calls?.length).tool_calls;
|
|
expect(replayed.map((call: any) => call.function.arguments)).toEqual([JSON.stringify({ path: "note.txt" }), "{}", "{}"]);
|
|
const results = Object.fromEntries(messages.filter((message: any) => message.role === "tool").map((message: any) => [message.tool_call_id, message.content]));
|
|
expect(results["call-read"]).toContain("fixture contents");
|
|
expect(results["call-missing"]).toBe("Unsupported tool: missing_tool");
|
|
expect(results["call-malformed"]).toContain("did not run the call");
|
|
expect(results["call-malformed"]).not.toContain("fixture contents");
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
|
|
test("saved sessions resume the named connection and reject endpoint rebinding", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const first = await runFx(["ask", "--json", "remember this"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (first.code !== 0) throw new Error(first.stdout + first.stderr);
|
|
const session = JSON.parse(first.stdout).session_id;
|
|
expect(session.length).toBeGreaterThan(0);
|
|
const second = await runFx(["ask", "--json", "--resume", session, "continue"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (second.code !== 0) throw new Error(second.stdout + second.stderr);
|
|
expect(f.requests).toHaveLength(2);
|
|
expect(f.requests[1].body.messages.some((message: any) => message.content?.includes("remember this"))).toBe(true);
|
|
f.settings.providers.local.base_url += "/changed";
|
|
f.save();
|
|
const rebound = await runFx(["ask", "--json", "--resume", session, "continue again"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
expect(rebound.code).not.toBe(0);
|
|
expect(f.requests).toHaveLength(2);
|
|
} finally { f.close(); }
|
|
}, 60000);
|
|
|
|
test.each(["inherited", "override", "unknown"])("subagent requests preserve connection and %s model capabilities", async mode => {
|
|
const childModel = mode === "inherited" ? "local-model" : mode === "override" ? "child-model" : "unknown-model";
|
|
const f = fixture(body => {
|
|
if (body.messages.some((message: any) => message.role === "user" && message.content?.includes("child-marker"))) return completion(body.model, "child reply");
|
|
if (body.messages.some((message: any) => message.role === "tool")) return completion(body.model, "parent reply");
|
|
return toolCompletion(body.model, "subagent", { request: { action: "run", task: "child-marker", ...(mode === "inherited" ? {} : { model: childModel }) } });
|
|
});
|
|
f.settings.providers.local.model_metadata["local-model"].max_output_tokens = 512;
|
|
(f.settings.providers.local.model_metadata as any)["child-model"] = { context_window: 32768, max_output_tokens: 1024, supports_tool_use: true };
|
|
f.save();
|
|
try {
|
|
const result = await runFx(["ask", "--json", "Delegate a task"], { cwd: f.workspace, env: f.env, timeoutMs: 30000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
expect(JSON.parse(result.stdout).output).toBe("parent reply");
|
|
expect(f.requests).toHaveLength(3);
|
|
expect(f.requests.map(request => request.body.model)).toEqual(["local-model", childModel, "local-model"]);
|
|
expect(f.requests.map(request => request.body.max_tokens)).toEqual([512, mode === "unknown" ? undefined : mode === "override" ? 1024 : 512, 512]);
|
|
expect(f.requests.every(request => request.authorization === null)).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 35000);
|
|
|
|
test("persistent child capabilities stay bound when the parent changes providers", async () => {
|
|
let callId = 0;
|
|
const f = fixture(body => {
|
|
if (!body.tools?.some((tool: any) => tool.function?.name === "subagent")) return completion(body.model, "child reply");
|
|
if (body.messages.at(-1)?.role === "tool") return completion(body.model, "parent reply");
|
|
return toolCompletion(body.model, "subagent", { request: { action: "message", agent: "reader", message: "read the child fixture" } }, `child-call-${++callId}`);
|
|
});
|
|
f.settings.models.local = "shared-model";
|
|
f.settings.models.remote = "shared-model";
|
|
(f.settings.providers.local as any).model_metadata = { "shared-model": { context_window: 32768, max_output_tokens: 512, supports_tool_use: true } };
|
|
(f.settings.providers.remote as any).model_metadata = { "shared-model": { context_window: 65536, max_output_tokens: 1024, supports_tool_use: true } };
|
|
f.save();
|
|
try {
|
|
const first = await runFx(["ask", "--json", "Create the named child"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote" }, timeoutMs: 30000 });
|
|
if (first.code !== 0) throw new Error(first.stdout + first.stderr);
|
|
const id = JSON.parse(first.stdout).session_id;
|
|
expect(f.requests.map(request => request.body.max_tokens)).toEqual([1024, 1024, 1024]);
|
|
const resumed = await runFx(["ask", "--json", "--resume", id, "Continue the named child"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "local" }, timeoutMs: 30000 });
|
|
if (resumed.code !== 0) throw new Error(resumed.stdout + resumed.stderr);
|
|
const resumedResult = f.requests.at(-1)!.body.messages.at(-1).content;
|
|
if (!JSON.parse(resumedResult).ok) throw new Error(resumedResult);
|
|
expect(f.requests.map(request => request.body.max_tokens)).toEqual([1024, 1024, 1024, 512, 1024, 512]);
|
|
expect(f.requests[3].authorization).toBeNull();
|
|
expect(f.requests[4].authorization).toBe(`Bearer ${f.env.FX_TEST_PROVIDER_TOKEN}`);
|
|
expect(f.requests[5].authorization).toBeNull();
|
|
f.settings.providers.remote.base_url += "/changed";
|
|
f.save();
|
|
const rebound = await runFx(["ask", "--json", "--resume", id, "Continue the named child again"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "local" }, timeoutMs: 30000 });
|
|
if (rebound.code !== 0) throw new Error(rebound.stdout + rebound.stderr);
|
|
expect(f.requests).toHaveLength(8);
|
|
expect(f.requests.slice(6).every(request => request.authorization === null && request.path === "/v1/chat/completions")).toBe(true);
|
|
expect(f.requests[7].body.messages.at(-1).content).toContain("child_failed");
|
|
} finally { f.close(); }
|
|
}, 100000);
|
|
|
|
test("configured child compaction uses the model capability lookup", async () => {
|
|
let parentTurns = 0;
|
|
const f = fixture(body => {
|
|
if (body.model === "child-model") {
|
|
// Context pressure comes from the child's own replies. Compaction keeps
|
|
// them word for word, shortening the longest, so it needs no model call.
|
|
return completion(body.model, "child reply " + "detail ".repeat(2000), Math.ceil(JSON.stringify(body).length / 4));
|
|
}
|
|
if (body.messages.at(-1)?.role === "tool") return completion(body.model, "parent reply");
|
|
parentTurns++;
|
|
return toolCompletion(body.model, "subagent", { request: { action: "message", agent: "reader", message: `child context ${parentTurns}`, ...(parentTurns === 1 ? { model: "child-model" } : {}) } }, `context-call-${parentTurns}`);
|
|
});
|
|
(f.settings.providers.local.model_metadata as any)["child-model"] = { context_window: 32768, max_output_tokens: 512, supports_tool_use: true };
|
|
f.save();
|
|
let id: string | undefined;
|
|
try {
|
|
for (let turn = 0; turn < 12; turn++) {
|
|
const result = await runFx(["ask", "--json", ...(id ? ["--resume", id] : []), "Continue the child context"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
id = JSON.parse(result.stdout).session_id;
|
|
const returned = f.requests.filter(request => request.body.model === "local-model").at(-1)!.body.messages.at(-1).content;
|
|
if (!JSON.parse(returned).ok) throw new Error(returned);
|
|
expect(JSON.parse(returned).ok).toBe(true);
|
|
}
|
|
const childRequests = f.requests.filter(request => request.body.model === "child-model");
|
|
// Only the child model's own small context window makes the child compact.
|
|
expect(childRequests.some(request => JSON.stringify(request.body.messages).includes("<compacted_conversation>"))).toBe(true);
|
|
expect(childRequests.every(request => request.body.max_tokens > 0 && request.body.max_tokens <= 512)).toBe(true);
|
|
expect(f.requests.every(request => request.path === "/v1/chat/completions" && request.authorization === null)).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 90000);
|
|
|
|
test("automatic permission review uses the custom connection", async () => {
|
|
const f = fixture(body => {
|
|
if (body.tools?.some((tool: any) => tool.function?.name === "permission_decision")) return toolCompletion(body.model, "permission_decision", { decision: "clear", rationale: "fixture review" });
|
|
if (body.messages.some((message: any) => message.role === "tool")) return completion(body.model, "edit finished");
|
|
return toolCompletion(body.model, "shell", { request: { action: "run", command: "python3 -c 'print(42)'" } });
|
|
});
|
|
try {
|
|
f.settings.permission_mode = "auto";
|
|
(f.settings.providers.local as any).reviewer_model = "review-model";
|
|
(f.settings.providers.local.model_metadata as any)["review-model"] = { context_window: 262144, max_output_tokens: 512 };
|
|
f.save();
|
|
const result = await runFx(["ask", "--json", "--no-save", "Run the Python snippet"], { cwd: f.workspace, env: f.env, timeoutMs: 30000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
const review = f.requests.find(request => request.body.tools?.some((tool: any) => tool.function?.name === "permission_decision"));
|
|
expect(review?.body.model).toBe("review-model");
|
|
expect(review?.body.max_tokens).toBe(512);
|
|
expect(f.requests.every(request => request.authorization === null)).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 35000);
|
|
|
|
test.each(["clear", "caution", "missing", "malformed", "refused", "truncated", "incomplete"])("custom review response recovery preserves %s outcome", async outcome => {
|
|
let marker = "";
|
|
let reviews = 0;
|
|
const f = fixture(body => {
|
|
if (body.tools?.some((tool: any) => tool.function?.name === "permission_decision")) {
|
|
reviews++;
|
|
if (reviews === 1 || (outcome === "missing" && reviews === 2)) {
|
|
if (outcome === "malformed") return rawToolCalls(body.model, [{ id: "review-1", name: "permission_decision", arguments: "{\"decision\":\"clear\"" }]);
|
|
if (outcome === "refused" || outcome === "truncated" || outcome === "incomplete") {
|
|
const chunks = [
|
|
{ id: "review", model: body.model, choices: [{ index: 0, delta: outcome === "refused" ? { refusal: "Cannot review this." } : { content: "Incomplete review." }, finish_reason: null }] },
|
|
{ id: "review", model: body.model, choices: [{ index: 0, delta: {}, finish_reason: outcome === "truncated" ? "length" : "stop" }] },
|
|
];
|
|
return new Response(chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join("") + (outcome === "incomplete" ? "" : "data: [DONE]\n\n"), { headers: { "content-type": "text/event-stream" } });
|
|
}
|
|
return completion(body.model, "The action looks safe.");
|
|
}
|
|
return toolCompletion(body.model, "permission_decision", { decision: outcome === "caution" ? "caution" : "clear" });
|
|
}
|
|
if (body.messages.some((message: any) => message.role === "tool")) return completion(body.model, "review finished");
|
|
return toolCompletion(body.model, "shell", { request: { action: "run", command: `python3 -c 'from pathlib import Path; p=Path(${JSON.stringify(marker)}); p.write_text(p.read_text()+"one\\n")'` } });
|
|
});
|
|
try {
|
|
marker = join(f.workspace, "review-effects.txt");
|
|
writeFileSync(marker, "");
|
|
f.settings.permission_mode = "auto";
|
|
f.save();
|
|
const result = await runFx(["ask", "--json", "--no-save", "Run the local fixture once."], { cwd: f.workspace, env: f.env, timeoutMs: 15000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
expect(readFileSync(marker, "utf8")).toBe(["clear", "malformed"].includes(outcome) ? "one\n" : "");
|
|
expect(reviews).toBe(["clear", "caution", "missing", "malformed"].includes(outcome) ? 2 : 1);
|
|
const requests = f.requests.filter(request => request.body.tools?.some((tool: any) => tool.function?.name === "permission_decision"));
|
|
if (reviews === 2) expect(requests[1].body).toEqual(requests[0].body);
|
|
expect(f.requests.every(request => request.path === "/v1/chat/completions")).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 20000);
|
|
|
|
test.each(["caution", "invalid"])("%s custom permission reviews never authorize execution", async decision => {
|
|
let marker = "";
|
|
const f = fixture(body => {
|
|
if (body.tools?.some((tool: any) => tool.function?.name === "permission_decision")) return toolCompletion(body.model, "permission_decision", { decision, rationale: "fixture review" });
|
|
if (body.messages.some((message: any) => message.role === "tool")) return completion(body.model, "action held");
|
|
return toolCompletion(body.model, "shell", { request: { action: "run", command: `python3 -c 'open("${marker}", "w").write("bad")'` } });
|
|
});
|
|
try {
|
|
marker = join(f.workspace, "review-must-not-run.txt");
|
|
f.settings.permission_mode = "auto";
|
|
f.save();
|
|
const result = await runFx(["ask", "--json", "--no-save", "Run the Python snippet"], { cwd: f.workspace, env: f.env, timeoutMs: 15000 });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
expect(f.requests.some(request => request.body.tools?.some((tool: any) => tool.function?.name === "permission_decision"))).toBe(true);
|
|
expect(existsSync(marker)).toBe(false);
|
|
expect(f.requests.every(request => request.path === "/v1/chat/completions")).toBe(true);
|
|
} finally { f.close(); }
|
|
}, 20000);
|
|
|
|
test("resume uses its own key when a different bearer connection is the default", async () => {
|
|
const f = fixture();
|
|
try {
|
|
(f.settings.providers.local as any).auth = { type: "bearer", env: "FX_TEST_LOCAL_TOKEN" };
|
|
f.save();
|
|
const env = { ...f.env, FX_TEST_LOCAL_TOKEN: "local-only-token" };
|
|
const first = await runFx(["ask", "--json", "remember remote"], { cwd: f.workspace, env: { ...env, FX_PROVIDER: "remote" }, timeoutMs: 20000 });
|
|
if (first.code !== 0) throw new Error(first.stdout + first.stderr);
|
|
const id = JSON.parse(first.stdout).session_id;
|
|
const resumed = await runFx(["ask", "--json", "--resume", id, "continue"], { cwd: f.workspace, env, timeoutMs: 20000 });
|
|
if (resumed.code !== 0) throw new Error(resumed.stdout + resumed.stderr);
|
|
expect(f.requests).toHaveLength(2);
|
|
expect(f.requests[1].body.model).toBe("remote-model");
|
|
expect(f.requests[1].authorization).toBe("Bearer own-provider-token");
|
|
} finally { f.close(); }
|
|
}, 45000);
|
|
|
|
test("malformed workspace provider overrides fail before any network request", async () => {
|
|
const f = fixture();
|
|
try {
|
|
f.settings.provider = "gateway";
|
|
(f.settings as any).workspaces = { [f.workspace]: { provider: "local", permission_mode: "invalid" } };
|
|
f.save();
|
|
const result = await runFx(["ask", "--json", "--no-save", "local-only prompt"], { cwd: f.workspace, env: f.env, timeoutMs: 10000 });
|
|
expect(result.code).not.toBe(0);
|
|
expect(f.requests).toHaveLength(0);
|
|
} finally { f.close(); }
|
|
}, 15000);
|
|
|
|
test("truncated tool streams cannot execute a partially delivered call", async () => {
|
|
const f = fixture(body => new Response(`data: ${JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "partial", type: "function", function: { name: "write_file", arguments: JSON.stringify({ path: "must-not-exist.txt", content: "unsafe" }) } }] }, finish_reason: "tool_calls" }] })}\n\n`, { headers: { "content-type": "text/event-stream" } }));
|
|
try {
|
|
f.settings.permission_mode = "full-access";
|
|
(f.settings as any).yolo_acknowledged = true;
|
|
f.save();
|
|
const result = await runFx(["ask", "--json", "--no-save", "Write must-not-exist.txt"], { cwd: f.workspace, env: f.env, timeoutMs: 10000 });
|
|
expect(result.code).not.toBe(0);
|
|
expect(existsSync(join(f.workspace, "must-not-exist.txt"))).toBe(false);
|
|
expect(f.requests).toHaveLength(1);
|
|
} finally { f.close(); }
|
|
}, 15000);
|
|
|
|
test.each(["headers", "stream"])("cancellation stops stalled custom %s without replaying the POST", async phase => {
|
|
const f = fixture(() => phase === "headers" ? new Promise<Response>(() => {}) : new Response(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode('data: {"choices":[{"index":0,"delta":{"content":"partial"}}]}\n\n')); } }), { headers: { "content-type": "text/event-stream" } }));
|
|
const env: Record<string, string | undefined> = { ...process.env };
|
|
for (const [key, value] of Object.entries(f.env)) {
|
|
if (value === undefined) delete env[key]; else env[key] = value;
|
|
}
|
|
const child = Bun.spawn([FX_BIN, "ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
|
const output = Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
|
let forced = false;
|
|
const timer = setTimeout(() => { forced = true; child.kill("SIGKILL"); }, 8000);
|
|
try {
|
|
const until = Date.now() + 5000;
|
|
while (f.requests.length === 0 && child.exitCode === null && Date.now() < until) await Bun.sleep(10);
|
|
expect(f.requests).toHaveLength(1);
|
|
child.kill("SIGINT");
|
|
expect(await child.exited).toBe(130);
|
|
await output;
|
|
expect(forced).toBe(false);
|
|
expect(f.requests).toHaveLength(1);
|
|
} finally {
|
|
clearTimeout(timer);
|
|
if (child.exitCode === null) child.kill("SIGKILL");
|
|
await child.exited;
|
|
await output;
|
|
f.close();
|
|
}
|
|
}, 12000);
|
|
|
|
test("a connection without a saved model runs from --model or FX_MODEL and rejects a blank FX_MODEL", async () => {
|
|
const f = fixture();
|
|
try {
|
|
delete (f.settings.models as Record<string, string>).local;
|
|
f.save();
|
|
const chatModels = () => f.requests.filter(request => request.path === "/v1/chat/completions").map(request => request.body.model);
|
|
|
|
const flag = await runFx(["ask", "--json", "--no-save", "--model", "flag-model", "hello"], { cwd: f.workspace, env: f.env, timeoutMs: 10000 });
|
|
if (flag.code !== 0) throw new Error(flag.stdout + flag.stderr);
|
|
const env = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_MODEL: " env-model " }, timeoutMs: 10000 });
|
|
if (env.code !== 0) throw new Error(env.stdout + env.stderr);
|
|
expect(chatModels()).toEqual(["flag-model", "env-model"]);
|
|
|
|
const status = await runFx(["status", "--json"], { cwd: f.workspace, env: { ...f.env, FX_MODEL: "env-model" } });
|
|
expect(status.code).toBe(0);
|
|
expect(JSON.parse(status.stdout)).toMatchObject({ model: "env-model", model_origin: "FX_MODEL", model_source: "local" });
|
|
|
|
const blank = await runFx(["status"], { cwd: f.workspace, env: { ...f.env, FX_MODEL: " " } });
|
|
expect(blank.code).toBe(1);
|
|
expect(blank.stderr).toBe("fx: no model is selected for this connection; save one under \"models\" in ~/.fx/settings.json, or set a model for this run with --model or FX_MODEL\n");
|
|
expect(chatModels()).toHaveLength(2);
|
|
expect(JSON.parse(readFileSync(f.settingsPath, "utf8")).models.local).toBeUndefined();
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
|
|
test("redirects cannot forward credentials to another connection", async () => {
|
|
const target = fixture();
|
|
const f = fixture(() => new Response("redirect", { status: 307, headers: { location: `${target.settings.providers.local.base_url}/chat/completions` } }));
|
|
try {
|
|
const result = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote" } });
|
|
expect(result.code).not.toBe(0);
|
|
expect(f.requests).toHaveLength(1);
|
|
expect(target.requests).toHaveLength(0);
|
|
} finally { f.close(); target.close(); }
|
|
});
|
|
|
|
test("provider error diagnostics redact the selected key", async () => {
|
|
const f = fixture(() => Response.json({ error: { message: "rejected own-provider-token" } }, { status: 401, headers: { "retry-after": "3" } }));
|
|
try {
|
|
const result = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote" }, timeoutMs: 10000 });
|
|
expect(result.code).toBe(1);
|
|
expect(() => JSON.parse(result.stdout)).not.toThrow();
|
|
expect(result.stderr).not.toContain("panic");
|
|
expect(result.stdout + result.stderr).not.toContain("own-provider-token");
|
|
expect(f.requests).toHaveLength(1);
|
|
} finally { f.close(); }
|
|
}, 15000);
|
|
|
|
test.each(["slashes", "unicode", "plain", "duplicate"])("provider errors redact opaque credentials encoded as %s", async encoding => {
|
|
const token = "alpha/beta/gamma";
|
|
const encoded = encoding === "slashes" ? token.replaceAll("/", "\\/") : [...token].map(char => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`).join("");
|
|
const body = encoding === "plain" ? `rejected ${token}` : encoding === "duplicate" ? `{"error":{"message":"rejected ${encoded}"},"${token}":0,"${encoded}":1}` : `{"error":{"message":"rejected ${encoded}","code":"${encoded}"}}`;
|
|
const f = fixture(() => new Response(body, { status: 400 }));
|
|
try {
|
|
const result = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote", FX_TEST_PROVIDER_TOKEN: token } });
|
|
expect(result.code).toBe(1);
|
|
expect(() => JSON.parse(result.stdout)).not.toThrow();
|
|
expect(result.stdout + result.stderr).not.toContain(token);
|
|
expect(result.stdout + result.stderr).toContain(encoding === "duplicate" ? "could not be decoded" : "rejected");
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("corrupt profile JSON cannot silently choose Gateway", async () => {
|
|
const f = fixture();
|
|
try {
|
|
writeFileSync(f.settingsPath, '{"provider":"local",');
|
|
const result = await runFx(["ask", "--json", "--no-save", "local-only prompt"], { cwd: f.workspace, env: f.env, timeoutMs: 8000 });
|
|
expect(result.code).not.toBe(0);
|
|
expect(f.requests).toHaveLength(0);
|
|
} finally { f.close(); }
|
|
}, 12000);
|
|
|
|
test("invalid bearer header bytes are rejected before sending", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const result = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote", FX_TEST_PROVIDER_TOKEN: "bad\nheader" } });
|
|
expect(result.code).not.toBe(0);
|
|
expect(f.requests).toHaveLength(0);
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("process overrides select a connection and opaque model without rewriting settings", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const before = readFileSync(f.settingsPath, "utf8");
|
|
const result = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote", FX_MODEL: "unlisted-model" } });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
expect(f.requests).toHaveLength(1);
|
|
expect(f.requests[0].body.model).toBe("unlisted-model");
|
|
expect(f.requests[0].authorization).toBe("Bearer own-provider-token");
|
|
expect(readFileSync(f.settingsPath, "utf8")).toBe(before);
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("project files cannot replace profile connections", async () => {
|
|
const f = fixture();
|
|
try {
|
|
writeFileSync(join(f.workspace, ".fx.json"), JSON.stringify({ provider: "remote", providers: { local: "invalid" } }));
|
|
const result = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: f.env });
|
|
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
|
|
expect(f.requests).toHaveLength(1);
|
|
expect(f.requests[0].body.model).toBe("local-model");
|
|
expect(f.requests[0].authorization).toBeNull();
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("saved history remains readable after its connection is removed", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const saved = await runFx(["ask", "--json", "remember this"], { cwd: f.workspace, env: f.env });
|
|
if (saved.code !== 0) throw new Error(saved.stdout + saved.stderr);
|
|
const id = JSON.parse(saved.stdout).session_id;
|
|
delete (f.settings.providers as any).local;
|
|
f.save();
|
|
const history = await runFx(["session", id, "--json"], { cwd: f.workspace, env: f.env });
|
|
if (history.code !== 0) throw new Error(history.stdout + history.stderr);
|
|
expect(history.stdout).toContain("remember this");
|
|
expect(f.requests).toHaveLength(1);
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
|
|
test("ignored workspace provider definitions stay inert when a sibling workspace field is malformed", async () => {
|
|
const f = fixture();
|
|
try {
|
|
(f.settings as any).workspaces = {
|
|
[f.workspace]: {
|
|
providers: { shadow: { protocol: "openai-chat-completions", base_url: "http://127.0.0.1:1/v1", auth: { type: "none" } } },
|
|
effort: 42,
|
|
},
|
|
};
|
|
f.save();
|
|
const result = await runFx(["status", "--json"], { cwd: f.workspace, env: f.env });
|
|
expect(result.code).toBe(0);
|
|
const status = JSON.parse(result.stdout);
|
|
expect(status.model_source).toBe("local");
|
|
expect(status.provider_endpoint).toBe(f.settings.providers.local.base_url);
|
|
expect(status.connected_providers).toContain("local");
|
|
expect(status.connected_providers).not.toContain("shadow");
|
|
expect(result.stderr).toContain("malformed_settings");
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("ignored workspace provider definitions with broken protocols stay inert during recovery", async () => {
|
|
const f = fixture();
|
|
try {
|
|
(f.settings as any).workspaces = {
|
|
[f.workspace]: {
|
|
providers: { shadow: { protocol: "bogus", base_url: "http://127.0.0.1:1/v1", auth: { type: "none" } } },
|
|
effort: 42,
|
|
},
|
|
};
|
|
f.save();
|
|
const result = await runFx(["status", "--json"], { cwd: f.workspace, env: f.env });
|
|
expect(result.code).toBe(0);
|
|
const status = JSON.parse(result.stdout);
|
|
expect(status.provider_endpoint).toBe(f.settings.providers.local.base_url);
|
|
expect(result.stderr).toContain("malformed_settings");
|
|
} finally { f.close(); }
|
|
});
|
|
|
|
test("unknown connections and missing keys fail without a Gateway fallback", async () => {
|
|
const f = fixture();
|
|
try {
|
|
const unknown = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "missing" } });
|
|
expect(unknown.code).not.toBe(0);
|
|
const missingKey = await runFx(["ask", "--json", "--no-save", "hello"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "remote", FX_TEST_PROVIDER_TOKEN: undefined } });
|
|
expect(missingKey.code).not.toBe(0);
|
|
expect(f.requests).toHaveLength(0);
|
|
} finally { f.close(); }
|
|
}, 25000);
|
|
});
|