mirrors/fx
0
0 Fork 0
mirror of https://github.com/vercel-labs/fx.git synced 2026-10-28 10:15:17 +08:00
Code Activity

Revert "Keep per-step runtime context at the message tail for prompt cache stability"

Browse Source
This commit is contained in:
Pranit authored and GitHub committed 2026-09-20 03:33:10 -04:00
1 parent 9ce23affe2
commit f3bf3cf47c
30 files changed
+143 -398
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/pgso/corpus.json
+1 -2
View File
@@ -126,7 +126,6 @@
{"name": "verify-tui-slash-menu", "argv": ["bun", "test", "--max-concurrency", "1", "./tui-slash-menu.test.ts"], "test_file": "tui-slash-menu.test.ts"},
{"name": "verify-web-fetch-permission-progress", "argv": ["bun", "test", "--max-concurrency", "1", "./web-fetch-permission-progress.test.ts"], "test_file": "web-fetch-permission-progress.test.ts", "requires_tmux": false},
{"name": "verify-web-search-permission-progress", "argv": ["bun", "test", "--max-concurrency", "1", "./web-search-permission-progress.test.ts"], "test_file": "web-search-permission-progress.test.ts", "requires_tmux": false},
{"name": "verify-yolo-permission-mode", "argv": ["bun", "test", "--max-concurrency", "1", "./yolo-permission-mode.test.ts"], "test_file": "yolo-permission-mode.test.ts"},
{"name": "verify-prompt-cache-stability", "argv": ["bun", "test", "--max-concurrency", "1", "./prompt-cache-stability.test.ts"], "test_file": "prompt-cache-stability.test.ts", "requires_tmux": false}
{"name": "verify-yolo-permission-mode", "argv": ["bun", "test", "--max-concurrency", "1", "./yolo-permission-mode.test.ts"], "test_file": "yolo-permission-mode.test.ts"}
]
}
scripts/pgso/tests/test_corpus.py
+1 -2
View File
@@ -75,7 +75,6 @@ VERIFICATION_E2E_TESTS = (
"web-fetch-permission-progress.test.ts",
"web-search-permission-progress.test.ts",
"yolo-permission-mode.test.ts",
"prompt-cache-stability.test.ts",
)
EXCLUDED_E2E_TESTS = (
@@ -371,7 +370,7 @@ class PgsoCorpusTests(unittest.TestCase):
tuple(test_file for test_file, _ in corpus.intentional_exclusions),
)
self.assertEqual(36, len(corpus.scenarios))
self.assertEqual(58, len(corpus.candidate_scenarios))
self.assertEqual(57, len(corpus.candidate_scenarios))
self.assertEqual(
{
"direct-help": 100,
sdk/node/test-term-workspace.mjs
+12 -21
View File
@@ -114,35 +114,26 @@ function textResponse(value) {
]);
}
// The volatile runtime context rides the message tail as trailing user
// messages; locate the last conversation user message instead. Mirrors
// isRuntimeOverlayMessage in tests/e2e/conditional-guidance-oracle.ts (the
// source of truth for the markers); plain node here cannot import the TS
// oracle, and the subagent-delivery and explicit-skill markers do not occur
// in this fixture's traffic.
function lastConversationUserIndex(prompt) {
for (let index = prompt.length - 1; index >= 0; index -= 1) {
const message = prompt[index];
if (message.role !== "user") continue;
const text = JSON.stringify(message.content);
if (text.includes("<fx-turn-context>") || text.includes('"Runtime context:')) continue;
return index;
}
return -1;
}
function toolResult(body, id) {
const prompt = body.prompt || [];
const lastUser = lastConversationUserIndex(prompt);
let lastUser = -1;
for (let index = prompt.length - 1; index >= 0; index -= 1) {
if (prompt[index].role === "user") {
lastUser = index;
break;
}
}
return prompt.slice(lastUser + 1)
.flatMap((message) => Array.isArray(message.content) ? message.content : [])
.find((part) => part.type === "tool-result" && part.toolCallId === id);
}
function latestUserText(body) {
const prompt = body.prompt || [];
const index = lastConversationUserIndex(prompt);
return index < 0 ? "" : JSON.stringify(prompt[index].content);
for (let index = (body.prompt || []).length - 1; index >= 0; index -= 1) {
const message = body.prompt[index];
if (message.role === "user") return JSON.stringify(message.content);
}
return "";
}
function requireResult(body, id, expected) {
sdk/tests/test-term.mjs
+1 -4
View File
@@ -184,10 +184,7 @@ if (!(secondRequestAt < streamFinishedAt)) throw new Error("terminal waited for
if (postSubmitText.includes(`┋ ${liveDraft}`)) throw new Error("terminal exposed tool-only pending UI during immediate steering");
if (!postSubmitText.includes(liveDraft)) throw new Error("terminal did not commit the steering user row after cutoff");
if (!postSubmitText.includes("Thinking")) throw new Error("terminal hid activity during immediate steering");
// The steering directive is the last user message carrying <user_steering>;
// volatile runtime context rides the message tail after it.
const steeringUser = secondRequestBody.prompt?.filter((message) => message.role === "user")
.findLast((message) => JSON.stringify(message.content ?? []).includes("<user_steering>"));
const steeringUser = secondRequestBody.prompt?.filter((message) => message.role === "user").at(-1);
const steeringText = steeringUser?.content?.filter((part) => part.type === "text").map((part) => part.text);
const steeringRequest = JSON.stringify(secondRequestBody.prompt);
if (
src/builtins/context.zig
+6 -6
View File
@@ -2994,15 +2994,15 @@ fn appendTransient(input: TransientContextInput, arena: Allocator, messages: *st
"{s}\nRuntime context: this is a noninteractive run without live question UI; when a user-owned decision remains after inspection, stop and surface a concrete blocker in freeform text with the available options. Do not recommend or label one option as preferred.",
.{turn_context},
);
try messages.append(arena, .{ .role = .user, .content = content });
try messages.append(arena, .{ .role = .system, .content = content });
try appendWorkspaceAccessContext(input.access_scope, arena, messages);
try messages.append(arena, .{ .role = .user, .content = permissionModeContext(input.permission_mode) });
try messages.append(arena, .{ .role = .system, .content = permissionModeContext(input.permission_mode) });
if (input.stale_shell_handles) try messages.append(arena, .{
.role = .user,
.role = .system,
.content = stale_shell_handles_context,
});
if (input.interactive) try messages.append(arena, .{
.role = .user,
.role = .system,
.content = "Runtime context: if this turn changes files, choose focused verification from the touched areas first. Use changed paths in tool calls and results to select checks; avoid generic or expensive verification unless those paths justify it or the user requested it. Tests under tests/evals can be deterministic; do not assume they require live models. Preserve exact verification evidence in the final summary.",
});
}
@@ -3030,7 +3030,7 @@ fn appendWorkspaceAccessContext(
try model_context_encoding.writeScalar(&note.writer, entry.path);
try note.writer.writeByte('\n');
}
try messages.append(arena, .{ .role = .user, .content = try note.toOwnedSlice() });
try messages.append(arena, .{ .role = .system, .content = try note.toOwnedSlice() });
}
const PromptContextFixture = struct {
@@ -3108,7 +3108,7 @@ test "runtime context composes exact auto mode with noninteractive blockers" {
try expectContains(messages.items[0].content.?, "Do not recommend or label one option as preferred");
try expectNotContains(messages.items[0].content.?, "ask_user_question");
try std.testing.expectEqual(@as(usize, 2), messages.items.len);
try std.testing.expectEqual(types.ChatRole.user, messages.items[1].role);
try std.testing.expectEqual(types.ChatRole.system, messages.items[1].role);
try std.testing.expectEqualStrings(
"Runtime context: permission mode is auto. After configured rules, session grants, and deterministic safe-tool authority, fx sends each unresolved action to a narrow safety reviewer. A clear result authorizes only that exact action. A caution or unavailable result holds only that action and returns advice without opening a permission screen, disabling tools, or ending the turn. Exact cautions are reused for this turn; choose a materially different safe action or explain why no safe path remains. Tool admission and exact live revalidation remain authoritative.",
messages.items[1].content.?,
src/core/agent/runtime/orchestrator.zig
+16 -22
View File
@@ -2848,7 +2848,7 @@ fn appendPreparedParentTurnContext(
if (prepared.content.len == 0) return .{};
try messages.ensureUnusedCapacity(arena, 1);
messages.appendAssumeCapacity(.{
.role = .user,
.role = .system,
.content = prepared.content,
});
return .{ .acknowledgements = prepared.acknowledgements };
@@ -3472,19 +3472,19 @@ fn build_provider_prompt_with_response_language_control(
compaction_history_tail: []const ChatMessage,
compacted_suffix_len: usize,
) !ProviderPromptProjection {
const effective_prefix = if (enforce_response_language and origin == .root) blk: {
const projected = try alloc.alloc(ChatMessage, stable_prefix.len + 1);
@memcpy(projected[0..stable_prefix.len], stable_prefix);
projected[stable_prefix.len] = .{
const effective_overlay = if (enforce_response_language and origin == .root) blk: {
const projected = try alloc.alloc(ChatMessage, ephemeral_overlay.len + 1);
@memcpy(projected[0..ephemeral_overlay.len], ephemeral_overlay);
projected[ephemeral_overlay.len] = .{
.role = .system,
.content = response_language_control,
};
break :blk projected;
} else stable_prefix;
} else ephemeral_overlay;
var prompt = try buildProviderPromptForCompactionWindow(
alloc,
effective_prefix,
ephemeral_overlay,
stable_prefix,
effective_overlay,
durable_history,
current_user_message,
within_turn_suffix,
@@ -3513,7 +3513,7 @@ test "compacted request keeps the pending user prompt after the handoff" {
var projected = try build_provider_prompt_with_response_language_control(
std.testing.allocator,
&.{.{ .role = .system, .content = "stable" }},
&.{.{ .role = .user, .content = "overlay" }},
&.{.{ .role = .system, .content = "overlay" }},
&.{.{ .role = .user, .content = "removed history" }},
.{ .role = .user, .content = "pending prompt" },
&.{
@@ -3530,14 +3530,14 @@ test "compacted request keeps the pending user prompt after the handoff" {
defer projected.instructions.deinit(std.testing.allocator);
defer projected.messages.deinit(std.testing.allocator);
try std.testing.expectEqual(@as(usize, 1), projected.instructions.items.len);
try std.testing.expectEqual(@as(usize, 2), projected.instructions.items.len);
try std.testing.expectEqualStrings("stable", projected.instructions.items[0].content.?);
try std.testing.expectEqualStrings("overlay", projected.instructions.items[1].content.?);
const expected = [_][]const u8{
"handoff",
"retained tail",
"pending prompt",
"new suffix",
"overlay",
};
try std.testing.expectEqual(expected.len, projected.messages.items.len);
for (expected, projected.messages.items) |content, message| {
@@ -4310,13 +4310,10 @@ test "context overflow recovery is typed safe and bounded" {
fn isPostVisionAssistantPrefillRejection(
status: std.http.Status,
detail: []const u8,
conversation_tail: ?ChatMessage,
messages: []const ChatMessage,
) bool {
if (status != .bad_request) return false;
// Judge the semantic conversation tail, not the request tail: the per-step
// runtime overlay rides at the end of the message list, after the tool
// result this recovery is looking for.
const tail = conversation_tail orelse return false;
if (status != .bad_request or messages.len == 0) return false;
const tail = messages[messages.len - 1];
if (tail.role != .tool or
!std.mem.eql(u8, tail.tool_name orelse return false, "vision"))
{
@@ -6949,7 +6946,7 @@ fn processQueuedPromptLoop(
}
var ephemeral_overlay: std.ArrayList(ChatMessage) = .empty;
if (skills.explicit) |section| {
if (section.text.len > 0) try ephemeral_overlay.append(overlay_arena, .{ .role = .user, .content = section.text });
if (section.text.len > 0) try ephemeral_overlay.append(overlay_arena, .{ .role = .system, .content = section.text });
}
try deps.append_runtime_context(deps.ctx, overlay_arena, &ephemeral_overlay);
var parent_turn_delivery = try appendPreparedParentTurnContext(
@@ -8244,10 +8241,7 @@ fn processQueuedPromptLoop(
isPostVisionAssistantPrefillRejection(
if (response_failure) |failure| failureHttpStatus(failure.kind) else .ok,
if (response_failure) |failure| failure.detail orelse "" else "",
if (within_turn_suffix.items.len > 0)
within_turn_suffix.items[within_turn_suffix.items.len - 1]
else
null,
request_messages,
))
{
try within_turn_suffix.append(arena, .{
src/core/agent/runtime/prompt_context.zig
+8 -16
View File
@@ -472,15 +472,10 @@ pub fn buildProviderPrompt(
errdefer messages.deinit(alloc);
try instructions.appendSlice(alloc, stable_prefix);
try instructions.appendSlice(alloc, ephemeral_overlay);
try messages.appendSlice(alloc, durable_history);
try messages.append(alloc, current_user_message);
try messages.appendSlice(alloc, within_turn_suffix);
// The overlay carries per-step volatile runtime context (date, git state,
// permission mode, subagent deliveries). Provider prompt caching reuses the
// longest unchanged leading span, so the overlay rides at the very end of
// the message list: a mid-turn change then discards only the overlay
// instead of invalidating the entire conversation history behind it.
try messages.appendSlice(alloc, ephemeral_overlay);
return .{
.instructions = instructions,
.messages = messages,
@@ -494,7 +489,7 @@ test "buildProviderPrompt separates instructions from chronological messages" {
.{ .role = .system, .content = "stable project context" },
};
const overlay = [_]ChatMessage{
.{ .role = .user, .content = "volatile runtime overlay" },
.{ .role = .system, .content = "volatile runtime overlay" },
};
const history = [_]ChatMessage{
.{ .role = .user, .content = "history user prompt" },
@@ -508,17 +503,15 @@ test "buildProviderPrompt separates instructions from chronological messages" {
var prompt = try buildProviderPrompt(alloc, &stable_prefix, &overlay, &history, current, &suffix);
defer prompt.deinit(alloc);
try std.testing.expectEqual(@as(usize, 2), prompt.instructions.items.len);
try std.testing.expectEqual(@as(usize, 5), prompt.messages.items.len);
try std.testing.expectEqual(@as(usize, 3), prompt.instructions.items.len);
try std.testing.expectEqual(@as(usize, 4), prompt.messages.items.len);
try std.testing.expectEqualStrings("stable system prompt", prompt.instructions.items[0].content.?);
try std.testing.expectEqualStrings("stable project context", prompt.instructions.items[1].content.?);
try std.testing.expectEqualStrings("volatile runtime overlay", prompt.instructions.items[2].content.?);
try std.testing.expectEqualStrings("history user prompt", prompt.messages.items[0].content.?);
try std.testing.expectEqualStrings("history assistant answer", prompt.messages.items[1].content.?);
try std.testing.expectEqualStrings("current user prompt", prompt.messages.items[2].content.?);
try std.testing.expectEqualStrings("within turn assistant", prompt.messages.items[3].content.?);
// The volatile overlay rides the message tail so a mid-turn change does
// not invalidate the cacheable prefix that covers all prior history.
try std.testing.expectEqualStrings("volatile runtime overlay", prompt.messages.items[4].content.?);
}
test "buildProviderPrompt keeps compacted session context out of instructions" {
@@ -587,7 +580,7 @@ test "buildProviderPrompt keeps compacted session context out of instructions" {
.{ .role = .system, .content = "stable system prompt" },
.{ .role = .system, .content = "stable project context" },
};
const overlay = [_]ChatMessage{.{ .role = .user, .content = "ephemeral overlay" }};
const overlay = [_]ChatMessage{.{ .role = .system, .content = "ephemeral overlay" }};
const current = ChatMessage{ .role = .user, .content = "current portable prompt" };
const suffix = [_]ChatMessage{.{ .role = .assistant, .content = "within-turn suffix" }};
var prompt = try buildProviderPrompt(
@@ -631,9 +624,8 @@ test "buildProviderPrompt keeps compacted session context out of instructions" {
try std.testing.expectEqual(@as(usize, 1), late_summary_count);
try std.testing.expectEqual(@as(usize, 1), file_evidence_count);
try std.testing.expectEqual(@as(usize, 1), interruption_count);
try std.testing.expectEqualStrings("current portable prompt", prompt.messages.items[prompt.messages.items.len - 3].content.?);
try std.testing.expectEqualStrings("within-turn suffix", prompt.messages.items[prompt.messages.items.len - 2].content.?);
try std.testing.expectEqualStrings("ephemeral overlay", prompt.messages.items[prompt.messages.items.len - 1].content.?);
try std.testing.expectEqualStrings("current portable prompt", prompt.messages.items[prompt.messages.items.len - 2].content.?);
try std.testing.expectEqualStrings("within-turn suffix", prompt.messages.items[prompt.messages.items.len - 1].content.?);
}
test "provider request measurement includes serialized structure" {
src/core/agent/runtime/tests/gateway_flow.zig
+21 -26
View File
@@ -690,10 +690,6 @@ test "processQueuedPrompt recovers when a model rejects post-Vision assistant pr
defer gateway.deinit();
var hooks = FakeAgentRuntimeDeps.init(alloc);
hooks.tool_registry = .{ .tools = test_support.vision_agent_test_tools[0..] };
// Keep the volatile runtime overlay present so the recovery classifier is
// exercised against the real request shape: the vision tool result is the
// semantic conversation tail, no longer the last request message.
hooks.runtime_context_text = "runtime context over vision tail";
defer hooks.deinit();
var vision_runtime = VisionAgentToolRuntime{ .alloc = alloc };
defer vision_runtime.deinit();
@@ -709,10 +705,13 @@ test "processQueuedPrompt recovers when a model rejects post-Vision assistant pr
try std.testing.expectEqual(@as(usize, 4), gateway.request_bodies.items.len);
try expectGatewayPromptTextCount(&gateway, 2, "FX logo", 1);
try expectGatewayPromptTailText(&gateway, 2, .user, "runtime context over vision tail");
try expectGatewayPromptTextCount(&gateway, 3, "Continue from the preceding tool result.", 1);
try expectBodyContainsInOrder(&gateway, 3, &.{ "FX logo", "Continue from the preceding tool result.", "runtime context over vision tail" });
try expectGatewayPromptTailText(&gateway, 3, .user, "runtime context over vision tail");
try expectGatewayPromptTailText(&gateway, 2, .tool, "FX logo");
try expectGatewayPromptTailText(
&gateway,
3,
.user,
"Continue from the preceding tool result.",
);
try std.testing.expectEqual(@as(?std.http.Status, null), hooks.http_status);
try std.testing.expectEqualStrings("Recovered final answer", hooks.finish_assistant_text.?);
}
@@ -4599,7 +4598,7 @@ test "processQueuedPrompt enables Gateway automatic caching across model familie
}
}
test "processQueuedPrompt places transient overlay after history and current prompt" {
test "processQueuedPrompt places transient overlay before history and current prompt" {
const alloc = std.testing.allocator;
const completions = [_]FakeCompletion{.{ .content = "No" }};
var gateway = FakeGateway.init(alloc, &completions);
@@ -4628,12 +4627,10 @@ test "processQueuedPrompt places transient overlay after history and current pro
const current_idx = std.mem.indexOf(u8, body, "is it still running") orelse return error.TestExpectedEqual;
const runtime_idx = std.mem.indexOf(u8, body, "runtime tail context unique") orelse return error.TestExpectedEqual;
try std.testing.expect(system_idx < static_idx);
try std.testing.expect(static_idx < history_idx);
try std.testing.expect(static_idx < runtime_idx);
try std.testing.expect(runtime_idx < history_idx);
try std.testing.expect(history_idx < current_idx);
// The volatile runtime context trails the current prompt: a mid-turn
// refresh then only replaces the tail instead of the history prefix.
try std.testing.expect(current_idx < runtime_idx);
try expectGatewayPromptFinalUserText(&gateway, 0, "runtime tail context unique");
try expectGatewayPromptFinalUserText(&gateway, 0, "is it still running");
try std.testing.expectEqual(@as(usize, 0), hooks.executed_names.items.len);
}
@@ -5014,8 +5011,8 @@ test "processQueuedPrompt keeps supplied system prompt components in stable orde
try runFakePrompt(&gateway, &hooks, config, job);
try std.testing.expectEqual(@as(usize, 2), gateway.request_bodies.items.len);
const first_roles = [_]types.ChatRole{ .system, .system, .system, .system, .system, .system, .system, .user, .assistant, .user, .user };
const second_roles = [_]types.ChatRole{ .system, .system, .system, .system, .system, .system, .system, .user, .assistant, .user, .assistant, .tool, .user };
const first_roles = [_]types.ChatRole{ .system, .system, .system, .system, .system, .system, .system, .system, .user, .assistant, .user };
const second_roles = [_]types.ChatRole{ .system, .system, .system, .system, .system, .system, .system, .system, .user, .assistant, .user, .assistant, .tool };
try expectGatewayPromptRoles(&gateway, 0, &first_roles);
try expectGatewayPromptRoles(&gateway, 1, &second_roles);
inline for (&.{ @as(usize, 0), @as(usize, 1) }) |request_index| {
@@ -5034,9 +5031,9 @@ test "processQueuedPrompt keeps supplied system prompt components in stable orde
"host guidance-order instructions",
"model guidance-order overlay",
"static guidance-order context",
"transient guidance-order context",
"past assistant guidance-order needle",
"user prompt",
"transient guidance-order context",
};
try expectBodyContainsInOrder(&gateway, request_index, &order);
}
@@ -5094,11 +5091,11 @@ test "processQueuedPrompt refreshes runtime overlay each step and preserves turn
try expectBodyContains(&gateway, 1, "Checking.");
try expectBodyContains(&gateway, 1, "\"toolName\":\"read_file\"");
try expectBodyContains(&gateway, 1, "\"value\":\"ok\"");
const first_request_roles = [_]types.ChatRole{ .system, .system, .user, .user };
const second_request_roles = [_]types.ChatRole{ .system, .system, .user, .assistant, .tool, .user };
const first_request_roles = [_]types.ChatRole{ .system, .system, .system, .user };
const second_request_roles = [_]types.ChatRole{ .system, .system, .system, .user, .assistant, .tool };
try expectGatewayPromptRoles(&gateway, 0, &first_request_roles);
try expectGatewayPromptRoles(&gateway, 1, &second_request_roles);
const second_request_order = [_][]const u8{ "user prompt", "Checking.", "\"value\":\"ok\"", "runtime overlay step two" };
const second_request_order = [_][]const u8{ "runtime overlay step two", "user prompt", "Checking.", "\"value\":\"ok\"" };
try expectBodyContainsInOrder(&gateway, 1, &second_request_order);
}
@@ -5290,7 +5287,7 @@ test "processQueuedPrompt projects history exactly once into each gateway reques
}
}
test "processQueuedPrompt keeps completed history before the current user prompt" {
test "processQueuedPrompt keeps completed history before the final current user prompt" {
const alloc = std.testing.allocator;
var history = [_]HistoryTurn{.{ .assistant = .{
.user = .{ .text = @constCast("prior user structural needle") },
@@ -5309,7 +5306,7 @@ test "processQueuedPrompt keeps completed history before the current user prompt
try runFakePrompt(&gateway, &hooks, fixture.config(), job);
const expected_roles = [_]types.ChatRole{ .system, .system, .user, .assistant, .user, .user };
const expected_roles = [_]types.ChatRole{ .system, .system, .system, .user, .assistant, .user };
try expectGatewayPromptRoles(&gateway, 0, &expected_roles);
try expectGatewayPromptTextCount(&gateway, 0, "prior user structural needle", 1);
try expectGatewayPromptTextCount(&gateway, 0, "prior assistant structural needle", 1);
@@ -5319,11 +5316,9 @@ test "processQueuedPrompt keeps completed history before the current user prompt
const history_idx = std.mem.indexOf(u8, body, "prior assistant structural needle") orelse return error.TestExpectedEqual;
const runtime_idx = std.mem.indexOf(u8, body, "runtime context structural needle") orelse return error.TestExpectedEqual;
const current_idx = std.mem.indexOf(u8, body, "current structural prompt needle") orelse return error.TestExpectedEqual;
// The volatile runtime context rides the message tail so a mid-turn
// change keeps the whole history cacheable behind an unchanged prefix.
try std.testing.expect(runtime_idx < history_idx);
try std.testing.expect(history_idx < current_idx);
try std.testing.expect(current_idx < runtime_idx);
try expectGatewayPromptFinalUserText(&gateway, 0, "runtime context structural needle");
try expectGatewayPromptFinalUserText(&gateway, 0, "current structural prompt needle");
try expectGatewayPromptTextCount(&gateway, 0, "Earlier messages are session history from previous turns.", 0);
try expectGatewayPromptTextCount(&gateway, 0, "Do not re-run, re-answer, or continue earlier user turns", 0);
}
src/core/agent/runtime/tests/interruption_flow.zig
+2 -2
View File
@@ -121,8 +121,8 @@ test "processQueuedPrompt sends former intent text normally with tools" {
const body = gateway.request_bodies.items[0];
const runtime_idx = std.mem.indexOf(u8, body, "runtime context unique") orelse return error.TestExpectedEqual;
const current_idx = std.mem.indexOf(u8, body, text) orelse return error.TestExpectedEqual;
try std.testing.expect(current_idx < runtime_idx);
try expectGatewayPromptFinalUserText(&gateway, 0, "runtime context unique");
try std.testing.expect(runtime_idx < current_idx);
try expectGatewayPromptFinalUserText(&gateway, 0, text);
try expectBodyContains(&gateway, 0, "\"name\":\"read_file\"");
try expectBodyNotContains(&gateway, 0, removed_direct_question_guidance);
try expectBodyNotContains(&gateway, 0, removed_resume_guidance);
src/core/agent/runtime/tests/support.zig
+2 -2
View File
@@ -1026,9 +1026,9 @@ pub const FakeAgentRuntimeDeps = struct {
const index = @min(self.runtime_context_index, self.runtime_context_texts.len - 1);
const text = self.runtime_context_texts[index];
self.runtime_context_index += 1;
try messages.append(arena, .{ .role = .user, .content = try arena.dupe(u8, text) });
try messages.append(arena, .{ .role = .system, .content = try arena.dupe(u8, text) });
} else if (self.runtime_context_text) |text| {
try messages.append(arena, .{ .role = .user, .content = try arena.dupe(u8, text) });
try messages.append(arena, .{ .role = .system, .content = try arena.dupe(u8, text) });
}
}
src/core/app/app_agent_runtime.zig
+1 -1
View File
@@ -1547,7 +1547,7 @@ fn appendTestTransientContext(input: context_contract.TransientContextInput, all
"provider transient:{s}:{s}",
.{ input.workspace_root, @tagName(input.permission_mode) },
);
try messages.append(alloc, .{ .role = .user, .content = content });
try messages.append(alloc, .{ .role = .system, .content = content });
}
const test_context_provider = context_contract.Provider{
src/core/workspace/context_contract.zig
+24 -24
View File
@@ -556,31 +556,31 @@ pub fn writeEntrypointLayoutSnapshot(writer: *std.Io.Writer) !void {
\\model_visible_layout:
\\- entrypoint: interactive
\\ static_context_refresh: one applicable snapshot before enqueue; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and history
\\ intentional_difference: live terminal approvals and clarification UI
\\- entrypoint: fx ask
\\ static_context_refresh: one applicable snapshot before the prompt; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and empty or saved history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and empty or saved history
\\ intentional_difference: approval-required actions become noninteractive blockers
\\- entrypoint: ACP
\\ static_context_refresh: one applicable snapshot per ACP prompt including accepted local resource targets; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and ACP session history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and ACP session history
\\ intentional_difference: ACP protocol maps prompts, refusals, and updates to JSON-RPC session messages
\\- entrypoint: subagent
\\ static_context_refresh: the launching surface's snapshot with empty child delivery state; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and canonical child session history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and canonical child session history
\\ intentional_difference: separate child session history, parent-projected approvals, and bounded parent/child delivery
\\
);
@@ -714,31 +714,31 @@ test "entrypoint model-visible layout snapshot covers major entrypoints" {
\\model_visible_layout:
\\- entrypoint: interactive
\\ static_context_refresh: one applicable snapshot before enqueue; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and history
\\ intentional_difference: live terminal approvals and clarification UI
\\- entrypoint: fx ask
\\ static_context_refresh: one applicable snapshot before the prompt; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and empty or saved history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and empty or saved history
\\ intentional_difference: approval-required actions become noninteractive blockers
\\- entrypoint: ACP
\\ static_context_refresh: one applicable snapshot per ACP prompt including accepted local resource targets; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and ACP session history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and ACP session history
\\ intentional_difference: ACP protocol maps prompts, refusals, and updates to JSON-RPC session messages
\\- entrypoint: subagent
\\ static_context_refresh: the launching surface's snapshot with empty child delivery state; scoped deltas attach before affected tool execution
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog
\\ stable_prefix_initial_order: system_prompt, effective_custom_tool_guidance, visible_skills, optional_model_prompt_overlay, optional_interruption_or_resume_intent_context, shared_project_context, mcp_server_catalog, optional_prepared_parent_turn_delivery_context
\\ stable_prefix_later_additions: applicable_project_context_deltas committed mid-turn when a tool batch has applicable targets
\\ per_step_tail_overlay_order: explicit_skill_chunks, transient_runtime_context, optional_prepared_parent_turn_delivery_context
\\ user_prompt_position: after stable system context and canonical child session history; per-step volatile tail overlay rides the message tail as user-role context so mid-turn changes never invalidate the cached conversation prefix
\\ per_step_overlay_order: explicit_skill_chunks, transient_runtime_context
\\ user_prompt_position: after stable system context and canonical child session history
\\ intentional_difference: separate child session history, parent-projected approvals, and bounded parent/child delivery
\\
,
tests/e2e/acp.test.ts
+1 -2
View File
@@ -21,7 +21,6 @@ import {
AUTO_EXA_SERIALIZED_TOOL_NAMES,
customProviderGuidanceState,
findUnavailableCapabilityReferences,
lastConversationUserIndex,
parseGatewayRequest,
serializedToolNames,
toolShapesWithoutDescriptions,
@@ -2386,7 +2385,7 @@ describe("acp: model-independent", () => {
prompt: Array<{ role: string; content: Array<{ type: string; text?: string }> }>;
};
expect(request.maxOutputTokens).toBe(64_000);
const user = request.prompt[lastConversationUserIndex(request.prompt)];
const user = request.prompt.findLast((message) => message.role === "user");
expect(user?.content.find((part) => part.type === "text")?.text).toBe(submitted);
expect(client.stderr).toBe("");
} finally {
tests/e2e/ci-shard-weights.json
-1
View File
@@ -1,5 +1,4 @@
[
{ "file": "prompt-cache-stability.test.ts", "weight": 2 },
{ "file": "auto-mode-reliability.test.ts", "weight": 4 },
{ "file": "review-model-override.test.ts", "weight": 4 },
{ "file": "acp.test.ts", "weight": 29 },
tests/e2e/cli.test.ts
+1 -4
View File
@@ -32,7 +32,6 @@ import {
fakeGatewaySse,
startFakeGateway,
} from "./tmux-helpers";
import { lastConversationUserIndex } from "./conditional-guidance-oracle";
const TIMEOUT = 15_000;
const NO_GATEWAY_AUTH = {
@@ -4065,9 +4064,7 @@ describe("cli: ask success", () => {
const request = JSON.parse(gateway.requests[index]!.body) as {
prompt: Array<{ role: string; content: Array<{ type: string; text?: string }> }>;
};
// The volatile runtime context rides the message tail; the submitted
// prompt is the last non-overlay user message.
const user = request.prompt[lastConversationUserIndex(request.prompt)];
const user = request.prompt.findLast((message) => message.role === "user");
expect(user?.content.find((part) => part.type === "text")?.text).toBe(prompt);
}
tests/e2e/compaction-policy.test.ts
+2 -3
View File
@@ -6,7 +6,6 @@ import { join, resolve } from "node:path";
process.env.FX_E2E_DISABLE_DOTENV = "1";
const { fakeGatewayFinalText, startDynamicFakeGateway } = await import("./tmux-helpers");
const { isRuntimeOverlayMessage, lastConversationUserIndex } = await import("./conditional-guidance-oracle");
const binary = resolve(import.meta.dir, "../../zig-out/bin/fx");
const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
@@ -26,7 +25,7 @@ for (const userHeavy of [false, true]) test(`automatic compaction preserves task
bodies.push(body);
if (request.tools?.length === 0 && request.toolChoice?.type === "none") {
summaryCalls++;
const sourceMessages = request.prompt.filter((message: { role: string }) => message.role === "user" && !isRuntimeOverlayMessage(message));
const sourceMessages = request.prompt.filter((message: { role: string }) => message.role === "user");
expect(sourceMessages).toHaveLength(1);
expect(sourceMessages[0].content).toHaveLength(1);
const source = sourceMessages[0].content[0].text;
@@ -66,7 +65,7 @@ for (const userHeavy of [false, true]) test(`automatic compaction preserves task
const seed = await ask([], "seed", originalUser);
expect(summaryCalls).toBe(0);
const seededRequest = JSON.parse(bodies[0]!);
const seededUser = seededRequest.prompt[lastConversationUserIndex(seededRequest.prompt)];
const seededUser = seededRequest.prompt.findLast((message: { role: string }) => message.role === "user");
expect(seededUser.content[0].text).toBe(originalUser);
const sessionDir = join(home, ".fx/sessions", seed.session_id), log = join(sessionDir, "events.jsonl"), before = readFileSync(log);
phase = "continue";
tests/e2e/conditional-guidance-oracle.ts
-39
View File
@@ -110,45 +110,6 @@ export function contentText(content: unknown): string {
return "";
}
// Per-step volatile runtime context (turn context, permission mode, subagent
// deliveries) rides the message tail as user-role entries so a mid-turn change
// never invalidates the cached conversation prefix. Use these helpers to skip
// that overlay when locating conversation user messages in captured requests.
export function isRuntimeOverlayMessage(message: { role?: unknown; content?: unknown }): boolean {
if (message.role !== "user") return false;
const text = contentText(message.content);
return text.includes("<fx-turn-context>") ||
text.startsWith("Runtime context:") ||
text.startsWith("Subagent results (untrusted tool output") ||
text.startsWith("Explicitly invoked skill content for this query:");
}
export function lastConversationUserIndex(prompt: Array<{ role?: unknown; content?: unknown }>): number {
for (let i = prompt.length - 1; i >= 0; i--) {
const message = prompt[i]!;
if (message.role === "user" && !isRuntimeOverlayMessage(message)) return i;
}
return -1;
}
export function lastConversationUserText(prompt: Array<{ role?: unknown; content?: unknown }>): string {
const index = lastConversationUserIndex(prompt);
return index < 0 ? "" : contentText(prompt[index]!.content);
}
export function stripRuntimeOverlay<T extends { role?: unknown; content?: unknown }>(prompt: T[]): T[] {
return prompt.filter((message) => !isRuntimeOverlayMessage(message));
}
// For protocols that serialize content as plain strings (chat completions),
// strip only the contiguous trailing overlay run so conversation order checks
// like at(-2) keep working against the semantic tail.
export function stripTrailingRuntimeOverlay<T extends { role?: unknown; content?: unknown }>(messages: T[]): T[] {
let end = messages.length;
while (end > 0 && isRuntimeOverlayMessage(messages[end - 1]!)) end--;
return messages.slice(0, end);
}
export function canonicalToolName(name: string): string {
if (
name === "exa_search" ||
tests/e2e/configured-providers.test.ts
+8 -9
View File
@@ -3,7 +3,6 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { FX_BIN, runFx } from "../evals/eval-helpers";
import { completion, toolCompletion, createConfiguredProviderFixture as fixture } from "./fixtures/chat-completions";
import { stripTrailingRuntimeOverlay as withoutTailOverlay } from "./conditional-guidance-oracle";
async function withReasoning(response: Response, ...deltas: Record<string, unknown>[]) {
const prefix = deltas.map(delta => `data: ${JSON.stringify({ choices: [{ index: 0, delta }] })}\n\n`).join("");
@@ -28,7 +27,7 @@ describe("configured providers", () => {
const toolTurn = f.requests[1].body.messages.find((message: any) => message.role === "assistant" && message.tool_calls?.length);
expect(toolTurn[field]).toBe("Inspect note first.");
expect(toolTurn._tool_call_ids).toBeUndefined();
expect(withoutTailOverlay(f.requests[1].body.messages).at(-1).content).toContain("fixture contents");
expect(f.requests[1].body.messages.at(-1).content).toContain("fixture contents");
const resumed = await runFx(["ask", "--json", "--resume", saved.session_id, "Continue the saved task"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
if (resumed.code !== 0) throw new Error(resumed.stdout + resumed.stderr);
expect(JSON.parse(resumed.stdout).output).toBe("resumed reply");
@@ -54,7 +53,7 @@ describe("configured providers", () => {
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
expect(JSON.parse(result.stdout).output).toBe("The two files were read.");
expect(f.requests).toHaveLength(4);
const messages = withoutTailOverlay(f.requests[3].body.messages);
const messages = f.requests[3].body.messages;
expect(messages.at(-1).content).toBe("Summarize what you just did.");
expect(messages.at(-2).reasoning_content).toBe("Both files have been read.");
expect(messages.at(-2).tool_calls).toBeUndefined();
@@ -82,7 +81,7 @@ describe("configured providers", () => {
expect(toolTurn.reasoning_details).toEqual([firstDetail, secondDetail]);
expect(toolTurn.tool_calls[0].id).toBe(callId);
expect(toolTurn._tool_call_ids).toBeUndefined();
expect(withoutTailOverlay(f.requests[1].body.messages).at(-1).tool_call_id).toBe(callId);
expect(f.requests[1].body.messages.at(-1).tool_call_id).toBe(callId);
const resumed = await runFx(["ask", "--json", "--resume", saved.session_id, "Continue the saved task"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
if (resumed.code !== 0) throw new Error(resumed.stdout + resumed.stderr);
expect(f.requests).toHaveLength(3);
@@ -254,7 +253,7 @@ describe("configured providers", () => {
let callId = 0;
const f = fixture(body => {
if (!body.tools?.some((tool: any) => tool.function?.name === "subagent")) return completion(body.model, "child reply");
if (withoutTailOverlay(body.messages).at(-1)?.role === "tool") return completion(body.model, "parent reply");
if (body.messages.at(-1)?.role === "tool") return completion(body.model, "parent reply");
return toolCompletion(body.model, "subagent", { request: { action: "message", agent: "reader", message: "read the child fixture" } }, `child-call-${++callId}`);
});
f.settings.models.local = "shared-model";
@@ -269,7 +268,7 @@ describe("configured providers", () => {
expect(f.requests.map(request => request.body.max_tokens)).toEqual([1024, 1024, 1024]);
const resumed = await runFx(["ask", "--json", "--resume", id, "Continue the named child"], { cwd: f.workspace, env: { ...f.env, FX_PROVIDER: "local" }, timeoutMs: 30000 });
if (resumed.code !== 0) throw new Error(resumed.stdout + resumed.stderr);
const resumedResult = withoutTailOverlay(f.requests.at(-1)!.body.messages).at(-1).content;
const resumedResult = f.requests.at(-1)!.body.messages.at(-1).content;
if (!JSON.parse(resumedResult).ok) throw new Error(resumedResult);
expect(f.requests.map(request => request.body.max_tokens)).toEqual([1024, 1024, 1024, 512, 1024, 512]);
expect(f.requests[3].authorization).toBeNull();
@@ -281,7 +280,7 @@ describe("configured providers", () => {
if (rebound.code !== 0) throw new Error(rebound.stdout + rebound.stderr);
expect(f.requests).toHaveLength(8);
expect(f.requests.slice(6).every(request => request.authorization === null && request.path === "/v1/chat/completions")).toBe(true);
expect(withoutTailOverlay(f.requests[7].body.messages).at(-1).content).toContain("child_failed");
expect(f.requests[7].body.messages.at(-1).content).toContain("child_failed");
} finally { f.close(); }
}, 100000);
@@ -296,7 +295,7 @@ describe("configured providers", () => {
}
return completion(body.model, "child reply", Math.ceil(JSON.stringify(body).length / 4));
}
if (withoutTailOverlay(body.messages).at(-1)?.role === "tool") return completion(body.model, "parent reply");
if (body.messages.at(-1)?.role === "tool") return completion(body.model, "parent reply");
parentTurns++;
return toolCompletion(body.model, "subagent", { request: { action: "message", agent: "reader", message: `child context ${parentTurns} ` + "detail ".repeat(2000), ...(parentTurns === 1 ? { model: "child-model" } : {}) } }, `context-call-${parentTurns}`);
});
@@ -308,7 +307,7 @@ describe("configured providers", () => {
const result = await runFx(["ask", "--json", ...(id ? ["--resume", id] : []), "Continue the child context"], { cwd: f.workspace, env: f.env, timeoutMs: 20000 });
if (result.code !== 0) throw new Error(result.stdout + result.stderr);
id = JSON.parse(result.stdout).session_id;
const returned = withoutTailOverlay(f.requests.filter(request => request.body.model === "local-model").at(-1)!.body.messages).at(-1).content;
const returned = f.requests.filter(request => request.body.model === "local-model").at(-1)!.body.messages.at(-1).content;
if (!JSON.parse(returned).ok) throw new Error(returned);
expect(JSON.parse(returned).ok).toBe(true);
}
tests/e2e/gateway-stream-lifecycle.test.ts
+9 -14
View File
@@ -23,12 +23,8 @@ import {
AUTO_EXA_WITHOUT_DURABLE_TOOLS_SERIALIZED_TOOL_NAMES,
customProviderGuidanceState,
findUnavailableCapabilityReferences,
isRuntimeOverlayMessage,
lastConversationUserIndex,
lastConversationUserText,
parseGatewayRequest,
serializedToolNames,
stripRuntimeOverlay,
toolByName,
toolShapesWithoutDescriptions,
WEB_SEARCH_GUIDANCE,
@@ -88,7 +84,7 @@ for (const action of ["run", "message"] as const) for (const stop of [false, tru
writeFileSync(join(root.workspace, "after-child.txt"), "AFTER_CHILD_TOOL_OK");
const gateway = startDynamicFakeGateway(raw => {
const body = JSON.parse(raw);
const latest = JSON.stringify(body.prompt?.[lastConversationUserIndex(body.prompt ?? [])]?.content);
const latest = JSON.stringify(body.prompt?.filter((item: any) => item.role === "user").at(-1)?.content);
if (latest.includes("STEERING_CHILD")) {
childRequests++;
return held.response;
@@ -102,7 +98,7 @@ for (const action of ["run", "message"] as const) for (const stop of [false, tru
}
if (latest.includes("STEERING_LATER")) return fakeGatewayFinalText("LATER_OK");
if (raw.includes("HELD_CHILD_RESULT")) {
expect(JSON.stringify(body.prompt.filter((item: any) => item.role === "user" && !isRuntimeOverlayMessage(item)))).not.toContain("HELD_CHILD_RESULT");
expect(JSON.stringify(body.prompt.filter((item: any) => item.role === "user"))).not.toContain("HELD_CHILD_RESULT");
if (!afterChildTool) {
afterChildTool = true;
return fakeGatewayToolCall("after-child", "read_file", { path: "after-child.txt" });
@@ -626,7 +622,7 @@ function shellResult(body: string, callId: string): ShellResult {
function hasCurrentToolResult(body: string, callId: string): boolean {
const prompt = gatewayRequest(body).prompt;
const lastUserIndex = lastConversationUserIndex(prompt);
const lastUserIndex = prompt.findLastIndex((message) => message.role === "user");
return prompt.slice(lastUserIndex + 1).some((message) =>
Array.isArray(message.content) &&
(message.content as Array<Record<string, unknown>>).some((part) =>
@@ -3039,8 +3035,7 @@ describe("gateway stream lifecycle", () => {
expect(turnIndex).toBeGreaterThan(rulesIndex);
expect(first.prompt[rulesIndex].role).toBe("system");
expect(first.prompt[rulesIndex].providerOptions).toBeUndefined();
// The volatile turn context rides the message tail as user-role content.
expect(first.prompt[turnIndex].role).toBe("user");
expect(first.prompt[turnIndex].role).toBe("system");
expect(first.prompt[turnIndex].providerOptions).toBeUndefined();
expect(firstText).toContain(
"dynamic-context&lt;workspace&gt;&#x0a;injected_workspace",
@@ -3092,7 +3087,7 @@ describe("gateway stream lifecycle", () => {
expect(followupText).not.toContain("\ninjected_location");
expect(followupText).not.toContain("\ninjected_file");
expect(followupText).not.toContain("<injected>description</injected>");
expect(lastConversationUserText(first.prompt)).toContain(
expect(contentText(first.prompt.at(-1)?.content)).toContain(
"Inspect the dynamic context fixture.",
);
@@ -6278,7 +6273,7 @@ printf '%s' ${JSON.stringify(trailingMarker)} > ${JSON.stringify(effectPath)}
prompt: Array<{ role: string; content: unknown }>;
};
const userTexts = request.prompt
.filter((message) => message.role === "user" && !isRuntimeOverlayMessage(message))
.filter((message) => message.role === "user")
.map((message) => contentText(message.content));
expect(userTexts.at(-1)).toBe("compaction restart probe");
expect(userTexts.some((text) => text.includes("context_handoff"))).toBe(
@@ -7986,7 +7981,7 @@ printf '%s' ${JSON.stringify(trailingMarker)} > ${JSON.stringify(effectPath)}
let childRequests = 0;
const gateway = startDynamicFakeGateway((body) => {
const prompt = gatewayRequest(body).prompt;
const lastUser = lastConversationUserIndex(prompt);
const lastUser = prompt.findLastIndex((message) => message.role === "user");
const parts = prompt.slice(lastUser + 1).flatMap((message) => Array.isArray(message.content) ? message.content : []) as Array<Record<string, unknown>>;
const result = parts.find((part) => part.type === "tool-result" && part.toolCallId === "replayed_delegation");
if (result) {
@@ -9385,7 +9380,7 @@ printf '%s' ${JSON.stringify(trailingMarker)} > ${JSON.stringify(effectPath)}
expect(assistants).toHaveLength(2);
expect(assistants.map((message: { content: Array<{ type: string; toolCallId: string }> }) => message.content.map((part) => [part.type, part.toolCallId])))
.toEqual([[["tool-call", "read_1"]], [["tool-call", "read_2"]]]);
expect(stripRuntimeOverlay(request.prompt).at(-1)).toEqual({
expect(request.prompt.at(-1)).toEqual({
role: "user",
content: [{ type: "text", text: "Summarize what you just did." }],
});
@@ -9888,7 +9883,7 @@ printf '%s' ${JSON.stringify(trailingMarker)} > ${JSON.stringify(effectPath)}
expect(json.tool_calls).toEqual([{ name: "read_file", status: "success" }]);
expect(gateway.requestCount()).toBe(3);
expectOnlyLeadingSystemMessages(gateway.requests[2]!.body);
expect(stripRuntimeOverlay(retryPrompt).at(-1)?.role).toBe("tool");
expect(retryPrompt.at(-1)?.role).toBe("tool");
expect(toolResultOutput(gateway.requests[2]!.body, "read_retry_order")).toContain(
"deterministic fixture",
);
tests/e2e/permission-mode-context.ts
+2 -8
View File
@@ -15,11 +15,7 @@ export function expectPermissionModeContext(
};
const messages = request.prompt.map((message) => ({
role: message.role,
text: typeof message.content === "string"
? message.content
: Array.isArray(message.content)
? message.content.map((part: { text?: string }) => part?.text ?? "").join("")
: "",
text: typeof message.content === "string" ? message.content : "",
}));
const expected = permissionModeContext[mode];
const matching = messages.filter((message) => message.text === expected);
@@ -29,7 +25,5 @@ export function expectPermissionModeContext(
if (candidateMode === mode) continue;
expect(messages.some((message) => message.text === context)).toBe(false);
}
// Runtime context rides the message tail as user-role content so mid-turn
// changes preserve the cached conversation prefix.
expect(matching[0]!.role).toBe("user");
expect(matching[0]!.role).toBe("system");
}
tests/e2e/prompt-cache-stability.test.ts
-156
View File
@@ -1,156 +0,0 @@
import { afterEach, expect, test } from "bun:test";
import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { runFx } from "../evals/eval-helpers";
import { contentText, isRuntimeOverlayMessage } from "./conditional-guidance-oracle";
import { fakeGatewayFinalText, fakeGatewayToolCall, startDynamicFakeGateway } from "./tmux-helpers";
// Prompt-cache stability contract for gateway models. fx advertises
// providerOptions.gateway.caching="auto"; provider-side caching reuses the
// longest unchanged leading span of the request, so per-step volatile runtime
// context must ride at the END of the message list. If it ever moves back
// into the leading instruction prefix, a mid-turn change (like the git
// worktree flipping dirty after fx edits a file) silently invalidates the
// entire conversation history cache and every later step re-processes it.
const MODEL = "openai/gpt-5";
const TIMEOUT = 30_000;
const roots: string[] = [];
const gateways: Array<{ stop(): void }> = [];
afterEach(() => {
for (const gateway of gateways.splice(0)) gateway.stop();
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});
function createGitWorkspace() {
const root = realpathSync(mkdtempSync(join(tmpdir(), "fx-prompt-cache-e2e-")));
const home = join(root, "home");
const workspace = join(root, "workspace");
mkdirSync(join(home, ".fx"), { recursive: true });
mkdirSync(workspace, { recursive: true });
writeFileSync(join(home, ".fx", "settings.json"), "{}");
// Keep the index at or below 32 entries so fx's bounded dirty check is
// active: this is the path where an fx edit flips git_worktree to dirty.
writeFileSync(join(workspace, "tracked.txt"), "ORIGINAL_CONTENT\n");
execFileSync("git", ["init", "-q"], { cwd: workspace });
execFileSync("git", ["config", "user.email", "fx-e2e@example.com"], { cwd: workspace });
execFileSync("git", ["config", "user.name", "fx-e2e"], { cwd: workspace });
execFileSync("git", ["add", "tracked.txt"], { cwd: workspace });
execFileSync("git", ["commit", "-qm", "init"], { cwd: workspace });
roots.push(root);
return { home, workspace: realpathSync(workspace) };
}
type CapturedMessage = { role: string; content?: unknown };
type CapturedRequest = {
prompt: CapturedMessage[];
tools?: unknown;
providerOptions?: { gateway?: { caching?: string } };
};
function messageKey(message: CapturedMessage): string {
return JSON.stringify(message);
}
function messageText(message: CapturedMessage): string {
return contentText(message.content);
}
const isRuntimeContextMessage = isRuntimeOverlayMessage;
// The previous request with its trailing runtime-context messages removed
// must be an exact leading prefix of the next request: only the volatile tail
// and newly appended step messages may differ.
function expectCachePrefixPreserved(
previous: CapturedMessage[],
current: CapturedMessage[],
) {
let tail = previous.length;
while (tail > 0 && isRuntimeContextMessage(previous[tail - 1])) tail--;
const overlayCount = previous.length - tail;
expect(overlayCount).toBeGreaterThan(0);
for (const message of previous.slice(tail)) {
expect(message.role).toBe("user");
}
const stablePrefix = previous.slice(0, tail).map(messageKey);
const currentHead = current.slice(0, tail).map(messageKey);
expect(currentHead).toEqual(stablePrefix);
// Runtime context stays out of the leading system instruction block.
expect(current.slice(0, tail).some(isRuntimeContextMessage)).toBe(false);
}
async function runEditTurn(): Promise<CapturedRequest[]> {
const fixture = createGitWorkspace();
const agentRequests: CapturedRequest[] = [];
const gateway = startDynamicFakeGateway((raw) => {
const body = JSON.parse(raw);
if (Array.isArray(body.prompt) && body.prompt.length > 2) {
agentRequests.push(body as CapturedRequest);
}
const toolMessages = (raw.match(/"role":"tool"/g) ?? []).length;
if (toolMessages === 0) {
return fakeGatewayToolCall("call_write", "write_file", {
path: "tracked.txt",
content: "WROTE_TRACKED_CONTENT\n",
});
}
if (toolMessages === 1) {
return fakeGatewayToolCall("call_read", "read_file", { path: "tracked.txt" });
}
return fakeGatewayFinalText("done");
});
gateways.push(gateway);
const result = await runFx(["ask", "Update tracked.txt then read it back"], {
cwd: fixture.workspace,
timeoutMs: TIMEOUT,
env: {
HOME: fixture.home,
AI_GATEWAY_API_KEY: "fake-prompt-cache-key",
VERCEL_OIDC_TOKEN: undefined,
FX_GATEWAY_BASE_URL: gateway.baseUrl,
FX_GATEWAY_CHAT_URL: gateway.chatUrl,
FX_MODEL: MODEL,
FX_PERMISSION_MODE: "full-access",
FX_AUTO_UPGRADE: "0",
FX_SOUND: "0",
NO_COLOR: "1",
},
});
expect(result.code).toBe(0);
expect(agentRequests.length).toBeGreaterThanOrEqual(3);
return agentRequests;
}
test("gateway requests advertise auto prompt caching with byte-stable tools", async () => {
const requests = await runEditTurn();
for (const request of requests) {
expect(request.providerOptions?.gateway?.caching).toBe("auto");
}
const toolJson = requests.map((request) => JSON.stringify(request.tools ?? []));
for (const tools of toolJson.slice(1)) {
expect(tools).toBe(toolJson[0]);
}
});
test("mid-turn git dirty flip preserves the cached history prefix", async () => {
const requests = await runEditTurn();
// The fixture's write_file must actually flip the worktree state, otherwise
// the prefix assertions below are vacuous.
const sawDirtyFlip = requests.some((request, index) => {
if (index === 0) return false;
const before = requests[index - 1].prompt.map(messageText).join("\n");
const after = request.prompt.map(messageText).join("\n");
return before.includes("git_worktree: unknown") && after.includes("git_worktree: dirty");
});
expect(sawDirtyFlip).toBe(true);
for (let i = 1; i < requests.length; i++) {
expectCachePrefixPreserved(requests[i - 1].prompt, requests[i].prompt);
}
});
tests/e2e/tui-command-permissions.test.ts
+1 -2
View File
@@ -31,7 +31,6 @@ import {
TmuxSession,
tmuxAvailable,
} from "./tmux-helpers";
import { lastConversationUserIndex } from "./conditional-guidance-oracle";
const TIMEOUT = 30_000;
const MODEL = "openai/gpt-5";
@@ -327,7 +326,7 @@ function currentUserText(body: string): string {
prompt?: Array<{ role?: string; content?: unknown }>;
};
return contentText(
request.prompt?.[lastConversationUserIndex(request.prompt ?? [])]?.content,
request.prompt?.findLast((message) => message.role === "user")?.content,
);
}
tests/e2e/tui-composer-edit-contracts.test.ts
+1 -2
View File
@@ -22,7 +22,6 @@ import {
TmuxSession,
tmuxAvailable,
} from "./tmux-helpers";
import { lastConversationUserIndex } from "./conditional-guidance-oracle";
const HAS_TMUX = tmuxAvailable();
const tmuxTest = test.skipIf(!HAS_TMUX);
@@ -212,7 +211,7 @@ function userParts(requestIndex = 0): Array<{
content: Array<{ type: string; text?: string }>;
}>;
};
const message = body.prompt[lastConversationUserIndex(body.prompt)];
const message = body.prompt.findLast((entry) => entry.role === "user");
return message?.content ?? [];
}
tests/e2e/tui-file-picker.test.ts
+2 -3
View File
@@ -23,7 +23,6 @@ import {
TmuxSession,
tmuxAvailable,
} from "./tmux-helpers";
import { lastConversationUserIndex } from "./conditional-guidance-oracle";
const HAS_TMUX = tmuxAvailable();
const tmuxTest = test.skipIf(!HAS_TMUX);
@@ -1859,7 +1858,7 @@ describe("@ file picker", () => {
await active.waitForStableComposer(TIMEOUT);
expect(gateway?.requests).toHaveLength(1);
const request = JSON.parse(gateway!.requests[0]!.body);
expect(request.prompt[lastConversationUserIndex(request.prompt)]).toEqual({
expect(request.prompt.at(-1)).toEqual({
role: "user",
content: [{ type: "text", text: "@~/home-target.txt Reply with the words SHORTCUT, FLOW, and OK joined by underscores." }],
});
@@ -1988,7 +1987,7 @@ describe("@ file picker", () => {
const request = JSON.parse(gateway!.requests[0]!.body) as {
prompt: Array<{ role: string; content: unknown }>;
};
expect(request.prompt[lastConversationUserIndex(request.prompt)]).toEqual({
expect(request.prompt.at(-1)).toEqual({
role: "user",
content: [{
type: "text",
tests/e2e/tui-gateway-stream-lifecycle.test.ts
+10 -12
View File
@@ -21,8 +21,6 @@ import {
AUTO_EXA_SERIALIZED_TOOL_NAMES,
customProviderGuidanceState,
findUnavailableCapabilityReferences,
lastConversationUserIndex,
lastConversationUserText,
parseGatewayRequest,
serializedToolNames,
toolShapesWithoutDescriptions,
@@ -80,7 +78,7 @@ for (const { cancelBeforeConsumption, lateFeedback } of [
const request = JSON.parse(raw), child = !raw.includes('"name":"subagent"');
if (child) {
childCalls++;
const latest = lastConversationUserText(request.prompt);
const latest = contentText(request.prompt.findLast((m: any) => m.role === "user")?.content);
if (lateFeedback && latest.includes("CHILD_FEEDBACK_TOKEN")) {
expect(raw).toContain("CHILD_STEERING_ORIGINAL_DONE");
followupDone = true;
@@ -114,7 +112,7 @@ for (const { cancelBeforeConsumption, lateFeedback } of [
expect(raw).toContain("CHILD_STEERING_ORIGINAL_DONE");
return fakeGatewayFinalText("CHILD_STEERING_ALL_DONE");
}
const latestParent = lastConversationUserText(request.prompt);
const latestParent = contentText(request.prompt.findLast((m: any) => m.role === "user")?.content);
if ((raw.includes("CHILD_STEERING_ORIGINAL_DONE") || latestParent.includes("FOLLOWUP_AFTER_CANCELLATION")) && !followup) {
followup = true; return fakeGatewayToolCall("child-steering-follow", "subagent",
{ request: { action: "message", agent: "reviewer", message: "FOLLOW_SAME_CHILD: retain the original work and feedback." } });
@@ -4151,9 +4149,9 @@ describe.skipIf(!tmuxAvailable())("TUI gateway stream lifecycle", () => {
const continuedRequest = JSON.parse(continuedBody) as {
prompt: Array<{ role?: string; content?: unknown }>;
};
const continuedUser = continuedRequest.prompt[
lastConversationUserIndex(continuedRequest.prompt)
];
const continuedUser = continuedRequest.prompt.filter((message) =>
message.role === "user"
).at(-1);
expect(continuedUser).toBeDefined();
expect(Array.isArray(continuedUser!.content)).toBe(true);
const continuedParts = continuedUser!.content as Array<Record<string, unknown>>;
@@ -4445,9 +4443,9 @@ describe.skipIf(!tmuxAvailable())("TUI gateway stream lifecycle", () => {
const steeringRequest = JSON.parse(steeringBody) as {
prompt: Array<{ role?: string; content?: unknown }>;
};
const steeringUser = steeringRequest.prompt[
lastConversationUserIndex(steeringRequest.prompt)
];
const steeringUser = steeringRequest.prompt.filter((message) =>
message.role === "user"
).at(-1);
expect(steeringUser).toBeDefined();
expect(Array.isArray(steeringUser!.content)).toBe(true);
const steeringParts = steeringUser!.content as Array<Record<string, unknown>>;
@@ -5631,7 +5629,7 @@ describe.skipIf(!tmuxAvailable())("TUI gateway stream lifecycle", () => {
const request = JSON.parse(body) as {
prompt: Array<{ role: string; content: unknown }>;
};
const user = lastConversationUserText(request.prompt);
const user = contentText(request.prompt.findLast((message) => message.role === "user")?.content);
const stage = user.match(/HANDOFF_(SEED|CANCEL|NEXT|AGAIN)/)?.[1];
if (!stage) throw new Error("missing handoff request stage");
if (user.startsWith("CHILD_HANDOFF_")) {
@@ -5737,7 +5735,7 @@ describe.skipIf(!tmuxAvailable())("TUI gateway stream lifecycle", () => {
const childGates = tasks.map(() => new Promise<void>((resolve) => releases.push(resolve)));
const rowGateway = startDynamicFakeGateway(async (body) => {
const request = JSON.parse(body) as { prompt?: Array<{ role?: string; content?: unknown }> };
const userText = lastConversationUserText(request.prompt ?? []);
const userText = contentText(request.prompt?.findLast((message) => message.role === "user")?.content);
const child = tasks.findIndex((task) => userText.includes(task));
if (child >= 0 && !userText.includes(rootPrompt)) {
await childGates[child];
tests/e2e/tui-input-navigation.test.ts
+2 -3
View File
@@ -20,7 +20,6 @@ import {
TmuxSession,
tmuxAvailable,
} from "./tmux-helpers";
import { lastConversationUserIndex } from "./conditional-guidance-oracle";
import {
assertPaneContains,
assertSingleFooter,
@@ -587,7 +586,7 @@ tmuxTest(
role: string;
content: Array<{ type: string; text?: string }>;
}>;
const finalUser = messages[lastConversationUserIndex(messages)];
const finalUser = messages[messages.length - 1];
expect(finalUser?.role).toBe("user");
expect(finalUser?.content[0]?.text).toBe(prompt);
@@ -1732,7 +1731,7 @@ tmuxTest(
role: string;
content: Array<{ type: string; text?: string }>;
}>;
const user = request[lastConversationUserIndex(request)];
const user = request.findLast((message) => message.role === "user");
expect(user?.content[0]?.text).toBe(submission);
const transcript = await active.capturePaneEscapes();
tests/e2e/tui-interrupt-recovery.test.ts
+1 -2
View File
@@ -24,7 +24,6 @@ import {
TmuxSession,
tmuxAvailable,
} from "./tmux-helpers";
import { isRuntimeOverlayMessage } from "./conditional-guidance-oracle";
const SKIP = !tmuxAvailable();
const TIMEOUT = 30_000;
@@ -591,7 +590,7 @@ describe.skipIf(SKIP)("tui: interrupt recovery", () => {
);
expect(
followUpRequest.prompt
.filter((entry) => entry.role !== "system" && !isRuntimeOverlayMessage(entry))
.filter((entry) => entry.role !== "system")
.map((entry) => entry.role),
).toEqual([
"user",
tests/e2e/tui-resize.test.ts
+4 -4
View File
@@ -31,7 +31,6 @@ import {
tmuxAvailable,
tmuxRawPasteFlags,
} from "./tmux-helpers";
import { lastConversationUserText } from "./conditional-guidance-oracle";
import {
findActiveFileApprovalBlock,
findFooterBlocks,
@@ -876,9 +875,10 @@ async function waitForSubmittedUserText(
content?: Array<{ type: string; text?: string }>;
}>;
};
// The volatile runtime context rides the message tail; the submitted prompt
// is the last conversation user message before it.
return lastConversationUserText(request.prompt) || undefined;
return [...request.prompt]
.reverse()
.find((message) => message.role === "user")
?.content?.find((part) => part.type === "text")?.text;
}
async function waitForGatewayRequestCount(
tests/e2e/tui-resume.test.ts
+1 -2
View File
@@ -34,7 +34,6 @@ import {
TmuxSession,
tmuxAvailable,
} from "./tmux-helpers";
import { lastConversationUserIndex } from "./conditional-guidance-oracle";
import { readTapeFrames } from "./render-lab/tape";
const TIMEOUT = 30_000;
@@ -2804,7 +2803,7 @@ test.skipIf(!tmuxAvailable())(
role: string;
content: Array<{ type: string; text?: string }>;
}>;
const finalUser = messages[lastConversationUserIndex(messages)];
const finalUser = messages[messages.length - 1];
expect(finalUser?.role).toBe("user");
expect(finalUser?.content[0]?.text).toBe(fullViewDraft);
const afterSubmit = await active.capturePane();
tests/e2e/vision-route-fake-gateway.test.ts
+3 -4
View File
@@ -21,7 +21,6 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { FX_BIN, REPO_ROOT, runFx } from "../evals/eval-helpers";
import { fakeGatewaySse, fakeGatewayTitleDefault, hasEmptyComposer, TITLE_GENERATION_MARKER, TmuxSession, tmuxAvailable } from "./tmux-helpers";
import { stripRuntimeOverlay } from "./conditional-guidance-oracle";
const TIMEOUT = 15_000;
const GLM_MODEL = "zai/glm-5.2-fast";
@@ -542,7 +541,7 @@ function textFromContent(content: unknown): string {
function lastUserText(body: string): string {
const parsed = JSON.parse(body) as { prompt: Array<{ role?: string; content?: unknown }> };
const users = stripRuntimeOverlay(parsed.prompt.filter((entry) => entry.role === "user"));
const users = parsed.prompt.filter((entry) => entry.role === "user");
expect(users.length).toBeGreaterThan(0);
return textFromContent(users[users.length - 1].content);
}
@@ -779,8 +778,8 @@ describe("Vision route fake Gateway", () => {
expect(gateway.chatRequests).toHaveLength(4);
const rejectedPrompt = JSON.parse(gateway.chatRequests[2].body).prompt;
expect(stripRuntimeOverlay(rejectedPrompt).at(-1)).toMatchObject({ role: "tool" });
const retryPrompt = stripRuntimeOverlay(JSON.parse(gateway.chatRequests[3].body).prompt);
expect(rejectedPrompt.at(-1)).toMatchObject({ role: "tool" });
const retryPrompt = JSON.parse(gateway.chatRequests[3].body).prompt;
expect(retryPrompt.at(-1)).toMatchObject({ role: "user" });
expect(JSON.stringify(retryPrompt.at(-1))).toContain(
"Continue from the preceding tool result.",