From ca6e11ff10a8f6ba4dfe55b144df4e8643c25425 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 17 Sep 2026 11:25:35 -0400 Subject: [PATCH] Route Jev reviews through the gateway evaluation endpoint by default When review_model selects typesafeai/jev without TYPESAFE_API_KEY, reviews now post to the gateway's /v4/ai/evaluation-model endpoint with the session gateway credential and the typesafe-ai/jev catalog id. Setting TYPESAFE_API_KEY keeps talking to TypeSafe's API directly. A failing evaluation endpoint holds actions unexecuted like any unavailable reviewer. --- README.md | 2 +- .../gateway/typesafe_permission_reviewer.zig | 155 ++++++++++++++++-- tests/e2e/review-model-override.test.ts | 54 +++++- tests/e2e/tmux-helpers.ts | 30 ++++ 4 files changed, 222 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 838f12085..2e1aa1c7a 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Inside the shell, run `/help` to browse interactive commands. In auto mode, a valid structured safety decision remains usable when the reviewer adds commentary. If the response has no valid decision, fx retries the review once within its original 30-second deadline. A safety caution is never retried for approval. If review still fails, the action stays unexecuted and the agent can continue with other tools. -By default, reviews are answered by a language model through your configured connection. The `review_model` setting in `~/.fx/settings.json` (or the `FX_REVIEW_MODEL` environment variable, which wins) changes the reviewer model. Any AI Gateway chat model id works and keeps the same review protocol. Setting it to `typesafeai/jev` routes each review to TypeSafe's System One API instead, which returns a typed decision with calibrated probabilities and a confidence score; that path requires a `TYPESAFE_API_KEY` in the environment. Without a working reviewer, actions that need review stay unexecuted rather than falling back silently. The review policy, the action under review, and the hold-or-authorize behavior are identical either way. +By default, reviews are answered by a language model through your configured connection. The `review_model` setting in `~/.fx/settings.json` (or the `FX_REVIEW_MODEL` environment variable, which wins) changes the reviewer model. Any AI Gateway chat model id works and keeps the same review protocol. Setting it to `typesafeai/jev` routes each review to TypeSafe's Jev evaluation model instead, which returns a typed decision with calibrated probabilities and a confidence score. Jev is reached through the gateway's evaluation endpoint with your existing gateway credential; setting `TYPESAFE_API_KEY` talks to TypeSafe's API directly instead. Without a working reviewer, actions that need review stay unexecuted rather than falling back silently. The review policy, the action under review, and the hold-or-authorize behavior are identical either way. ### Long conversations diff --git a/src/builtins/gateway/typesafe_permission_reviewer.zig b/src/builtins/gateway/typesafe_permission_reviewer.zig index 9b2b5e89d..a86773519 100644 --- a/src/builtins/gateway/typesafe_permission_reviewer.zig +++ b/src/builtins/gateway/typesafe_permission_reviewer.zig @@ -18,6 +18,10 @@ const Allocator = std.mem.Allocator; const default_endpoint = "https://api.typesafe.ai/v1/systemone"; const jev_model = "jev-latest"; +/// Gateway catalog id and evaluation-endpoint model slug. +const gateway_model_id = "typesafe-ai/jev"; +const gateway_eval_suffix = "/v4/ai/evaluation-model"; +const gateway_chat_suffix = "/v4/ai/language-model"; pub const review_model_id = "typesafeai/jev"; const decision_instructions = @@ -37,8 +41,53 @@ const Config = struct { api_key: []const u8, endpoint: []const u8, cancel_flag: ?*std.atomic.Value(bool) = null, + /// True when the request goes to the gateway evaluation-model endpoint + /// with gateway credentials and protocol headers instead of TypeSafe direct. + via_gateway: bool = false, }; +const Route = struct { + api_key: []const u8, + endpoint: []const u8, + via_gateway: bool, + model: []const u8, +}; + +/// Writes the evaluation-model sibling of a gateway chat URL into out and +/// returns it. Returns null when the URL lacks the language-model suffix or +/// out is too small. +fn gatewayEvalEndpoint(chat_url: []const u8, out: []u8) ?[]const u8 { + if (!std.mem.endsWith(u8, chat_url, gateway_chat_suffix)) return null; + const prefix_len = chat_url.len - gateway_chat_suffix.len; + const total = prefix_len + gateway_eval_suffix.len; + if (total > out.len) return null; + @memcpy(out[0..prefix_len], chat_url[0..prefix_len]); + @memcpy(out[prefix_len..total], gateway_eval_suffix); + return out[0..total]; +} + +/// Picks the Jev transport: TypeSafe direct when TYPESAFE_API_KEY is set, +/// otherwise the gateway evaluation-model endpoint with the gateway +/// credential. eval_endpoint is scratch storage for the derived URL. +fn resolveRoute(direct_api_key: ?[]const u8, direct_base_url: ?[]const u8, gateway_credential: []const u8, gateway_chat_url: []const u8, eval_endpoint: []u8) ?Route { + if (direct_api_key) |key| { + if (std.mem.trim(u8, key, " \t\r\n").len > 0) return .{ + .api_key = key, + .endpoint = direct_base_url orelse default_endpoint, + .via_gateway = false, + .model = jev_model, + }; + } + if (gateway_credential.len == 0) return null; + const endpoint = gatewayEvalEndpoint(gateway_chat_url, eval_endpoint) orelse return null; + return .{ + .api_key = gateway_credential, + .endpoint = endpoint, + .via_gateway = true, + .model = gateway_model_id, + }; +} + /// Whether a resolved review-model id selects the TypeSafe Jev reviewer. /// Accepts the gateway catalog spelling as an alias. pub fn isJevModelId(model: []const u8) bool { @@ -48,31 +97,40 @@ pub fn isJevModelId(model: []const u8) bool { } /// Provider-compatible entry point. Called from the builtin gateway reviewer -/// when isJevModelId() matches the resolved review model. A missing API key -/// degrades to an unconfigured transport outcome, which holds the action like -/// any unavailable review. +/// when isJevModelId() matches the resolved review model. TypeSafe direct when +/// TYPESAFE_API_KEY is set; otherwise the gateway evaluation-model endpoint +/// with the session's gateway credential. Missing credentials degrade to an +/// unconfigured transport outcome, which holds the action like any +/// unavailable review. pub fn review( _: ?*anyopaque, alloc: Allocator, input: permission_auto_classifier.ProviderInput, request: permission_auto_classifier.ReviewRequest, ) anyerror!permission_auto_classifier.ParseOutcome { - const api_key = io_mod.getenv("TYPESAFE_API_KEY") orelse { - debug_trace.logf("permission", "event=auto_review_typesafe result=permanent_failure reason=missing_typesafe_api_key", .{}); + var eval_endpoint_buf: [4096]u8 = undefined; + const route = resolveRoute( + io_mod.getenv("TYPESAFE_API_KEY"), + io_mod.getenv("TYPESAFE_BASE_URL"), + input.credential, + input.endpoint, + &eval_endpoint_buf, + ) orelse { + debug_trace.logf("permission", "event=auto_review_typesafe result=permanent_failure reason=missing_reviewer_credential", .{}); return .{ .invalid = .transport_unconfigured }; }; - const endpoint = io_mod.getenv("TYPESAFE_BASE_URL") orelse default_endpoint; var config = Config{ - .api_key = api_key, - .endpoint = endpoint, + .api_key = route.api_key, + .endpoint = route.endpoint, .cancel_flag = input.cancel_flag, + .via_gateway = route.via_gateway, }; - debug_trace.logf("permission", "event=auto_review_typesafe_selected endpoint={s}", .{config.endpoint}); + debug_trace.logf("permission", "event=auto_review_typesafe_selected endpoint={s} via_gateway={}", .{ config.endpoint, route.via_gateway }); return permission_auto_classifier.Reviewer.withTransportModel(.{ .context = @ptrCast(&config), .build_fn = buildReviewBody, .send_fn = sendReview, - }, input.cancel_flag, permission_auto_classifier.Reviewer.default_timeout_ms, jev_model).review(alloc, request); + }, input.cancel_flag, permission_auto_classifier.Reviewer.default_timeout_ms, route.model).review(alloc, request); } /// Composes the System One request body from the reviewer's already-composed @@ -263,12 +321,17 @@ fn sendReview( const auth_header = std.fmt.allocPrint(alloc, "Bearer {s}", .{config.api_key}) catch |err| return err; defer alloc.free(auth_header); + const gateway_extra = [_]std.http.Header{ + .{ .name = "ai-gateway-protocol-version", .value = "0.0.1" }, + .{ .name = "ai-language-model-id", .value = gateway_model_id }, + }; + var client: std.http.Client = .{ .allocator = alloc, .io = io_mod.getIo() }; defer client.deinit(); var out: std.Io.Writer.Allocating = .init(alloc); defer out.deinit(); - debug_trace.logf("permission", "event=auto_review_typesafe_transport_start payload_bytes={d}", .{payload.len}); + debug_trace.logf("permission", "event=auto_review_typesafe_transport_start payload_bytes={d} via_gateway={}", .{ payload.len, config.via_gateway }); const result = client.fetch(.{ .location = .{ .url = config.endpoint }, .method = .POST, @@ -278,6 +341,7 @@ fn sendReview( .content_type = .{ .override = "application/json" }, .accept_encoding = .omit, }, + .extra_headers = if (config.via_gateway) &gateway_extra else &.{}, .response_writer = &out.writer, .redirect_behavior = .unhandled, }) catch |err| { @@ -436,22 +500,34 @@ const FakeJevServer = struct { server: std.Io.net.Server, url: []u8, mode: Mode, + path: []const u8 = "/v1/systemone", thread: ?std.Thread = null, failure: ?anyerror = null, + captured_head: [4096]u8 = undefined, + captured_len: usize = 0, fn init(mode: Mode) !FakeJevServer { + return initWithPath(mode, "/v1/systemone"); + } + + fn initWithPath(mode: Mode, path: []const u8) !FakeJevServer { var self = FakeJevServer{ .server = undefined, .url = undefined, .mode = mode, + .path = path, }; const address = try std.Io.net.IpAddress.parse("127.0.0.1", 0); self.server = try address.listen(self.io_backend.io(), .{ .reuse_address = true }); errdefer self.server.deinit(self.io_backend.io()); - self.url = try std.fmt.allocPrint(std.testing.allocator, "http://127.0.0.1:{d}/v1/systemone", .{self.server.socket.address.getPort()}); + self.url = try std.fmt.allocPrint(std.testing.allocator, "http://127.0.0.1:{d}{s}", .{ self.server.socket.address.getPort(), path }); return self; } + fn captured(self: *const FakeJevServer) []const u8 { + return self.captured_head[0..self.captured_len]; + } + fn start(self: *FakeJevServer) !void { std.debug.assert(self.thread == null); self.thread = try std.Thread.spawn(.{}, serveOne, .{self}); @@ -475,7 +551,7 @@ const FakeJevServer = struct { const zio = self.io_backend.io(); var stream = try self.server.accept(zio); defer stream.close(zio); - // Discard the request head and body. + // Read the request head (captured for assertions) and discard the body. var socket_buffer: [4096]u8 = undefined; var reader = stream.reader(zio, &socket_buffer); var head_bytes: usize = 0; @@ -485,6 +561,10 @@ const FakeJevServer = struct { error.EndOfStream => break, else => return err, }; + if (head_bytes < self.captured_head.len) { + self.captured_head[head_bytes] = byte; + self.captured_len = head_bytes + 1; + } head_bytes += 1; tail = .{ tail[1], tail[2], tail[3], byte }; if (std.mem.eql(u8, &tail, "\r\n\r\n")) break; @@ -513,3 +593,52 @@ test "isJevModelId matches canonical and gateway spellings only" { try std.testing.expect(!isJevModelId("openai/gpt-5.6-luna")); try std.testing.expect(!isJevModelId("")); } + +test "route resolution prefers TypeSafe direct and derives the gateway evaluation endpoint" { + var buf: [4096]u8 = undefined; + + const direct = resolveRoute("ts-key", null, "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?; + try std.testing.expect(!direct.via_gateway); + try std.testing.expectEqualStrings("ts-key", direct.api_key); + try std.testing.expectEqualStrings(default_endpoint, direct.endpoint); + try std.testing.expectEqualStrings(jev_model, direct.model); + + const custom_base = resolveRoute("ts-key", "http://127.0.0.1:1/v1/systemone", "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?; + try std.testing.expectEqualStrings("http://127.0.0.1:1/v1/systemone", custom_base.endpoint); + + const blank_direct = resolveRoute(" ", null, "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?; + try std.testing.expect(blank_direct.via_gateway); + + const gateway = resolveRoute(null, null, "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?; + try std.testing.expect(gateway.via_gateway); + try std.testing.expectEqualStrings("gw-key", gateway.api_key); + try std.testing.expectEqualStrings("https://ai-gateway.vercel.sh/v4/ai/evaluation-model", gateway.endpoint); + try std.testing.expectEqualStrings(gateway_model_id, gateway.model); + + try std.testing.expect(resolveRoute(null, null, "", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf) == null); + try std.testing.expect(resolveRoute(null, null, "gw-key", "https://example.test/chat", &buf) == null); +} + +test "gateway transport posts to the evaluation-model endpoint with protocol headers" { + const alloc = std.testing.allocator; + var server = try FakeJevServer.initWithPath(.ok, "/v4/ai/evaluation-model"); + defer server.deinit(); + try server.start(); + var cancel = std.atomic.Value(bool).init(false); + var config = Config{ .api_key = "gw-key", .endpoint = server.url, .via_gateway = true }; + const outcome = try sendReview(@ptrCast(&config), alloc, gateway_model_id, "{}", .fromNow(io_mod.getIo(), .{ .clock = .awake, .raw = .fromMilliseconds(5000) }), &cancel); + switch (outcome) { + .completion => |owned| { + var held = owned; + defer held.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), held.completion.tool_calls.len); + try std.testing.expect(std.mem.find(u8, held.completion.tool_calls[0].arguments_json, "\"decision\":\"clear\"") != null); + }, + else => return error.TestUnexpectedResult, + } + const head = server.captured(); + try std.testing.expect(std.mem.find(u8, head, "POST /v4/ai/evaluation-model HTTP") != null); + try std.testing.expect(std.mem.find(u8, head, "authorization: Bearer gw-key") != null); + try std.testing.expect(std.mem.find(u8, head, "ai-gateway-protocol-version: 0.0.1") != null); + try std.testing.expect(std.mem.find(u8, head, "ai-language-model-id: typesafe-ai/jev") != null); +} diff --git a/tests/e2e/review-model-override.test.ts b/tests/e2e/review-model-override.test.ts index 77b0155ba..4d6cb990a 100644 --- a/tests/e2e/review-model-override.test.ts +++ b/tests/e2e/review-model-override.test.ts @@ -279,13 +279,12 @@ describe("review model override", () => { ); test( - "override without TYPESAFE_API_KEY holds the action instead of executing unreviewed", + "without TYPESAFE_API_KEY the Jev review goes through the gateway evaluation endpoint", async () => { const root = createIsolatedRoot(); - const jev = startJevStub("clear"); const gateway = startFakeGateway([ fakeShellRun("cmd_1", `printf '%s' "${EXECUTED_MARKER}"`), - fakeGatewayFinalText("Review unavailable, holding."), + fakeGatewayFinalText("Ran the command."), ]); gateways.push(gateway); const result = await runFx( @@ -295,14 +294,59 @@ describe("review model override", () => { env: overrideEnv(root, gateway, { FX_REVIEW_MODEL: JEV_MODEL_ID, TYPESAFE_API_KEY: undefined, - TYPESAFE_BASE_URL: jev.url, + TYPESAFE_BASE_URL: undefined, + }), + timeoutMs: TIMEOUT, + }, + ); + expect(result.code).toBe(0); + expect(result.stdout).toContain(EXECUTED_MARKER); + expect(gateway.evaluationRequests).toHaveLength(1); + expect(gateway.classifierRequests).toHaveLength(0); + const review = JSON.parse(gateway.evaluationRequests[0].body); + expect(review.model).toBe("typesafe-ai/jev"); + expect(review.questions.decision.type).toBe("choice"); + expect( + gateway.evaluationRequests[0].headers.get("authorization"), + ).toBe("Bearer fake-review-model-override-key"); + expect( + gateway.evaluationRequests[0].headers.get("ai-language-model-id"), + ).toBe("typesafe-ai/jev"); + }, + TIMEOUT, + ); + + test( + "a failing Jev evaluation endpoint holds the action instead of executing unreviewed", + async () => { + const root = createIsolatedRoot(); + const gateway = startFakeGateway( + [ + fakeShellRun("cmd_1", `printf '%s' "${EXECUTED_MARKER}"`), + fakeGatewayFinalText("Review unavailable, holding."), + ], + { + evaluationResponse: new Response("upstream unavailable", { + status: 500, + }), + }, + ); + gateways.push(gateway); + const result = await runFx( + ["ask", "--auto", "--quiet", "--json", "Print the marker."], + { + cwd: root.workspace, + env: overrideEnv(root, gateway, { + FX_REVIEW_MODEL: JEV_MODEL_ID, + TYPESAFE_API_KEY: undefined, + TYPESAFE_BASE_URL: undefined, }), timeoutMs: TIMEOUT, }, ); expect(result.code).toBe(0); expect(result.stdout).not.toContain(EXECUTED_MARKER); - expect(jev.requests).toHaveLength(0); + expect(gateway.evaluationRequests.length).toBeGreaterThan(0); expect(gateway.classifierRequests).toHaveLength(0); }, TIMEOUT, diff --git a/tests/e2e/tmux-helpers.ts b/tests/e2e/tmux-helpers.ts index db3acd330..c6c092c8e 100644 --- a/tests/e2e/tmux-helpers.ts +++ b/tests/e2e/tmux-helpers.ts @@ -330,6 +330,9 @@ export type FakeGatewayOptions = { generationId: string, request: Request, ) => Response | Promise; + // Response for the /v4/ai/evaluation-model endpoint (TypeSafe Jev through + // the gateway). Defaults to a clear Jev decision. + evaluationResponse?: FakeGatewayResponse; }; // Session title generation calls carry this instruction regardless of the @@ -364,6 +367,7 @@ function serveFakeGateway( ) { const requests: Array<{ body: string; headers: Headers }> = []; const classifierRequests: Array<{ body: string; headers: Headers }> = []; + const evaluationRequests: Array<{ body: string; headers: Headers }> = []; const classifierResponses = [...(options.classifierResponses ?? [])]; const titleRequests: Array<{ body: string; headers: Headers }> = []; const titleResponses = [...(options.titleResponses ?? [])]; @@ -395,6 +399,31 @@ function serveFakeGateway( } return new Response("not found", { status: 404 }); } + if ( + req.method === "POST" && + new URL(req.url).pathname === "/v4/ai/evaluation-model" + ) { + evaluationRequests.push({ + body: await req.text(), + headers: new Headers(req.headers), + }); + const evaluationResponse = options.evaluationResponse ?? + Response.json({ + model: "jev-1.13.0", + answers: { + decision: { + type: "choice", + choice: "clear", + probabilities: { clear: 0.99, caution: 0.01 }, + confidence: 0.97, + }, + }, + usage: { input_tokens: 100, output_tokens: 10 }, + }); + return typeof evaluationResponse === "function" + ? await evaluationResponse(evaluationRequests[evaluationRequests.length - 1].body) + : evaluationResponse; + } if (req.method !== "POST") return new Response("not found", { status: 404 }); const body = await req.text(); const headers = new Headers(req.headers); @@ -419,6 +448,7 @@ function serveFakeGateway( chatUrl: `http://127.0.0.1:${server.port}/v4/ai/language-model`, requests, classifierRequests, + evaluationRequests, titleRequests, generationRequests, modelRequests,