Route Jev reviews through the gateway evaluation endpoint by default

When review_model selects typesafeai/jev without TYPESAFE_API_KEY, reviews now post to the gateway's /v4/ai/evaluation-model endpoint with the session gateway credential and the typesafe-ai/jev catalog id. Setting TYPESAFE_API_KEY keeps talking to TypeSafe's API directly. A failing evaluation endpoint holds actions unexecuted like any unavailable reviewer.

README.md

@@ -47,7 +47,7 @@ Inside the shell, run `/help` to browse interactive commands.
 
 In auto mode, a valid structured safety decision remains usable when the reviewer adds commentary. If the response has no valid decision, fx retries the review once within its original 30-second deadline. A safety caution is never retried for approval. If review still fails, the action stays unexecuted and the agent can continue with other tools.
 
-By default, reviews are answered by a language model through your configured connection. The `review_model` setting in `~/.fx/settings.json` (or the `FX_REVIEW_MODEL` environment variable, which wins) changes the reviewer model. Any AI Gateway chat model id works and keeps the same review protocol. Setting it to `typesafeai/jev` routes each review to TypeSafe's System One API instead, which returns a typed decision with calibrated probabilities and a confidence score; that path requires a `TYPESAFE_API_KEY` in the environment. Without a working reviewer, actions that need review stay unexecuted rather than falling back silently. The review policy, the action under review, and the hold-or-authorize behavior are identical either way.
+By default, reviews are answered by a language model through your configured connection. The `review_model` setting in `~/.fx/settings.json` (or the `FX_REVIEW_MODEL` environment variable, which wins) changes the reviewer model. Any AI Gateway chat model id works and keeps the same review protocol. Setting it to `typesafeai/jev` routes each review to TypeSafe's Jev evaluation model instead, which returns a typed decision with calibrated probabilities and a confidence score. Jev is reached through the gateway's evaluation endpoint with your existing gateway credential; setting `TYPESAFE_API_KEY` talks to TypeSafe's API directly instead. Without a working reviewer, actions that need review stay unexecuted rather than falling back silently. The review policy, the action under review, and the hold-or-authorize behavior are identical either way.
 
 ### Long conversations
 

src/builtins/gateway/typesafe_permission_reviewer.zig

@@ -18,6 +18,10 @@ const Allocator = std.mem.Allocator;
 
 const default_endpoint = "https://api.typesafe.ai/v1/systemone";
 const jev_model = "jev-latest";
+/// Gateway catalog id and evaluation-endpoint model slug.
+const gateway_model_id = "typesafe-ai/jev";
+const gateway_eval_suffix = "/v4/ai/evaluation-model";
+const gateway_chat_suffix = "/v4/ai/language-model";
 pub const review_model_id = "typesafeai/jev";
 
 const decision_instructions =
@@ -37,8 +41,53 @@ const Config = struct {
     api_key: []const u8,
     endpoint: []const u8,
     cancel_flag: ?*std.atomic.Value(bool) = null,
+    /// True when the request goes to the gateway evaluation-model endpoint
+    /// with gateway credentials and protocol headers instead of TypeSafe direct.
+    via_gateway: bool = false,
 };
 
+const Route = struct {
+    api_key: []const u8,
+    endpoint: []const u8,
+    via_gateway: bool,
+    model: []const u8,
+};
+
+/// Writes the evaluation-model sibling of a gateway chat URL into out and
+/// returns it. Returns null when the URL lacks the language-model suffix or
+/// out is too small.
+fn gatewayEvalEndpoint(chat_url: []const u8, out: []u8) ?[]const u8 {
+    if (!std.mem.endsWith(u8, chat_url, gateway_chat_suffix)) return null;
+    const prefix_len = chat_url.len - gateway_chat_suffix.len;
+    const total = prefix_len + gateway_eval_suffix.len;
+    if (total > out.len) return null;
+    @memcpy(out[0..prefix_len], chat_url[0..prefix_len]);
+    @memcpy(out[prefix_len..total], gateway_eval_suffix);
+    return out[0..total];
+}
+
+/// Picks the Jev transport: TypeSafe direct when TYPESAFE_API_KEY is set,
+/// otherwise the gateway evaluation-model endpoint with the gateway
+/// credential. eval_endpoint is scratch storage for the derived URL.
+fn resolveRoute(direct_api_key: ?[]const u8, direct_base_url: ?[]const u8, gateway_credential: []const u8, gateway_chat_url: []const u8, eval_endpoint: []u8) ?Route {
+    if (direct_api_key) |key| {
+        if (std.mem.trim(u8, key, " \t\r\n").len > 0) return .{
+            .api_key = key,
+            .endpoint = direct_base_url orelse default_endpoint,
+            .via_gateway = false,
+            .model = jev_model,
+        };
+    }
+    if (gateway_credential.len == 0) return null;
+    const endpoint = gatewayEvalEndpoint(gateway_chat_url, eval_endpoint) orelse return null;
+    return .{
+        .api_key = gateway_credential,
+        .endpoint = endpoint,
+        .via_gateway = true,
+        .model = gateway_model_id,
+    };
+}
+
 /// Whether a resolved review-model id selects the TypeSafe Jev reviewer.
 /// Accepts the gateway catalog spelling as an alias.
 pub fn isJevModelId(model: []const u8) bool {
@@ -48,31 +97,40 @@ pub fn isJevModelId(model: []const u8) bool {
 }
 
 /// Provider-compatible entry point. Called from the builtin gateway reviewer
-/// when isJevModelId() matches the resolved review model. A missing API key
-/// degrades to an unconfigured transport outcome, which holds the action like
-/// any unavailable review.
+/// when isJevModelId() matches the resolved review model. TypeSafe direct when
+/// TYPESAFE_API_KEY is set; otherwise the gateway evaluation-model endpoint
+/// with the session's gateway credential. Missing credentials degrade to an
+/// unconfigured transport outcome, which holds the action like any
+/// unavailable review.
 pub fn review(
     _: ?*anyopaque,
     alloc: Allocator,
     input: permission_auto_classifier.ProviderInput,
     request: permission_auto_classifier.ReviewRequest,
 ) anyerror!permission_auto_classifier.ParseOutcome {
-    const api_key = io_mod.getenv("TYPESAFE_API_KEY") orelse {
-        debug_trace.logf("permission", "event=auto_review_typesafe result=permanent_failure reason=missing_typesafe_api_key", .{});
+    var eval_endpoint_buf: [4096]u8 = undefined;
+    const route = resolveRoute(
+        io_mod.getenv("TYPESAFE_API_KEY"),
+        io_mod.getenv("TYPESAFE_BASE_URL"),
+        input.credential,
+        input.endpoint,
+        &eval_endpoint_buf,
+    ) orelse {
+        debug_trace.logf("permission", "event=auto_review_typesafe result=permanent_failure reason=missing_reviewer_credential", .{});
         return .{ .invalid = .transport_unconfigured };
     };
-    const endpoint = io_mod.getenv("TYPESAFE_BASE_URL") orelse default_endpoint;
     var config = Config{
-        .api_key = api_key,
-        .endpoint = endpoint,
+        .api_key = route.api_key,
+        .endpoint = route.endpoint,
         .cancel_flag = input.cancel_flag,
+        .via_gateway = route.via_gateway,
     };
-    debug_trace.logf("permission", "event=auto_review_typesafe_selected endpoint={s}", .{config.endpoint});
+    debug_trace.logf("permission", "event=auto_review_typesafe_selected endpoint={s} via_gateway={}", .{ config.endpoint, route.via_gateway });
     return permission_auto_classifier.Reviewer.withTransportModel(.{
         .context = @ptrCast(&config),
         .build_fn = buildReviewBody,
         .send_fn = sendReview,
-    }, input.cancel_flag, permission_auto_classifier.Reviewer.default_timeout_ms, jev_model).review(alloc, request);
+    }, input.cancel_flag, permission_auto_classifier.Reviewer.default_timeout_ms, route.model).review(alloc, request);
 }
 
 /// Composes the System One request body from the reviewer's already-composed
@@ -263,12 +321,17 @@ fn sendReview(
     const auth_header = std.fmt.allocPrint(alloc, "Bearer {s}", .{config.api_key}) catch |err| return err;
     defer alloc.free(auth_header);
 
+    const gateway_extra = [_]std.http.Header{
+        .{ .name = "ai-gateway-protocol-version", .value = "0.0.1" },
+        .{ .name = "ai-language-model-id", .value = gateway_model_id },
+    };
+
     var client: std.http.Client = .{ .allocator = alloc, .io = io_mod.getIo() };
     defer client.deinit();
     var out: std.Io.Writer.Allocating = .init(alloc);
     defer out.deinit();
 
-    debug_trace.logf("permission", "event=auto_review_typesafe_transport_start payload_bytes={d}", .{payload.len});
+    debug_trace.logf("permission", "event=auto_review_typesafe_transport_start payload_bytes={d} via_gateway={}", .{ payload.len, config.via_gateway });
     const result = client.fetch(.{
         .location = .{ .url = config.endpoint },
         .method = .POST,
@@ -278,6 +341,7 @@ fn sendReview(
             .content_type = .{ .override = "application/json" },
             .accept_encoding = .omit,
         },
+        .extra_headers = if (config.via_gateway) &gateway_extra else &.{},
         .response_writer = &out.writer,
         .redirect_behavior = .unhandled,
     }) catch |err| {
@@ -436,22 +500,34 @@ const FakeJevServer = struct {
     server: std.Io.net.Server,
     url: []u8,
     mode: Mode,
+    path: []const u8 = "/v1/systemone",
     thread: ?std.Thread = null,
     failure: ?anyerror = null,
+    captured_head: [4096]u8 = undefined,
+    captured_len: usize = 0,
 
     fn init(mode: Mode) !FakeJevServer {
+        return initWithPath(mode, "/v1/systemone");
+    }
+
+    fn initWithPath(mode: Mode, path: []const u8) !FakeJevServer {
         var self = FakeJevServer{
             .server = undefined,
             .url = undefined,
             .mode = mode,
+            .path = path,
         };
         const address = try std.Io.net.IpAddress.parse("127.0.0.1", 0);
         self.server = try address.listen(self.io_backend.io(), .{ .reuse_address = true });
         errdefer self.server.deinit(self.io_backend.io());
-        self.url = try std.fmt.allocPrint(std.testing.allocator, "http://127.0.0.1:{d}/v1/systemone", .{self.server.socket.address.getPort()});
+        self.url = try std.fmt.allocPrint(std.testing.allocator, "http://127.0.0.1:{d}{s}", .{ self.server.socket.address.getPort(), path });
         return self;
     }
 
+    fn captured(self: *const FakeJevServer) []const u8 {
+        return self.captured_head[0..self.captured_len];
+    }
+
     fn start(self: *FakeJevServer) !void {
         std.debug.assert(self.thread == null);
         self.thread = try std.Thread.spawn(.{}, serveOne, .{self});
@@ -475,7 +551,7 @@ const FakeJevServer = struct {
         const zio = self.io_backend.io();
         var stream = try self.server.accept(zio);
         defer stream.close(zio);
-        // Discard the request head and body.
+        // Read the request head (captured for assertions) and discard the body.
         var socket_buffer: [4096]u8 = undefined;
         var reader = stream.reader(zio, &socket_buffer);
         var head_bytes: usize = 0;
@@ -485,6 +561,10 @@ const FakeJevServer = struct {
                 error.EndOfStream => break,
                 else => return err,
             };
+            if (head_bytes < self.captured_head.len) {
+                self.captured_head[head_bytes] = byte;
+                self.captured_len = head_bytes + 1;
+            }
             head_bytes += 1;
             tail = .{ tail[1], tail[2], tail[3], byte };
             if (std.mem.eql(u8, &tail, "\r\n\r\n")) break;
@@ -513,3 +593,52 @@ test "isJevModelId matches canonical and gateway spellings only" {
     try std.testing.expect(!isJevModelId("openai/gpt-5.6-luna"));
     try std.testing.expect(!isJevModelId(""));
 }
+
+test "route resolution prefers TypeSafe direct and derives the gateway evaluation endpoint" {
+    var buf: [4096]u8 = undefined;
+
+    const direct = resolveRoute("ts-key", null, "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?;
+    try std.testing.expect(!direct.via_gateway);
+    try std.testing.expectEqualStrings("ts-key", direct.api_key);
+    try std.testing.expectEqualStrings(default_endpoint, direct.endpoint);
+    try std.testing.expectEqualStrings(jev_model, direct.model);
+
+    const custom_base = resolveRoute("ts-key", "http://127.0.0.1:1/v1/systemone", "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?;
+    try std.testing.expectEqualStrings("http://127.0.0.1:1/v1/systemone", custom_base.endpoint);
+
+    const blank_direct = resolveRoute("  ", null, "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?;
+    try std.testing.expect(blank_direct.via_gateway);
+
+    const gateway = resolveRoute(null, null, "gw-key", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf).?;
+    try std.testing.expect(gateway.via_gateway);
+    try std.testing.expectEqualStrings("gw-key", gateway.api_key);
+    try std.testing.expectEqualStrings("https://ai-gateway.vercel.sh/v4/ai/evaluation-model", gateway.endpoint);
+    try std.testing.expectEqualStrings(gateway_model_id, gateway.model);
+
+    try std.testing.expect(resolveRoute(null, null, "", "https://ai-gateway.vercel.sh/v4/ai/language-model", &buf) == null);
+    try std.testing.expect(resolveRoute(null, null, "gw-key", "https://example.test/chat", &buf) == null);
+}
+
+test "gateway transport posts to the evaluation-model endpoint with protocol headers" {
+    const alloc = std.testing.allocator;
+    var server = try FakeJevServer.initWithPath(.ok, "/v4/ai/evaluation-model");
+    defer server.deinit();
+    try server.start();
+    var cancel = std.atomic.Value(bool).init(false);
+    var config = Config{ .api_key = "gw-key", .endpoint = server.url, .via_gateway = true };
+    const outcome = try sendReview(@ptrCast(&config), alloc, gateway_model_id, "{}", .fromNow(io_mod.getIo(), .{ .clock = .awake, .raw = .fromMilliseconds(5000) }), &cancel);
+    switch (outcome) {
+        .completion => |owned| {
+            var held = owned;
+            defer held.deinit(alloc);
+            try std.testing.expectEqual(@as(usize, 1), held.completion.tool_calls.len);
+            try std.testing.expect(std.mem.find(u8, held.completion.tool_calls[0].arguments_json, "\"decision\":\"clear\"") != null);
+        },
+        else => return error.TestUnexpectedResult,
+    }
+    const head = server.captured();
+    try std.testing.expect(std.mem.find(u8, head, "POST /v4/ai/evaluation-model HTTP") != null);
+    try std.testing.expect(std.mem.find(u8, head, "authorization: Bearer gw-key") != null);
+    try std.testing.expect(std.mem.find(u8, head, "ai-gateway-protocol-version: 0.0.1") != null);
+    try std.testing.expect(std.mem.find(u8, head, "ai-language-model-id: typesafe-ai/jev") != null);
+}

tests/e2e/review-model-override.test.ts

@@ -279,13 +279,12 @@ describe("review model override", () => {
   );
 
   test(
-    "override without TYPESAFE_API_KEY holds the action instead of executing unreviewed",
+    "without TYPESAFE_API_KEY the Jev review goes through the gateway evaluation endpoint",
     async () => {
       const root = createIsolatedRoot();
-      const jev = startJevStub("clear");
       const gateway = startFakeGateway([
         fakeShellRun("cmd_1", `printf '%s' "${EXECUTED_MARKER}"`),
-        fakeGatewayFinalText("Review unavailable, holding."),
+        fakeGatewayFinalText("Ran the command."),
       ]);
       gateways.push(gateway);
       const result = await runFx(
@@ -295,14 +294,59 @@ describe("review model override", () => {
           env: overrideEnv(root, gateway, {
             FX_REVIEW_MODEL: JEV_MODEL_ID,
             TYPESAFE_API_KEY: undefined,
-            TYPESAFE_BASE_URL: jev.url,
+            TYPESAFE_BASE_URL: undefined,
+          }),
+          timeoutMs: TIMEOUT,
+        },
+      );
+      expect(result.code).toBe(0);
+      expect(result.stdout).toContain(EXECUTED_MARKER);
+      expect(gateway.evaluationRequests).toHaveLength(1);
+      expect(gateway.classifierRequests).toHaveLength(0);
+      const review = JSON.parse(gateway.evaluationRequests[0].body);
+      expect(review.model).toBe("typesafe-ai/jev");
+      expect(review.questions.decision.type).toBe("choice");
+      expect(
+        gateway.evaluationRequests[0].headers.get("authorization"),
+      ).toBe("Bearer fake-review-model-override-key");
+      expect(
+        gateway.evaluationRequests[0].headers.get("ai-language-model-id"),
+      ).toBe("typesafe-ai/jev");
+    },
+    TIMEOUT,
+  );
+
+  test(
+    "a failing Jev evaluation endpoint holds the action instead of executing unreviewed",
+    async () => {
+      const root = createIsolatedRoot();
+      const gateway = startFakeGateway(
+        [
+          fakeShellRun("cmd_1", `printf '%s' "${EXECUTED_MARKER}"`),
+          fakeGatewayFinalText("Review unavailable, holding."),
+        ],
+        {
+          evaluationResponse: new Response("upstream unavailable", {
+            status: 500,
+          }),
+        },
+      );
+      gateways.push(gateway);
+      const result = await runFx(
+        ["ask", "--auto", "--quiet", "--json", "Print the marker."],
+        {
+          cwd: root.workspace,
+          env: overrideEnv(root, gateway, {
+            FX_REVIEW_MODEL: JEV_MODEL_ID,
+            TYPESAFE_API_KEY: undefined,
+            TYPESAFE_BASE_URL: undefined,
           }),
           timeoutMs: TIMEOUT,
         },
       );
       expect(result.code).toBe(0);
       expect(result.stdout).not.toContain(EXECUTED_MARKER);
-      expect(jev.requests).toHaveLength(0);
+      expect(gateway.evaluationRequests.length).toBeGreaterThan(0);
       expect(gateway.classifierRequests).toHaveLength(0);
     },
     TIMEOUT,

tests/e2e/tmux-helpers.ts

@@ -330,6 +330,9 @@ export type FakeGatewayOptions = {
     generationId: string,
     request: Request,
   ) => Response | Promise<Response>;
+  // Response for the /v4/ai/evaluation-model endpoint (TypeSafe Jev through
+  // the gateway). Defaults to a clear Jev decision.
+  evaluationResponse?: FakeGatewayResponse;
 };
 
 // Session title generation calls carry this instruction regardless of the
@@ -364,6 +367,7 @@ function serveFakeGateway(
 ) {
   const requests: Array<{ body: string; headers: Headers }> = [];
   const classifierRequests: Array<{ body: string; headers: Headers }> = [];
+  const evaluationRequests: Array<{ body: string; headers: Headers }> = [];
   const classifierResponses = [...(options.classifierResponses ?? [])];
   const titleRequests: Array<{ body: string; headers: Headers }> = [];
   const titleResponses = [...(options.titleResponses ?? [])];
@@ -395,6 +399,31 @@ function serveFakeGateway(
         }
         return new Response("not found", { status: 404 });
       }
+      if (
+        req.method === "POST" &&
+        new URL(req.url).pathname === "/v4/ai/evaluation-model"
+      ) {
+        evaluationRequests.push({
+          body: await req.text(),
+          headers: new Headers(req.headers),
+        });
+        const evaluationResponse = options.evaluationResponse ??
+          Response.json({
+            model: "jev-1.13.0",
+            answers: {
+              decision: {
+                type: "choice",
+                choice: "clear",
+                probabilities: { clear: 0.99, caution: 0.01 },
+                confidence: 0.97,
+              },
+            },
+            usage: { input_tokens: 100, output_tokens: 10 },
+          });
+        return typeof evaluationResponse === "function"
+          ? await evaluationResponse(evaluationRequests[evaluationRequests.length - 1].body)
+          : evaluationResponse;
+      }
       if (req.method !== "POST") return new Response("not found", { status: 404 });
       const body = await req.text();
       const headers = new Headers(req.headers);
@@ -419,6 +448,7 @@ function serveFakeGateway(
     chatUrl: `http://127.0.0.1:${server.port}/v4/ai/language-model`,
     requests,
     classifierRequests,
+    evaluationRequests,
     titleRequests,
     generationRequests,
     modelRequests,