mirror of
https://github.com/vercel-labs/fx.git
synced 2026-10-28 10:15:17 +08:00
65 lines
1.5 KiB
YAML
65 lines
1.5 KiB
YAML
# Single-task Terminal-Bench comparison eval.
|
|
jobs_dir: evals/jobs-out
|
|
n_attempts: 1
|
|
timeout_multiplier: 1.0
|
|
n_concurrent_trials: 1
|
|
quiet: false
|
|
|
|
environment:
|
|
type: docker
|
|
force_build: true
|
|
delete: true
|
|
|
|
agents:
|
|
- name: fx
|
|
import_path: evals.agents.fx_agent:FxAgent
|
|
model_name: anthropic/claude-opus-4.7
|
|
kwargs:
|
|
effort: high
|
|
- name: codex-gateway
|
|
import_path: evals.agents.codex_gateway:CodexGatewayAgent
|
|
model_name: openai/gpt-5.5
|
|
kwargs:
|
|
reasoning_effort: high
|
|
- name: claude-code-gateway
|
|
import_path: evals.agents.claude_gateway:ClaudeCodeGatewayAgent
|
|
model_name: anthropic/claude-opus-4.7
|
|
kwargs:
|
|
reasoning_effort: high
|
|
- name: opencode-gateway
|
|
import_path: evals.agents.opencode_gateway:OpenCodeGatewayAgent
|
|
model_name: openai/anthropic/claude-sonnet-4.6
|
|
|
|
datasets:
|
|
- name: terminal-bench/terminal-bench-2
|
|
task_names:
|
|
- terminal-bench/break-filter-js-from-html
|
|
download_dir: evals/.build/harbor-datasets
|
|
|
|
metrics:
|
|
- type: uv-script
|
|
kwargs:
|
|
script_path: evals/metrics/fx_metrics.py
|
|
|
|
artifacts:
|
|
- source: /logs/agent
|
|
destination: agent
|
|
- source: /logs/verifier
|
|
destination: verifier
|
|
- source: /logs/artifacts
|
|
destination: fx-run
|
|
- source: /app
|
|
destination: workspace
|
|
exclude:
|
|
- node_modules
|
|
- .git
|
|
- .cache
|
|
- __pycache__
|
|
- .pytest_cache
|
|
- dist
|
|
- build
|
|
- coverage
|
|
- target
|
|
- .venv
|
|
- venv
|