Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Switch image-create defaults to Opus 5.5 and GPT-6.1 Sol

Abi Rajaba12de71f003
diff --git a/backend/agent/providers/anthropic/provider.py b/backend/agent/providers/anthropic/provider.py
index 49d880da..26a38c6a 100644
--- a/backend/agent/providers/anthropic/provider.py
+++ b/backend/agent/providers/anthropic/provider.py
@@ -31,6 +31,11 @@ from llm import Llm
THINKING_MODELS: set[str] = set()
ADAPTIVE_THINKING_MODELS = {
+ Llm.CLAUDE_OPUS_5_5_LOW.value,
+ Llm.CLAUDE_OPUS_5_5_MEDIUM.value,
+ Llm.CLAUDE_OPUS_5_5_HIGH.value,
+ Llm.CLAUDE_OPUS_5_5_XHIGH.value,
+ Llm.CLAUDE_OPUS_5_5_MAX.value,
Llm.CLAUDE_OPUS_5_LOW.value,
Llm.CLAUDE_OPUS_5_MEDIUM.value,
Llm.CLAUDE_OPUS_5_HIGH.value,
@@ -50,6 +55,11 @@ ADAPTIVE_THINKING_MODELS = {
}
ANTHROPIC_MODEL_CONFIG: dict[Llm, dict[str, str]] = {
+ Llm.CLAUDE_OPUS_5_5_LOW: {"api_name": "claude-opus-5-5", "effort": "low"},
+ Llm.CLAUDE_OPUS_5_5_MEDIUM: {"api_name": "claude-opus-5-5", "effort": "medium"},
+ Llm.CLAUDE_OPUS_5_5_HIGH: {"api_name": "claude-opus-5-5", "effort": "high"},
+ Llm.CLAUDE_OPUS_5_5_XHIGH: {"api_name": "claude-opus-5-5", "effort": "xhigh"},
+ Llm.CLAUDE_OPUS_5_5_MAX: {"api_name": "claude-opus-5-5", "effort": "max"},
Llm.CLAUDE_OPUS_5_LOW: {"api_name": "claude-opus-5", "effort": "low"},
Llm.CLAUDE_OPUS_5_MEDIUM: {"api_name": "claude-opus-5", "effort": "medium"},
Llm.CLAUDE_OPUS_5_HIGH: {"api_name": "claude-opus-5", "effort": "high"},
diff --git a/backend/agent/providers/openai.py b/backend/agent/providers/openai.py
index d1e2cdbe..cd7d7a60 100644
--- a/backend/agent/providers/openai.py
+++ b/backend/agent/providers/openai.py
@@ -1,11 +1,16 @@
# pyright: reportUnknownVariableType=false
+import asyncio
import base64
import copy
+import io
import json
+import math
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
+from PIL import Image
+
from openai import AsyncOpenAI
from openai.types.chat import ChatCompletionMessageParam
@@ -25,6 +30,61 @@ from fs_logging.prompt_reports import PromptReportLogger
from llm import Llm, get_openai_api_name, get_openai_reasoning_effort
+# gpt-6.1-sol rejects "original"-detail images that need more than 30,000
+# 32x32 patches (a 2724x11746 landing-page screenshot needs 31,648).
+IMAGE_PATCH_SIZE = 32
+PATCH_LIMITED_MODEL_MAX_PATCHES: Dict[str, int] = {"gpt-6.1-sol": 30000}
+
+
+def _patch_count(width: int, height: int) -> int:
+ return math.ceil(width / IMAGE_PATCH_SIZE) * math.ceil(height / IMAGE_PATCH_SIZE)
+
+
+def _fit_image_to_patch_limit(image_url: str, max_patches: Optional[int]) -> str:
+ """Downscale a base64 data-URL image that exceeds the model's patch limit.
+
+ Remote URLs and undecodable data are returned unchanged.
+ """
+ if max_patches is None or not image_url.startswith("data:"):
+ return image_url
+ try:
+ header, encoded = image_url.split(",", 1)
+ image = Image.open(io.BytesIO(base64.b64decode(encoded)))
+ width, height = image.size
+ except Exception:
+ return image_url
+ if _patch_count(width, height) <= max_patches:
+ return image_url
+
+ scale = math.sqrt(max_patches / _patch_count(width, height))
+ new_width, new_height = width, height
+ while _patch_count(new_width, new_height) > max_patches:
+ new_width = max(1, math.floor(width * scale))
+ new_height = max(1, math.floor(height * scale))
+ scale *= 0.99
+ resized = image.resize((new_width, new_height), Image.Resampling.LANCZOS)
+ output_format = image.format if image.format in ("PNG", "JPEG", "WEBP") else "PNG"
+ if output_format == "JPEG" and resized.mode not in ("RGB", "L"):
+ resized = resized.convert("RGB")
+ buffer = io.BytesIO()
+ resized.save(buffer, format=output_format)
+ mime_type = f"image/{output_format.lower()}"
+ print(
+ f"[OPENAI IMAGE] Downscaled {width}x{height} -> {new_width}x{new_height} "
+ f"to fit {max_patches} patches ({header})"
+ )
+ return f"data:{mime_type};base64,{base64.b64encode(buffer.getvalue()).decode('ascii')}"
+
+
+def _get_max_patches_for_model(model: Llm) -> Optional[int]:
+ return PATCH_LIMITED_MODEL_MAX_PATCHES.get(get_openai_api_name(model))
+
+
+async def _fit_image_off_loop(image_url: str, max_patches: int) -> str:
+ # Resizing a tall screenshot takes ~2s, so keep it off the event loop.
+ return await asyncio.to_thread(_fit_image_to_patch_limit, image_url, max_patches)
+
+
def _convert_message_to_responses_input(
message: ChatCompletionMessageParam,
image_detail: str = "high",
@@ -60,6 +120,7 @@ def _get_image_detail_for_model(model: Llm) -> str:
"gpt-5.5",
"gpt-5.6-sol",
"gpt-5.6-terra",
+ "gpt-6.1-sol",
}:
return "original"
return "high"
@@ -440,8 +501,26 @@ class OpenAIProviderSession(ProviderSession):
_convert_message_to_responses_input(message, image_detail=image_detail)
for message in prompt_messages
]
+ self._max_patches = _get_max_patches_for_model(model)
+ # Prompt images are fitted on the first turn, where we can await.
+ self._prompt_images_fitted = self._max_patches is None
+
+ async def _fit_prompt_images(self) -> None:
+ if self._prompt_images_fitted or self._max_patches is None:
+ return
+ for item in self._input_items:
+ content = item.get("content")
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if part.get("type") == "input_image":
+ part["image_url"] = await _fit_image_off_loop(
+ part["image_url"], self._max_patches
+ )
+ self._prompt_images_fitted = True
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
+ await self._fit_prompt_images()
model_name = get_openai_api_name(self._model)
params: Dict[str, Any] = {
"model": model_name,
@@ -517,6 +596,10 @@ class OpenAIProviderSession(ProviderSession):
image_url = self._image_ref(part)
if image_url is None:
continue
+ if self._max_patches is not None:
+ image_url = await _fit_image_off_loop(
+ image_url, self._max_patches
+ )
output.append(
{
"type": "input_image",
diff --git a/backend/costs/pricing.py b/backend/costs/pricing.py
index 20b7539c..68d549b2 100644
--- a/backend/costs/pricing.py
+++ b/backend/costs/pricing.py
@@ -15,7 +15,9 @@ class ModelPricing:
# Pricing keyed by the API model name string sent to the provider.
MODEL_PRICING: Dict[str, ModelPricing] = {
# --- OpenAI ---
- # Verified against developers.openai.com/api/docs/pricing on 2026-07-24.
+ # Verified against developers.openai.com/api/docs/pricing on 2026-10-09.
+ # GPT-5.6/6.x also bill cache writes at 1.25x input, but the Responses API
+ # usage we read doesn't split them out, so cache_write stays 0 here.
"gpt-5.4-mini": ModelPricing(
input=0.75, output=4.50, cache_read=0.075
),
@@ -25,18 +27,25 @@ MODEL_PRICING: Dict[str, ModelPricing] = {
"gpt-5.5": ModelPricing(
input=5.00, output=30.00, cache_read=0.50
),
+ # Promotional price, "at least through November 21, 2026".
"gpt-5.6-sol": ModelPricing(
- input=5.00, output=30.00, cache_read=0.50
+ input=4.00, output=20.00, cache_read=0.40
),
"gpt-5.6-terra": ModelPricing(
- input=2.50, output=15.00, cache_read=0.25
+ input=2.00, output=12.00, cache_read=0.20
+ ),
+ "gpt-6.1-sol": ModelPricing(
+ input=2.00, output=10.00, cache_read=0.10
),
# --- Anthropic ---
# Verified against platform.claude.com/docs/en/about-claude/pricing on
- # 2026-07-24. cache_write is the 5-minute cache-write rate (1.25x input).
+ # 2026-10-09. cache_write is the 5-minute cache-write rate (1.25x input).
"claude-sonnet-4-6": ModelPricing(
input=3.00, output=15.00, cache_read=0.30, cache_write=3.75
),
+ "claude-opus-5-5": ModelPricing(
+ input=4.00, output=20.00, cache_read=0.20, cache_write=5.00
+ ),
"claude-opus-5": ModelPricing(
input=5.00, output=25.00, cache_read=0.50, cache_write=6.25
),
diff --git a/backend/llm.py b/backend/llm.py
index b0d60003..7cb14d58 100644
--- a/backend/llm.py
+++ b/backend/llm.py
@@ -23,8 +23,18 @@ class Llm(Enum):
GPT_5_6_SOL_XHIGH = "gpt-5.6-sol (xhigh thinking)"
GPT_5_6_SOL_MAX = "gpt-5.6-sol (max thinking)"
GPT_5_6_TERRA_LOW = "gpt-5.6-terra (low thinking)"
+ GPT_6_1_SOL_LOW = "gpt-6.1-sol (low thinking)"
+ GPT_6_1_SOL_MEDIUM = "gpt-6.1-sol (medium thinking)"
+ GPT_6_1_SOL_HIGH = "gpt-6.1-sol (high thinking)"
+ GPT_6_1_SOL_XHIGH = "gpt-6.1-sol (xhigh thinking)"
+ GPT_6_1_SOL_MAX = "gpt-6.1-sol (max thinking)"
# Claude
CLAUDE_SONNET_4_6 = "claude-sonnet-4-6"
+ CLAUDE_OPUS_5_5_LOW = "claude-opus-5-5 (low effort)"
+ CLAUDE_OPUS_5_5_MEDIUM = "claude-opus-5-5 (medium effort)"
+ CLAUDE_OPUS_5_5_HIGH = "claude-opus-5-5 (high effort)"
+ CLAUDE_OPUS_5_5_XHIGH = "claude-opus-5-5 (xhigh effort)"
+ CLAUDE_OPUS_5_5_MAX = "claude-opus-5-5 (max effort)"
CLAUDE_OPUS_5_LOW = "claude-opus-5 (low effort)"
CLAUDE_OPUS_5_MEDIUM = "claude-opus-5 (medium effort)"
CLAUDE_OPUS_5_HIGH = "claude-opus-5 (high effort)"
@@ -84,8 +94,18 @@ MODEL_PROVIDER: dict[Llm, str] = {
Llm.GPT_5_6_SOL_XHIGH: "openai",
Llm.GPT_5_6_SOL_MAX: "openai",
Llm.GPT_5_6_TERRA_LOW: "openai",
+ Llm.GPT_6_1_SOL_LOW: "openai",
+ Llm.GPT_6_1_SOL_MEDIUM: "openai",
+ Llm.GPT_6_1_SOL_HIGH: "openai",
+ Llm.GPT_6_1_SOL_XHIGH: "openai",
+ Llm.GPT_6_1_SOL_MAX: "openai",
# Anthropic models
Llm.CLAUDE_SONNET_4_6: "anthropic",
+ Llm.CLAUDE_OPUS_5_5_LOW: "anthropic",
+ Llm.CLAUDE_OPUS_5_5_MEDIUM: "anthropic",
+ Llm.CLAUDE_OPUS_5_5_HIGH: "anthropic",
+ Llm.CLAUDE_OPUS_5_5_XHIGH: "anthropic",
+ Llm.CLAUDE_OPUS_5_5_MAX: "anthropic",
Llm.CLAUDE_OPUS_5_LOW: "anthropic",
Llm.CLAUDE_OPUS_5_MEDIUM: "anthropic",
Llm.CLAUDE_OPUS_5_HIGH: "anthropic",
@@ -156,6 +176,12 @@ OPENAI_MODEL_CONFIG: dict[Llm, dict[str, str]] = {
Llm.GPT_5_6_SOL_XHIGH: {"api_name": "gpt-5.6-sol", "reasoning_effort": "xhigh"},
Llm.GPT_5_6_SOL_MAX: {"api_name": "gpt-5.6-sol", "reasoning_effort": "max"},
Llm.GPT_5_6_TERRA_LOW: {"api_name": "gpt-5.6-terra", "reasoning_effort": "low"},
+ # gpt-6.1-sol does not support the "none" reasoning effort.
+ Llm.GPT_6_1_SOL_LOW: {"api_name": "gpt-6.1-sol", "reasoning_effort": "low"},
+ Llm.GPT_6_1_SOL_MEDIUM: {"api_name": "gpt-6.1-sol", "reasoning_effort": "medium"},
+ Llm.GPT_6_1_SOL_HIGH: {"api_name": "gpt-6.1-sol", "reasoning_effort": "high"},
+ Llm.GPT_6_1_SOL_XHIGH: {"api_name": "gpt-6.1-sol", "reasoning_effort": "xhigh"},
+ Llm.GPT_6_1_SOL_MAX: {"api_name": "gpt-6.1-sol", "reasoning_effort": "max"},
}
diff --git a/backend/routes/eval_sets.py b/backend/routes/eval_sets.py
index 62c4b8a2..abc72f29 100644
--- a/backend/routes/eval_sets.py
+++ b/backend/routes/eval_sets.py
@@ -10,6 +10,10 @@ from pydantic import BaseModel
from evals import sessions as eval_sessions
from evals import sets as eval_sets
from fs_logging.agent_runs import open_index_db
+from routes.model_choice_sets import (
+ ALL_KEYS_MODELS_DEFAULT,
+ ALL_KEYS_MODELS_TEXT_CREATE,
+)
router = APIRouter()
@@ -95,6 +99,8 @@ class SessionMatrixResponse(BaseModel):
model_notes: Dict[str, str]
cells: List[MatrixCellModel]
unmatched_run_count: int
+ # Current production create defaults for the set's input kind.
+ default_models: List[str] = []
class ModelOrderRequest(BaseModel):
@@ -117,6 +123,14 @@ def _set_info_to_model(info: eval_sets.EvalSetInfo) -> EvalSetSummaryModel:
)
+def _default_models_for_set(set_info: Optional[eval_sets.EvalSetInfo]) -> List[str]:
+ if set_info is None:
+ return []
+ if set_info.kind == "text":
+ return [model.value for model in ALL_KEYS_MODELS_TEXT_CREATE]
+ return [model.value for model in ALL_KEYS_MODELS_DEFAULT]
+
+
def _session_to_model(session: eval_sessions.EvalSession) -> EvalSessionModel:
return EvalSessionModel(
session_id=session.session_id,
@@ -365,6 +379,7 @@ async def get_session_matrix(session_id: str) -> SessionMatrixResponse:
model_notes=model_notes,
cells=cells,
unmatched_run_count=unmatched_run_count,
+ default_models=_default_models_for_set(set_info),
)
diff --git a/backend/routes/model_choice_sets.py b/backend/routes/model_choice_sets.py
index a3398e69..718fa531 100644
--- a/backend/routes/model_choice_sets.py
+++ b/backend/routes/model_choice_sets.py
@@ -10,15 +10,11 @@ VIDEO_VARIANT_MODELS = (
# Image (Create)
-# Refreshed 2026-07-27 from judged evals on the jun-21 set (see the
-# "Jul 24 session" matrix): sol max is the quality anchor (8.2/10 avg),
-# opus 5 medium the best non-anchor (7.8), 3.1 pro the third take (7.2),
-# and 3-flash high the fast slot (zero failures across the set).
ALL_KEYS_MODELS_DEFAULT = (
- Llm.CLAUDE_OPUS_5_MEDIUM,
+ Llm.CLAUDE_OPUS_5_5_MEDIUM,
Llm.GEMINI_3_FLASH_PREVIEW_HIGH,
- Llm.GEMINI_3_1_PRO_PREVIEW_HIGH,
- Llm.GPT_5_6_SOL_MAX,
+ Llm.CLAUDE_OPUS_5_5_XHIGH,
+ Llm.GPT_6_1_SOL_HIGH,
)
# Text (Create)
diff --git a/backend/tests/test_eval_sessions.py b/backend/tests/test_eval_sessions.py
index 99030a9c..2d525691 100644
--- a/backend/tests/test_eval_sessions.py
+++ b/backend/tests/test_eval_sessions.py
@@ -395,6 +395,10 @@ async def test_session_matrix_text_set(
assert ("pdp", Llm.GPT_5_5_HIGH.value) in cells
assert matrix.unmatched_run_count == 0
+ from routes.model_choice_sets import ALL_KEYS_MODELS_TEXT_CREATE
+
+ assert matrix.default_models == [m.value for m in ALL_KEYS_MODELS_TEXT_CREATE]
+
@pytest.mark.asyncio
async def test_session_matrix_missing_set(
@@ -504,3 +508,17 @@ def test_count_pending_eval_tasks_with_set(
eval_set="set-a",
)
assert (pending, skipped) == (3, 0)
+
+
+async def test_session_matrix_default_models(
+ logs_path: Path, evals_dir: Path
+) -> None:
+ from routes.eval_sets import get_session_matrix
+ from routes.model_choice_sets import ALL_KEYS_MODELS_DEFAULT
+
+ _make_set(evals_dir, "set-a", {"one.png": b"image-one"})
+ session = create_session("set-a")
+
+ matrix = await get_session_matrix(session.session_id)
+ assert matrix.default_models == [m.value for m in ALL_KEYS_MODELS_DEFAULT]
diff --git a/backend/tests/test_model_selection.py b/backend/tests/test_model_selection.py
index 24087dfc..da2d3ebb 100644
--- a/backend/tests/test_model_selection.py
+++ b/backend/tests/test_model_selection.py
@@ -43,10 +43,10 @@ class TestModelSelectionAllKeys:
)
expected = [
- Llm.CLAUDE_OPUS_5_MEDIUM,
+ Llm.CLAUDE_OPUS_5_5_MEDIUM,
Llm.GEMINI_3_FLASH_PREVIEW_HIGH,
- Llm.GEMINI_3_1_PRO_PREVIEW_HIGH,
- Llm.GPT_5_6_SOL_MAX,
+ Llm.CLAUDE_OPUS_5_5_XHIGH,
+ Llm.GPT_6_1_SOL_HIGH,
]
assert models == expected
diff --git a/backend/tests/test_openai_provider_session.py b/backend/tests/test_openai_provider_session.py
index fcf087f5..92efd1df 100644
--- a/backend/tests/test_openai_provider_session.py
+++ b/backend/tests/test_openai_provider_session.py
@@ -332,3 +332,138 @@ async def test_openai_provider_session_keeps_high_image_detail_for_non_gpt_5_5()
assert image_part["type"] == "input_image"
assert image_part["detail"] == "high"
+
+
+def _png_data_url(width: int, height: int) -> str:
+ import base64
+ import io
+
+ from PIL import Image
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (width, height), (255, 255, 255)).save(buffer, format="PNG")
+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+
+
+def _data_url_size(url: str) -> tuple[int, int]:
+ import base64
+ import io
+
+ from PIL import Image
+
+ return Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1]))).size
+
+
+async def test_gpt_6_1_sol_downscales_images_over_patch_limit() -> None:
+ import base64
+
+ from agent.providers.openai import _patch_count
+ from agent.tools.types import ToolMultimodalPart
+
+ client = _FakeOpenAIClient()
+ tall = _png_data_url(2724, 11746) # 31,648 patches; the limit is 30,000
+ session = OpenAIProviderSession(
+ client=client, # type: ignore[arg-type]
+ model=Llm.GPT_6_1_SOL_HIGH,
+ prompt_messages=[
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Build from this screenshot."},
+ {"type": "image_url", "image_url": {"url": tall}},
+ ],
+ }
+ ],
+ tools=_test_tools(),
+ )
+ await session.stream_turn(_noop_event_sink)
+
+ image_part = client.responses.calls[0]["input"][0]["content"][1]
+ assert image_part["detail"] == "original"
+ width, height = _data_url_size(image_part["image_url"])
+ assert _patch_count(width, height) <= 30000
+ assert abs(width / height - 2724 / 11746) < 0.01
+
+ # Tool-result images (e.g. preview screenshots) are fitted too.
+ await session.append_tool_results(
+ ProviderTurn(assistant_text="", tool_calls=[], assistant_turn=[]),
+ [
+ ExecutedToolCall(
+ tool_call=ToolCall(id="call_1", name="screenshot_preview", arguments={}),
+ result=ToolExecutionResult(
+ ok=True,
+ result={"status": "ok"},
+ summary={},
+ multimodal_parts=[
+ ToolMultimodalPart(
+ display_name="preview.png",
+ mime_type="image/png",
+ data=base64.b64decode(tall.split(",", 1)[1]),
+ )
+ ],
+ ),
+ )
+ ],
+ )
+ await session.stream_turn(_noop_event_sink)
+ tool_output = client.responses.calls[1]["input"][-1]["output"]
+ width, height = _data_url_size(tool_output[1]["image_url"])
+ assert _patch_count(width, height) <= 30000
+
+
+async def test_images_under_patch_limit_and_other_models_are_untouched() -> None:
+ small = _png_data_url(1280, 4000)
+ tall = _png_data_url(2724, 11746)
+ for model, url in ((Llm.GPT_6_1_SOL_HIGH, small), (Llm.GPT_5_6_SOL_MAX, tall)):
+ client = _FakeOpenAIClient()
+ session = OpenAIProviderSession(
+ client=client, # type: ignore[arg-type]
+ model=model,
+ prompt_messages=[
+ {
+ "role": "user",
+ "content": [{"type": "image_url", "image_url": {"url": url}}],
+ }
+ ],
+ tools=_test_tools(),
+ )
+ await session.stream_turn(_noop_event_sink)
+ assert client.responses.calls[0]["input"][0]["content"][0]["image_url"] == url
+
+
+async def test_gpt_6_1_sol_fits_images_off_the_event_loop(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import asyncio
+
+ import agent.providers.openai as openai_provider
+
+ offloaded: list[Any] = []
+ real_to_thread = asyncio.to_thread
+
+ async def recording_to_thread(func: Any, *args: Any) -> Any:
+ offloaded.append(func)
+ return await real_to_thread(func, *args)
+
+ monkeypatch.setattr(openai_provider.asyncio, "to_thread", recording_to_thread)
+
+ client = _FakeOpenAIClient()
+ session = OpenAIProviderSession(
+ client=client, # type: ignore[arg-type]
+ model=Llm.GPT_6_1_SOL_HIGH,
+ prompt_messages=[
+ {
+ "role": "user",
+ "content": [
+ {"type": "image_url", "image_url": {"url": _png_data_url(64, 64)}}
+ ],
+ }
+ ],
+ tools=_test_tools(),
+ )
+ assert offloaded == [] # nothing is decoded while constructing the session
+ await session.stream_turn(_noop_event_sink)
+ assert offloaded == [openai_provider._fit_image_to_patch_limit]
diff --git a/frontend/src/components/evals/EvalComparePage.tsx b/frontend/src/components/evals/EvalComparePage.tsx
index 76f06901..7975fd8c 100644
--- a/frontend/src/components/evals/EvalComparePage.tsx
+++ b/frontend/src/components/evals/EvalComparePage.tsx
@@ -1,10 +1,13 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import toast from "react-hot-toast";
-import { useNavigate, useSearchParams } from "react-router-dom";
+import { useSearchParams } from "react-router-dom";
import {
BsBoxArrowUpRight,
+ BsChevronDown,
BsChevronLeft,
BsChevronRight,
+ BsChevronUp,
+ BsPinAngleFill,
} from "react-icons/bs";
import { HTTP_BACKEND_URL } from "../../config";
@@ -52,6 +55,8 @@ interface SessionMatrix {
model_notes: Record<string, string>;
cells: MatrixCell[];
unmatched_run_count: number;
+ // The current production default set for this session's input kind.
+ default_models: string[];
}
// Stable chip colors so a model keeps its hue as you toggle selections.
@@ -73,8 +78,17 @@ function shortModelName(model: string): string {
.replace(" effort)", ")");
}
+const CONTROLS_COLLAPSED_KEY = "evalCompare.controlsCollapsed";
+
+function readControlsCollapsed(): boolean {
+ try {
+ return window.localStorage.getItem(CONTROLS_COLLAPSED_KEY) === "1";
+ } catch {
+ return false;
+ }
+}
+
function EvalComparePage() {
- const navigate = useNavigate();
const [searchParams, setSearchParams] = useSearchParams();
const [sessions, setSessions] = useState<EvalSession[]>([]);
const [matrix, setMatrix] = useState<SessionMatrix | null>(null);
@@ -83,6 +97,22 @@ function EvalComparePage() {
// 0 = source screenshot, 1..N = selected models in order.
const [paneIndex, setPaneIndex] = useState(1);
const [connectionFailed, setConnectionFailed] = useState(false);
+ // Collapsing hides the eval nav and model chips so the output fills the screen.
+ const [controlsCollapsed, setControlsCollapsed] = useState(
+ readControlsCollapsed
+ );
+
+ const toggleControls = useCallback(() => {
+ setControlsCollapsed((current) => {
+ const next = !current;
+ try {
+ window.localStorage.setItem(CONTROLS_COLLAPSED_KEY, next ? "1" : "0");
+ } catch {
+ // Storage unavailable; the toggle still works for this page view.
+ }
+ return next;
+ });
+ }, []);
const [retryTick, setRetryTick] = useState(0);
const sessionId = searchParams.get("session");
@@ -152,12 +182,17 @@ function EvalComparePage() {
if (!response.ok) throw new Error("Request failed");
const data: SessionMatrix = await response.json();
setMatrix(data);
- // Default: first two models that actually have runs.
- setSelectedModels((current) =>
- current.length > 0
- ? current.filter((m) => data.models.includes(m))
- : data.models.slice(0, 2)
+ // Start from the production defaults; after that the selection is
+ // the user's. With no defaults in the session, fall back to the first
+ // two models that actually have runs.
+ const defaults = data.default_models.filter((m) =>
+ data.models.includes(m)
);
+ setSelectedModels((current) => {
+ const kept = current.filter((m) => data.models.includes(m));
+ if (kept.length > 0) return kept;
+ return defaults.length > 0 ? defaults : data.models.slice(0, 2);
+ });
setImageIndex(0);
setPaneIndex(1);
} catch (error) {
@@ -207,6 +242,11 @@ function EvalComparePage() {
loadMatrix(id);
};
+ const isDefaultModel = useCallback(
+ (model: string) => matrix?.default_models.includes(model) ?? false,
+ [matrix]
+ );
+
const toggleModel = (model: string) => {
setSelectedModels((current) =>
current.includes(model)
@@ -239,7 +279,8 @@ function EvalComparePage() {
const onKeyDown = (e: KeyboardEvent) => {
if (
e.target instanceof HTMLInputElement ||
- e.target instanceof HTMLSelectElement
+ e.target instanceof HTMLSelectElement ||
+ e.target instanceof HTMLTextAreaElement
) {
return;
}
@@ -248,6 +289,7 @@ function EvalComparePage() {
else if (e.key === "ArrowDown" || e.key === "]") stepImage(1);
else if (e.key === "ArrowUp" || e.key === "[") stepImage(-1);
else if (e.key === "s" || e.key === "0") setPaneIndex(0);
+ else if (e.key === "c") toggleControls();
else if (/^[1-9]$/.test(e.key)) {
const target = parseInt(e.key, 10);
if (target < panes.length) setPaneIndex(target);
@@ -255,15 +297,17 @@ function EvalComparePage() {
};
window.addEventListener("keydown", onKeyDown);
return () => window.removeEventListener("keydown", onKeyDown);
- }, [stepPane, stepImage, panes.length]);
+ }, [stepPane, stepImage, panes.length, toggleControls]);
const activePane = panes[Math.min(paneIndex, panes.length - 1)] ?? null;
return (
<div className="flex h-screen flex-col overflow-hidden bg-zinc-950 text-white">
- <div className="shrink-0">
- <EvalNavigation />
- </div>
+ {!controlsCollapsed && (
+ <div className="shrink-0">
+ <EvalNavigation />
+ </div>
+ )}
<div className="flex min-h-0 flex-1 flex-col gap-2 px-4 py-2">
<div className="flex shrink-0 flex-wrap items-center gap-2 rounded-xl border border-zinc-800 bg-zinc-900/80 px-3 py-2 text-xs">
<h1 className="text-sm font-semibold tracking-tight">Compare</h1>
@@ -279,23 +323,46 @@ function EvalComparePage() {
))}
</select>
<div className="mx-1 h-4 w-px bg-zinc-800" />
- {matrix?.models.map((model, index) => (
+ {controlsCollapsed && (
+ <span className="text-[11px] text-zinc-500">
+ {selectedModels.length} models selected
+ </span>
+ )}
+ {!controlsCollapsed && matrix?.models.map((model, index) => (
<button
key={model}
onClick={() => toggleModel(model)}
- title={matrix.model_notes[model] || model}
- className={`rounded-full border px-2 py-0.5 font-mono text-[11px] transition-colors ${
+ title={
+ isDefaultModel(model)
+ ? `Current default — selected by default. ${matrix.model_notes[model] || model}`
+ : matrix.model_notes[model] || model
+ }
+ className={`flex items-center gap-1 rounded-full border px-2 py-0.5 font-mono text-[11px] transition-colors ${
selectedModels.includes(model)
? MODEL_COLORS[index % MODEL_COLORS.length]
: "border-zinc-700 text-zinc-500 hover:text-zinc-300"
}`}
>
+ {isDefaultModel(model) && (
+ <BsPinAngleFill className="text-[9px] opacity-80" />
+ )}
{shortModelName(model)}
</button>
))}
- <span className="ml-auto hidden text-[11px] text-zinc-600 lg:inline">
- ←/→ switch output · ↑/↓ image · S source
- </span>
+ <div className="ml-auto" />
+ {!controlsCollapsed && (
+ <span className="hidden text-[11px] text-zinc-600 lg:inline">
+ ←/→ output · ↑/↓ image · S source · C collapse
+ </span>
+ )}
+ <button
+ onClick={toggleControls}
+ title={controlsCollapsed ? "Show models and nav (C)" : "Collapse (C)"}
+ className="flex items-center gap-1 rounded-md border border-zinc-700 px-2 py-1 text-[11px] text-zinc-300 hover:bg-zinc-800"
+ >
+ {controlsCollapsed ? <BsChevronDown /> : <BsChevronUp />}
+ {controlsCollapsed ? "Expand" : "Collapse"}
+ </button>
</div>
{currentImage && (
@@ -361,18 +428,16 @@ function EvalComparePage() {
{activePane?.run && (
<span className="ml-auto flex items-center gap-2">
- <button
- onClick={() =>
- navigate(
- `/evals/agent-runs?run=${encodeURIComponent(
- activePane.run!.run_id
- )}`
- )
- }
+ <a
+ href={`/evals/agent-runs?run=${encodeURIComponent(
+ activePane.run.run_id
+ )}`}
+ target="_blank"
+ rel="noreferrer"
className="rounded-md border border-zinc-700 px-2 py-1 text-zinc-300 hover:bg-zinc-800"
>
Run details
- </button>
+ </a>
<a
href={`${HTTP_BACKEND_URL}/agent-runs/${encodeURIComponent(
activePane.run.run_id
diff --git a/frontend/src/components/sidebar/Sidebar.tsx b/frontend/src/components/sidebar/Sidebar.tsx
index 562be82d..d9697644 100644
--- a/frontend/src/components/sidebar/Sidebar.tsx
+++ b/frontend/src/components/sidebar/Sidebar.tsx
@@ -70,7 +70,9 @@ function isSlowModel(model?: string): boolean {
return (
model === CodeGenerationModel.GEMINI_3_1_PRO_PREVIEW_HIGH ||
model === CodeGenerationModel.GEMINI_3_1_PRO_PREVIEW_MEDIUM ||
- model === CodeGenerationModel.GPT_5_6_SOL_MAX
+ model === CodeGenerationModel.GPT_5_6_SOL_MAX ||
+ model === CodeGenerationModel.CLAUDE_OPUS_5_5_XHIGH ||
+ model === CodeGenerationModel.GPT_6_1_SOL_HIGH
);
}
diff --git a/frontend/src/lib/models.ts b/frontend/src/lib/models.ts
index f8d678b2..243573c2 100644
--- a/frontend/src/lib/models.ts
+++ b/frontend/src/lib/models.ts
@@ -1,6 +1,11 @@
// Keep in sync with backend (llm.py)
// Order here matches dropdown order
export enum CodeGenerationModel {
+ CLAUDE_OPUS_5_5_LOW = "claude-opus-5-5 (low effort)",
+ CLAUDE_OPUS_5_5_MEDIUM = "claude-opus-5-5 (medium effort)",
+ CLAUDE_OPUS_5_5_HIGH = "claude-opus-5-5 (high effort)",
+ CLAUDE_OPUS_5_5_XHIGH = "claude-opus-5-5 (xhigh effort)",
+ CLAUDE_OPUS_5_5_MAX = "claude-opus-5-5 (max effort)",
CLAUDE_OPUS_5_LOW = "claude-opus-5 (low effort)",
CLAUDE_OPUS_5_MEDIUM = "claude-opus-5 (medium effort)",
CLAUDE_OPUS_5_HIGH = "claude-opus-5 (high effort)",
@@ -24,6 +29,9 @@ export enum CodeGenerationModel {
GPT_5_6_SOL_XHIGH = "gpt-5.6-sol (xhigh thinking)",
GPT_5_6_SOL_MAX = "gpt-5.6-sol (max thinking)",
GPT_5_6_TERRA_LOW = "gpt-5.6-terra (low thinking)",
+ GPT_6_1_SOL_HIGH = "gpt-6.1-sol (high thinking)",
+ GPT_6_1_SOL_XHIGH = "gpt-6.1-sol (xhigh thinking)",
+ GPT_6_1_SOL_MAX = "gpt-6.1-sol (max thinking)",
GPT_5_5_XHIGH = "gpt-5.5 (xhigh thinking)",
GPT_5_4_MINI_LOW = "gpt-5.4-mini (low thinking)",
GEMINI_3_FLASH_PREVIEW_HIGH = "gemini-3-flash-preview (high thinking)",
@@ -80,6 +88,15 @@ export function getVariantLabel(
export const CODE_GENERATION_MODEL_DESCRIPTIONS: {
[key in CodeGenerationModel]: { name: string };
} = {
+ "gpt-6.1-sol (high thinking)": {
+ name: "GPT 6.1 Sol (high)",
+ },
+ "gpt-6.1-sol (xhigh thinking)": {
+ name: "GPT 6.1 Sol (xhigh)",
+ },
+ "gpt-6.1-sol (max thinking)": {
+ name: "GPT 6.1 Sol (max)",
+ },
"gpt-5.6-sol (no thinking)": {
name: "GPT 5.6 Sol (none)",
},
@@ -119,6 +136,21 @@ export const CODE_GENERATION_MODEL_DESCRIPTIONS: {
"gpt-5.4-mini (low thinking)": {
name: "GPT 5.4 Mini (low)",
},
+ "claude-opus-5-5 (low effort)": {
+ name: "Claude Opus 5.5 (low)",
+ },
+ "claude-opus-5-5 (medium effort)": {
+ name: "Claude Opus 5.5 (medium)",
+ },
+ "claude-opus-5-5 (high effort)": {
+ name: "Claude Opus 5.5 (high)",
+ },
+ "claude-opus-5-5 (xhigh effort)": {
+ name: "Claude Opus 5.5 (xhigh)",
+ },
+ "claude-opus-5-5 (max effort)": {
+ name: "Claude Opus 5.5 (max)",
+ },
"claude-opus-5 (low effort)": {
name: "Claude Opus 5 (low)",
},