Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Support text-brief eval sets alongside image sets

Abi Raja00db2b41284e
diff --git a/backend/evals/core.py b/backend/evals/core.py
index b938e1aa..f0232356 100644
--- a/backend/evals/core.py
+++ b/backend/evals/core.py
@@ -13,26 +13,21 @@ from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS
from agent.runner import Agent
from fs_logging.agent_runs import AgentRunRecorder
from prompts.create.image import build_image_prompt_messages
+from prompts.create.text import build_text_prompt_messages
from prompts.prompt_types import Stack
from openai.types.chat import ChatCompletionMessageParam
-from typing import Any
+from typing import Any, List
-async def generate_code_for_image(
- image_url: str,
+async def _run_eval_agent(
+ prompt_messages: List[ChatCompletionMessageParam],
stack: Stack,
model: Llm,
- *,
- eval_set: str | None = None,
- eval_session_id: str | None = None,
- input_file: str | None = None,
+ input_mode: str,
+ eval_set: str | None,
+ eval_session_id: str | None,
+ input_file: str | None,
) -> str:
- prompt_messages = build_image_prompt_messages(
- image_data_urls=[image_url],
- stack=stack,
- text_prompt="",
- image_generation_enabled=True,
- )
async def send_message(
_: str,
__: str | None,
@@ -59,7 +54,7 @@ async def generate_code_for_image(
variant_index=0,
entry_point="eval",
stack=str(stack),
- input_mode="image",
+ input_mode=input_mode,
generation_type="create",
eval_session=eval_session_id,
eval_set=eval_set,
@@ -82,3 +77,55 @@ async def generate_code_for_image(
recorder=recorder,
)
return await runner.run(model, prompt_messages)
+
+
+async def generate_code_for_image(
+ image_url: str,
+ stack: Stack,
+ model: Llm,
+ *,
+ eval_set: str | None = None,
+ eval_session_id: str | None = None,
+ input_file: str | None = None,
+) -> str:
+ prompt_messages = build_image_prompt_messages(
+ image_data_urls=[image_url],
+ stack=stack,
+ text_prompt="",
+ image_generation_enabled=True,
+ )
+ return await _run_eval_agent(
+ prompt_messages,
+ stack,
+ model,
+ input_mode="image",
+ eval_set=eval_set,
+ eval_session_id=eval_session_id,
+ input_file=input_file,
+ )
+
+
+async def generate_code_for_text(
+ text_prompt: str,
+ stack: Stack,
+ model: Llm,
+ *,
+ eval_set: str | None = None,
+ eval_session_id: str | None = None,
+ input_file: str | None = None,
+) -> str:
+ """Text-create eval: same prompt construction as the app's text flow."""
+ prompt_messages = build_text_prompt_messages(
+ text_prompt=text_prompt,
+ stack=stack,
+ image_generation_enabled=True,
+ )
+ return await _run_eval_agent(
+ prompt_messages,
+ stack,
+ model,
+ input_mode="text",
+ eval_set=eval_set,
+ eval_session_id=eval_session_id,
+ input_file=input_file,
+ )
diff --git a/backend/evals/runner.py b/backend/evals/runner.py
index 9cbb655b..604fad42 100644
--- a/backend/evals/runner.py
+++ b/backend/evals/runner.py
@@ -8,8 +8,8 @@ from llm import Llm
from config import LOCAL_ASSET_BASE_URL
from prompts.prompt_types import Stack
from agent.engine import BudgetExceededError
-from .core import generate_code_for_image
-from .sets import get_set_inputs_dir
+from .core import generate_code_for_image, generate_code_for_text
+from .sets import get_set_inputs_dir, get_set_kind, list_set_briefs
from .utils import image_to_data_url
from .config import EVALS_DIR
@@ -52,6 +52,23 @@ def _resolve_eval_filenames(
return [f for f in os.listdir(input_dir) if f.endswith(".png")]
+def _resolve_eval_items(
+ input_files: Optional[List[str]], eval_set: Optional[str]
+) -> tuple[List[str], dict[str, str]]:
+ """Items to run and, for text sets, the id -> brief text mapping.
+
+ For image sets items are PNG filenames; for text sets they are brief ids
+ (which play the same role everywhere downstream: skip keys, output file
+ stems, and the run's recorded input_file).
+ """
+ if eval_set and get_set_kind(eval_set) == "text":
+ briefs = list_set_briefs(eval_set)
+ wanted = {os.path.basename(f) for f in input_files} if input_files else None
+ items = [b.id for b in briefs if wanted is None or b.id in wanted]
+ return items, {b.id: b.brief for b in briefs}
+ return _resolve_eval_filenames(input_files, _get_input_dir(eval_set)), {}
+
+
def _output_html_filename(original_filename: str, attempt_idx: int) -> str:
return f"{os.path.splitext(original_filename)[0]}_{attempt_idx}.html"
@@ -71,7 +88,7 @@ def count_pending_eval_tasks(
eval_set: Optional[str] = None,
skip_input_files: Optional[set[str]] = None,
) -> Tuple[int, int]:
- evals = _resolve_eval_filenames(input_files, _get_input_dir(eval_set))
+ evals, _ = _resolve_eval_items(input_files, eval_set)
if not diff_mode:
return len(evals) * n, 0
@@ -105,6 +122,7 @@ async def generate_code_and_time(
attempt_idx: int,
eval_set: Optional[str] = None,
eval_session_id: Optional[str] = None,
+ brief_text: Optional[str] = None,
) -> Tuple[str, int, Optional[str], Optional[float], Optional[Exception], int]:
"""
Generates code for an image, measures the time taken, and returns identifiers
@@ -117,14 +135,24 @@ async def generate_code_and_time(
while True:
start_time = time.perf_counter()
try:
- content = await generate_code_for_image(
- image_url=image_url,
- stack=stack,
- model=model,
- eval_set=eval_set,
- eval_session_id=eval_session_id,
- input_file=original_input_filename,
- )
+ if brief_text is not None:
+ content = await generate_code_for_text(
+ text_prompt=brief_text,
+ stack=stack,
+ model=model,
+ eval_set=eval_set,
+ eval_session_id=eval_session_id,
+ input_file=original_input_filename,
+ )
+ else:
+ content = await generate_code_for_image(
+ image_url=image_url,
+ stack=stack,
+ model=model,
+ eval_set=eval_set,
+ eval_session_id=eval_session_id,
+ input_file=original_input_filename,
+ )
end_time = time.perf_counter()
duration = end_time - start_time
return (
@@ -182,8 +210,9 @@ async def run_image_evals(
eval_session_id: Optional[str] = None,
skip_input_files: Optional[set[str]] = None,
) -> List[str]:
- INPUT_DIR = _get_input_dir(eval_set)
- evals = _resolve_eval_filenames(input_files, INPUT_DIR)
+ evals, briefs_by_id = _resolve_eval_items(input_files, eval_set)
+ is_text_set = bool(briefs_by_id)
+ INPUT_DIR = "" if is_text_set else _get_input_dir(eval_set)
if not stack:
raise ValueError("No stack was provided")
@@ -238,19 +267,20 @@ async def run_image_evals(
skipped_existing_tasks += 1
continue
- if data_url is None:
+ if not is_text_set and data_url is None:
data_url = await image_to_data_url(filepath)
current_model_for_task = (
selected_model if n_idx == 0 else Llm.GPT_5_5_LOW
)
coro = generate_code_and_time(
- image_url=data_url,
+ image_url=data_url or "",
stack=stack,
model=current_model_for_task,
original_input_filename=original_filename,
attempt_idx=n_idx,
eval_set=eval_set,
eval_session_id=eval_session_id,
+ brief_text=briefs_by_id.get(original_filename),
)
task_coroutines.append(coro)
diff --git a/backend/evals/sets.py b/backend/evals/sets.py
index bb3257f6..e43d8807 100644
--- a/backend/evals/sets.py
+++ b/backend/evals/sets.py
@@ -39,6 +39,16 @@ class EvalSetImage:
tags: list[str]
+@dataclass
+class EvalSetBrief:
+ """One text-eval item: the id plays the role a filename plays for images."""
+
+ id: str
+ title: str
+ brief: str
+ tests: str
+
+
@dataclass
class EvalSetInfo:
name: str
@@ -46,6 +56,7 @@ class EvalSetInfo:
created_at: Optional[str]
notes: str
image_count: int
+ kind: str = "image" # "image" | "text"
def get_sets_dir() -> str:
@@ -70,6 +81,55 @@ def _manifest_path(set_name: str) -> str:
return os.path.join(_get_set_dir(set_name), "manifest.json")
+def _briefs_path(set_name: str) -> str:
+ return os.path.join(_get_set_dir(set_name), "briefs.json")
+
+
+_BRIEF_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]*$")
+
+
+def get_set_kind(set_name: str) -> str:
+ """"text" when the set is a briefs.json collection, else "image"."""
+ if os.path.isfile(_briefs_path(set_name)):
+ return "text"
+ return "image"
+
+
+def list_set_briefs(set_name: str) -> list[EvalSetBrief]:
+ path = _briefs_path(set_name)
+ if not os.path.isfile(path):
+ raise EvalSetNotFoundError(f"Text eval set not found: {set_name}")
+ try:
+ with open(path, "r", encoding="utf-8") as f:
+ loaded = cast(dict[str, Any], json.load(f))
+ except (OSError, json.JSONDecodeError) as exc:
+ raise EvalSetNotFoundError(f"Unreadable briefs.json for {set_name}: {exc}")
+ briefs: list[EvalSetBrief] = []
+ raw_briefs = loaded.get("briefs")
+ entries = (
+ cast(list[object], raw_briefs) if isinstance(raw_briefs, list) else []
+ )
+ for entry in entries:
+ if not isinstance(entry, dict):
+ continue
+ record = cast(dict[str, Any], entry)
+ brief_id = str(record.get("id") or "")
+ brief = str(record.get("brief") or "")
+ if not _BRIEF_ID_PATTERN.match(brief_id) or not brief:
+ raise InvalidSetNameError(
+ f"Invalid brief entry in {set_name}: id={brief_id!r}"
+ )
+ briefs.append(
+ EvalSetBrief(
+ id=brief_id,
+ title=str(record.get("title") or brief_id),
+ brief=brief,
+ tests=str(record.get("tests") or ""),
+ )
+ )
+ return briefs
+
+
def _load_manifest(set_name: str) -> dict[str, Any]:
try:
with open(_manifest_path(set_name), "r", encoding="utf-8") as f:
@@ -181,6 +241,20 @@ def list_set_images(set_name: str) -> list[EvalSetImage]:
def get_set(set_name: str) -> EvalSetInfo:
+ if get_set_kind(set_name) == "text":
+ briefs = list_set_briefs(set_name)
+ with open(_briefs_path(set_name), "r", encoding="utf-8") as f:
+ meta = cast(dict[str, Any], json.load(f))
+ return EvalSetInfo(
+ name=set_name,
+ display_name=str(meta.get("display_name") or set_name),
+ created_at=(
+ str(meta["created_at"]) if meta.get("created_at") else None
+ ),
+ notes=str(meta.get("notes") or ""),
+ image_count=len(briefs),
+ kind="text",
+ )
images = list_set_images(set_name)
manifest = _load_manifest(set_name)
return EvalSetInfo(
@@ -191,6 +265,7 @@ def get_set(set_name: str) -> EvalSetInfo:
),
notes=str(manifest.get("notes") or ""),
image_count=len(images),
+ kind="image",
)
@@ -202,7 +277,11 @@ def list_sets() -> list[EvalSetInfo]:
for entry in sorted(os.listdir(sets_dir)):
if not _SET_NAME_PATTERN.match(entry):
continue
- if not os.path.isdir(os.path.join(sets_dir, entry, "inputs")):
+ set_dir = os.path.join(sets_dir, entry)
+ if not (
+ os.path.isdir(os.path.join(set_dir, "inputs"))
+ or os.path.isfile(os.path.join(set_dir, "briefs.json"))
+ ):
continue
infos.append(get_set(entry))
return infos
diff --git a/backend/routes/eval_sets.py b/backend/routes/eval_sets.py
index 713e1864..62c4b8a2 100644
--- a/backend/routes/eval_sets.py
+++ b/backend/routes/eval_sets.py
@@ -22,6 +22,7 @@ class EvalSetSummaryModel(BaseModel):
created_at: Optional[str]
notes: str
image_count: int
+ kind: str = "image"
class EvalSetImageModel(BaseModel):
@@ -31,8 +32,16 @@ class EvalSetImageModel(BaseModel):
tags: List[str]
+class EvalSetBriefModel(BaseModel):
+ id: str
+ title: str
+ brief: str
+ tests: str
+
+
class EvalSetDetailModel(EvalSetSummaryModel):
images: List[EvalSetImageModel]
+ briefs: List[EvalSetBriefModel] = []
class EvalSessionModel(BaseModel):
@@ -70,9 +79,11 @@ class MatrixCellModel(BaseModel):
class MatrixRowModel(BaseModel):
- filename: str
- sha256: str
- image_url: str
+ filename: str # image filename, or brief id for text sets
+ sha256: Optional[str] = None
+ image_url: Optional[str] = None
+ title: Optional[str] = None
+ brief: Optional[str] = None
class SessionMatrixResponse(BaseModel):
@@ -102,6 +113,7 @@ def _set_info_to_model(info: eval_sets.EvalSetInfo) -> EvalSetSummaryModel:
created_at=info.created_at,
notes=info.notes,
image_count=info.image_count,
+ kind=info.kind,
)
@@ -124,6 +136,18 @@ async def list_eval_sets() -> List[EvalSetSummaryModel]:
async def get_eval_set(set_name: str) -> EvalSetDetailModel:
try:
info = eval_sets.get_set(set_name)
+ if info.kind == "text":
+ briefs = eval_sets.list_set_briefs(set_name)
+ return EvalSetDetailModel(
+ **_set_info_to_model(info).model_dump(),
+ images=[],
+ briefs=[
+ EvalSetBriefModel(
+ id=b.id, title=b.title, brief=b.brief, tests=b.tests
+ )
+ for b in briefs
+ ],
+ )
images = eval_sets.list_set_images(set_name)
except eval_sets.InvalidSetNameError as e:
raise HTTPException(status_code=400, detail=str(e))
@@ -218,21 +242,32 @@ async def get_session_matrix(session_id: str) -> SessionMatrixResponse:
set_missing = False
set_info: Optional[eval_sets.EvalSetInfo] = None
images: List[eval_sets.EvalSetImage] = []
+ briefs: List[eval_sets.EvalSetBrief] = []
try:
set_info = eval_sets.get_set(session.eval_set)
- images = eval_sets.list_set_images(session.eval_set)
+ if set_info.kind == "text":
+ briefs = eval_sets.list_set_briefs(session.eval_set)
+ else:
+ images = eval_sets.list_set_images(session.eval_set)
except (eval_sets.EvalSetNotFoundError, eval_sets.InvalidSetNameError):
set_missing = True
- rows = [
- MatrixRowModel(
- filename=image.filename,
- sha256=image.sha256,
- image_url=f"/eval-sets/{session.eval_set}/images/{image.filename}",
- )
- for image in images
- ]
- set_filenames = {image.filename for image in images}
+ if briefs:
+ rows = [
+ MatrixRowModel(filename=b.id, title=b.title, brief=b.brief)
+ for b in briefs
+ ]
+ set_filenames = {b.id for b in briefs}
+ else:
+ rows = [
+ MatrixRowModel(
+ filename=image.filename,
+ sha256=image.sha256,
+ image_url=f"/eval-sets/{session.eval_set}/images/{image.filename}",
+ )
+ for image in images
+ ]
+ set_filenames = {image.filename for image in images}
filenames_by_sha: Dict[str, List[str]] = {}
for image in images:
filenames_by_sha.setdefault(image.sha256, []).append(image.filename)
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index be9cef1a..360e3019 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -244,6 +244,12 @@ async def run_evals(request: RunEvalsRequest) -> List[str]:
def _count_eval_files(selected_files: List[str], set_name: Optional[str] = None) -> int:
+ if set_name and eval_sets.get_set_kind(set_name) == "text":
+ brief_ids = {b.id for b in eval_sets.list_set_briefs(set_name)}
+ if selected_files:
+ return len([f for f in selected_files if f in brief_ids])
+ return len(brief_ids)
+
if selected_files:
return len([f for f in selected_files if f.endswith(".png")])
diff --git a/backend/tests/test_eval_sessions.py b/backend/tests/test_eval_sessions.py
index 1faad2b0..99030a9c 100644
--- a/backend/tests/test_eval_sessions.py
+++ b/backend/tests/test_eval_sessions.py
@@ -357,6 +357,45 @@ async def test_session_matrix(logs_path: Path, evals_dir: Path) -> None:
assert two_claude.runs[0].is_stale is True
+async def test_session_matrix_text_set(
+ logs_path: Path, evals_dir: Path
+) -> None:
+ import json as json_module
+
+ from routes.eval_sets import get_session_matrix
+
+ set_dir = evals_dir / "sets" / "briefs-v1"
+ set_dir.mkdir(parents=True)
+ (set_dir / "briefs.json").write_text(
+ json_module.dumps(
+ {
+ "briefs": [
+ {"id": "pdp", "title": "PDP", "brief": "Build a product page."},
+ {"id": "board", "brief": "Build a kanban board."},
+ ]
+ }
+ )
+ )
+ session = create_session("briefs-v1")
+ run = _record_run(
+ eval_session=session.session_id, eval_set="briefs-v1", input_file="pdp"
+ )
+ _set_run_status(run.run_id, "completed")
+
+ matrix = await get_session_matrix(session.session_id)
+ assert not matrix.set_missing
+ assert [(r.filename, r.title) for r in matrix.rows] == [
+ ("pdp", "PDP"),
+ ("board", "board"),
+ ]
+ assert matrix.rows[0].image_url is None
+ assert matrix.rows[0].brief == "Build a product page."
+ cells = {(c.filename, c.model) for c in matrix.cells}
+ assert ("pdp", Llm.GPT_5_5_HIGH.value) in cells
+ assert matrix.unmatched_run_count == 0
+
+
@pytest.mark.asyncio
async def test_session_matrix_missing_set(
logs_path: Path, evals_dir: Path
diff --git a/backend/tests/test_eval_sets.py b/backend/tests/test_eval_sets.py
index 17952abd..cadffca9 100644
--- a/backend/tests/test_eval_sets.py
+++ b/backend/tests/test_eval_sets.py
@@ -101,6 +101,102 @@ def test_set_name_validation(evals_dir: Path) -> None:
get_set_inputs_dir(bad_name)
+def _make_text_set(evals_dir: Path, name: str) -> None:
+ set_dir = evals_dir / "sets" / name
+ set_dir.mkdir(parents=True)
+ (set_dir / "briefs.json").write_text(
+ json.dumps(
+ {
+ "display_name": "Briefs",
+ "briefs": [
+ {
+ "id": "pdp",
+ "title": "PDP",
+ "brief": "Build a product page.",
+ "tests": "hierarchy",
+ },
+ {"id": "board", "brief": "Build a kanban board."},
+ ],
+ }
+ )
+ )
+
+
+def test_text_set_kind_and_briefs(evals_dir: Path) -> None:
+ from evals.sets import get_set_kind, list_set_briefs
+
+ _make_text_set(evals_dir, "briefs-v1")
+ _make_set(evals_dir, "imgs", {"a.png": b"a"})
+
+ assert get_set_kind("briefs-v1") == "text"
+ assert get_set_kind("imgs") == "image"
+
+ briefs = list_set_briefs("briefs-v1")
+ assert [(b.id, b.title) for b in briefs] == [("pdp", "PDP"), ("board", "board")]
+
+ info = get_set("briefs-v1")
+ assert info.kind == "text"
+ assert info.image_count == 2
+
+ sets = list_sets()
+ assert {(s.name, s.kind) for s in sets} == {
+ ("briefs-v1", "text"),
+ ("imgs", "image"),
+ }
+
+
+def test_text_set_invalid_brief_id_rejected(evals_dir: Path) -> None:
+ from evals.sets import list_set_briefs
+
+ set_dir = evals_dir / "sets" / "bad"
+ set_dir.mkdir(parents=True)
+ (set_dir / "briefs.json").write_text(
+ json.dumps({"briefs": [{"id": "../evil", "brief": "x"}]})
+ )
+ with pytest.raises(InvalidSetNameError):
+ list_set_briefs("bad")
+
+
+async def test_text_set_runner_flow(
+ evals_dir: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from evals.runner import count_pending_eval_tasks, run_image_evals
+
+ _make_text_set(evals_dir, "briefs-v1")
+
+ captured: list[dict[str, object]] = []
+
+ async def fake_text_gen(**kwargs: object) -> str:
+ captured.append(kwargs)
+ return "<html>text output</html>"
+
+ monkeypatch.setattr("evals.runner.generate_code_for_text", fake_text_gen)
+
+ pending, skipped = count_pending_eval_tasks(
+ stack="html_tailwind",
+ model="gpt-5.5 (high thinking)",
+ diff_mode=True,
+ eval_set="briefs-v1",
+ skip_input_files={"pdp"},
+ )
+ assert (pending, skipped) == (1, 1)
+
+ outputs = await run_image_evals(
+ stack="html_tailwind",
+ model="gpt-5.5 (high thinking)",
+ eval_set="briefs-v1",
+ eval_session_id="sess_x",
+ diff_mode=True,
+ skip_input_files={"pdp"},
+ )
+ assert outputs == ["board_0.html"]
+ assert len(captured) == 1
+ assert captured[0]["text_prompt"] == "Build a kanban board."
+ assert captured[0]["input_file"] == "board"
+ assert captured[0]["eval_session_id"] == "sess_x"
+
+
@pytest.mark.asyncio
async def test_image_path_resolution_and_route_guards(evals_dir: Path) -> None:
_make_set(evals_dir, "s1", {"a.png": b"png-bytes"})
diff --git a/frontend/src/components/evals/EvalComparePage.tsx b/frontend/src/components/evals/EvalComparePage.tsx
index f53bc97e..76f06901 100644
--- a/frontend/src/components/evals/EvalComparePage.tsx
+++ b/frontend/src/components/evals/EvalComparePage.tsx
@@ -37,8 +37,10 @@ interface MatrixCell {
interface MatrixRow {
filename: string;
- sha256: string;
- image_url: string;
+ sha256: string | null;
+ image_url: string | null;
+ title: string | null;
+ brief: string | null;
}
interface SessionMatrix {
@@ -80,6 +82,8 @@ function EvalComparePage() {
const [imageIndex, setImageIndex] = useState(0);
// 0 = source screenshot, 1..N = selected models in order.
const [paneIndex, setPaneIndex] = useState(1);
+ const [connectionFailed, setConnectionFailed] = useState(false);
+ const [retryTick, setRetryTick] = useState(0);
const sessionId = searchParams.get("session");
@@ -130,10 +134,11 @@ function EvalComparePage() {
const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`);
const data = await response.json();
setSessions(data.sessions);
+ setConnectionFailed(false);
return data.sessions as EvalSession[];
} catch (error) {
console.error("Error fetching sessions", error);
- toast.error("Failed to load sessions.");
+ setConnectionFailed(true);
return [];
}
}, []);
@@ -179,6 +184,23 @@ function EvalComparePage() {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
+ useEffect(() => {
+ if (!connectionFailed) return;
+ const timer = window.setTimeout(() => {
+ setRetryTick((n) => n + 1);
+ fetchSessions().then((loaded) => {
+ if (loaded.length === 0) return;
+ const target =
+ (sessionId && loaded.find((s) => s.session_id === sessionId)) ||
+ loaded.find((s) => s.is_active) ||
+ loaded[0];
+ if (target && !matrix) loadMatrix(target.session_id);
+ });
+ }, 2500);
+ return () => window.clearTimeout(timer);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [connectionFailed, retryTick]);
+
const switchSession = (id: string) => {
setSearchParams({ session: id }, { replace: true });
setMatrix(null);
@@ -305,7 +327,9 @@ function EvalComparePage() {
const isActive = index === Math.min(paneIndex, panes.length - 1);
const label =
pane.model === "__source__"
- ? "Source"
+ ? currentImage?.image_url
+ ? "Source"
+ : "Brief"
: shortModelName(pane.model);
const colorClass =
pane.model === "__source__"
@@ -367,7 +391,9 @@ function EvalComparePage() {
<div className="relative min-h-0 flex-1 overflow-hidden rounded-xl border border-zinc-800 bg-white">
{!matrix && (
- <p className="py-8 text-center text-sm text-zinc-500">Loading…</p>
+ <p className="py-8 text-center text-sm text-zinc-500">
+ {connectionFailed ? "Backend unreachable — retrying…" : "Loading…"}
+ </p>
)}
{matrix && images.length === 0 && (
<p className="py-8 text-center text-sm text-zinc-500">
@@ -387,11 +413,25 @@ function EvalComparePage() {
isActive ? "visible" : "invisible"
}`}
>
- <img
- src={`${HTTP_BACKEND_URL}${currentImage.image_url}`}
- alt={currentImage.filename}
- className="w-full"
- />
+ {currentImage.image_url ? (
+ <img
+ src={`${HTTP_BACKEND_URL}${currentImage.image_url}`}
+ alt={currentImage.filename}
+ className="w-full"
+ />
+ ) : (
+ <div className="mx-auto max-w-2xl px-8 py-12 text-zinc-900">
+ <div className="text-xs font-medium uppercase tracking-wide text-zinc-400">
+ Brief
+ </div>
+ <h2 className="mt-1 text-2xl font-semibold">
+ {currentImage.title ?? currentImage.filename}
+ </h2>
+ <p className="mt-4 text-base leading-7">
+ {currentImage.brief}
+ </p>
+ </div>
+ )}
</div>
);
}
diff --git a/frontend/src/components/evals/EvalSessionsPage.tsx b/frontend/src/components/evals/EvalSessionsPage.tsx
index df299fe9..c21258e7 100644
--- a/frontend/src/components/evals/EvalSessionsPage.tsx
+++ b/frontend/src/components/evals/EvalSessionsPage.tsx
@@ -41,8 +41,10 @@ interface MatrixCell {
interface MatrixRow {
filename: string;
- sha256: string;
- image_url: string;
+ sha256: string | null;
+ image_url: string | null;
+ title: string | null;
+ brief: string | null;
}
interface SessionMatrix {
@@ -162,6 +164,8 @@ function EvalSessionsPage() {
const [newSessionSet, setNewSessionSet] = useState("");
const [newSessionName, setNewSessionName] = useState("");
const [isCreating, setIsCreating] = useState(false);
+ const [connectionFailed, setConnectionFailed] = useState(false);
+ const [retryTick, setRetryTick] = useState(0);
const [editingNotesModel, setEditingNotesModel] = useState<string | null>(
null
);
@@ -202,10 +206,14 @@ function EvalSessionsPage() {
const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`);
const data = await response.json();
setSessions(data.sessions);
+ setConnectionFailed(false);
return data.sessions;
} catch (error) {
+ // Dev backends restart (--reload) and briefly refuse connections;
+ // flag it and let the retry effect re-attempt instead of settling
+ // into a misleading empty state.
console.error("Error fetching sessions", error);
- toast.error("Failed to load sessions.");
+ setConnectionFailed(true);
return [];
}
}, []);
@@ -215,14 +223,32 @@ function EvalSessionsPage() {
const response = await fetch(`${HTTP_BACKEND_URL}/eval-sets`);
const data: EvalSetSummary[] = await response.json();
setSets(data);
+ setConnectionFailed(false);
if (data.length > 0) {
setNewSessionSet((current) => current || data[0].name);
}
} catch (error) {
console.error("Error fetching eval sets", error);
+ setConnectionFailed(true);
}
}, []);
+ useEffect(() => {
+ if (!connectionFailed) return;
+ const timer = window.setTimeout(() => {
+ setRetryTick((n) => n + 1);
+ fetchSets();
+ fetchSessions().then((loaded) => {
+ if (loaded.length > 0 && !selectedSessionId) {
+ const active = loaded.find((s) => s.is_active) ?? loaded[0];
+ loadMatrix(active.session_id);
+ }
+ });
+ }, 2500);
+ return () => window.clearTimeout(timer);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [connectionFailed, retryTick]);
+
const loadMatrix = useCallback(async (sessionId: string) => {
setSelectedSessionId(sessionId);
setIsLoadingMatrix(true);
@@ -419,7 +445,12 @@ function EvalSessionsPage() {
<div className="px-2 py-1 text-xs uppercase tracking-wide text-zinc-500">
{sessions.length} session{sessions.length === 1 ? "" : "s"}
</div>
- {sessions.length === 0 && (
+ {connectionFailed && (
+ <p className="px-2 py-4 text-sm text-amber-300">
+ Backend unreachable — retrying…
+ </p>
+ )}
+ {!connectionFailed && sessions.length === 0 && (
<p className="px-2 py-4 text-sm text-zinc-400">
No sessions yet. Pick a set above and start one, or just run
evals with a set — a session is created automatically.
@@ -600,26 +631,40 @@ function EvalSessionsPage() {
{sortedRows.map((row) => (
<tr key={row.filename}>
<td className="sticky left-0 z-10 bg-zinc-900/95 px-2 py-1">
- <div className="flex items-center gap-2">
- <img
- src={`${HTTP_BACKEND_URL}${row.image_url}`}
- alt={row.filename}
- className="h-12 w-16 rounded border border-zinc-800 bg-zinc-800 object-cover"
- loading="lazy"
- />
- <span className="max-w-[160px] truncate font-mono text-[11px] text-zinc-300">
- {row.filename}
- </span>
- <a
- href={`${HTTP_BACKEND_URL}${row.image_url}`}
- target="_blank"
- rel="noreferrer"
- title="Open image in new tab"
- className="shrink-0 rounded p-1 text-zinc-600 hover:bg-zinc-700 hover:text-zinc-200"
+ {row.image_url ? (
+ <div className="flex items-center gap-2">
+ <img
+ src={`${HTTP_BACKEND_URL}${row.image_url}`}
+ alt={row.filename}
+ className="h-12 w-16 rounded border border-zinc-800 bg-zinc-800 object-cover"
+ loading="lazy"
+ />
+ <span className="max-w-[160px] truncate font-mono text-[11px] text-zinc-300">
+ {row.filename}
+ </span>
+ <a
+ href={`${HTTP_BACKEND_URL}${row.image_url}`}
+ target="_blank"
+ rel="noreferrer"
+ title="Open image in new tab"
+ className="shrink-0 rounded p-1 text-zinc-600 hover:bg-zinc-700 hover:text-zinc-200"
+ >
+ <BsBoxArrowUpRight className="text-[11px]" />
+ </a>
+ </div>
+ ) : (
+ <div
+ className="max-w-[220px]"
+ title={row.brief ?? row.filename}
>
- <BsBoxArrowUpRight className="text-[11px]" />
- </a>
- </div>
+ <div className="truncate text-[11px] font-medium text-zinc-200">
+ {row.title ?? row.filename}
+ </div>
+ <div className="line-clamp-2 text-[10px] leading-tight text-zinc-500">
+ {row.brief}
+ </div>
+ </div>
+ )}
</td>
{matrix.models.map((model) => {
const cell = cellsByKey.get(
diff --git a/frontend/src/components/evals/RunEvalsPage.tsx b/frontend/src/components/evals/RunEvalsPage.tsx
index 2923cc62..2d1fb2b1 100644
--- a/frontend/src/components/evals/RunEvalsPage.tsx
+++ b/frontend/src/components/evals/RunEvalsPage.tsx
@@ -128,10 +128,12 @@ function RunEvalsPage() {
);
if (!response.ok) throw new Error("Request failed");
const data = await response.json();
- setSetImages(data.images);
- setSelectedSetFiles(
- data.images.map((image: EvalSetImage) => image.filename)
- );
+ const items: EvalSetImage[] =
+ data.kind === "text"
+ ? data.briefs.map((b: { id: string }) => ({ filename: b.id }))
+ : data.images;
+ setSetImages(items);
+ setSelectedSetFiles(items.map((item) => item.filename));
} catch (error) {
console.error("Error fetching set images", error);
setSetImages([]);