Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add eval sets, sessions, and per-session matrix view

Abi Raja18de719bb0ba
diff --git a/backend/agent/providers/anthropic/provider.py b/backend/agent/providers/anthropic/provider.py
index 7b5f0f31..5c48ebcd 100644
--- a/backend/agent/providers/anthropic/provider.py
+++ b/backend/agent/providers/anthropic/provider.py
@@ -22,8 +22,8 @@ from agent.providers.anthropic.image import (
process_image,
process_image_bytes,
)
-from agent.providers.pricing import MODEL_PRICING
-from agent.providers.token_usage import TokenUsage
+from costs.pricing import MODEL_PRICING
+from costs.token_usage import TokenUsage
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
from fs_logging.agent_runs import AgentRunRecorder
from fs_logging.prompt_reports import PromptReportLogger
diff --git a/backend/agent/providers/gemini.py b/backend/agent/providers/gemini.py
index 60a53593..354c7589 100644
--- a/backend/agent/providers/gemini.py
+++ b/backend/agent/providers/gemini.py
@@ -18,8 +18,8 @@ from agent.providers.base import (
ProviderTurn,
StreamEvent,
)
-from agent.providers.pricing import MODEL_PRICING
-from agent.providers.token_usage import TokenUsage
+from costs.pricing import MODEL_PRICING
+from costs.token_usage import TokenUsage
from agent.tools import CanonicalToolDefinition, ToolCall
from fs_logging.agent_runs import AgentRunRecorder
from fs_logging.prompt_reports import PromptReportLogger
diff --git a/backend/agent/providers/openai.py b/backend/agent/providers/openai.py
index 2214c46b..340a693e 100644
--- a/backend/agent/providers/openai.py
+++ b/backend/agent/providers/openai.py
@@ -16,8 +16,8 @@ from agent.providers.base import (
ProviderTurn,
StreamEvent,
)
-from agent.providers.pricing import MODEL_PRICING
-from agent.providers.token_usage import TokenUsage
+from costs.pricing import MODEL_PRICING
+from costs.token_usage import TokenUsage
from agent.state import ensure_str
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
from fs_logging.agent_runs import AgentRunRecorder
@@ -423,7 +423,7 @@ class OpenAIProviderSession(ProviderSession):
model: Llm,
prompt_messages: List[ChatCompletionMessageParam],
tools: List[Dict[str, Any]],
- recorder: Optional["AgentRunRecorder"] = None,
+ recorder: Optional[AgentRunRecorder] = None,
):
self._client = client
self._model = model
diff --git a/backend/costs/__init__.py b/backend/costs/__init__.py
new file mode 100644
index 00000000..e86536ac
--- /dev/null
+++ b/backend/costs/__init__.py
@@ -0,0 +1,6 @@
+"""Model pricing and token accounting.
+
+Leaf package: it must not import from agent/, fs_logging/, or routes/ so
+that any layer (providers, logging, evals) can depend on it without
+creating import cycles. Keep this __init__ empty for the same reason.
+"""
diff --git a/backend/agent/providers/pricing.py b/backend/costs/pricing.py
similarity index 100%
rename from backend/agent/providers/pricing.py
rename to backend/costs/pricing.py
diff --git a/backend/agent/providers/token_usage.py b/backend/costs/token_usage.py
similarity index 98%
rename from backend/agent/providers/token_usage.py
rename to backend/costs/token_usage.py
index 2141a3c4..a9b0619e 100644
--- a/backend/agent/providers/token_usage.py
+++ b/backend/costs/token_usage.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass
-from agent.providers.pricing import ModelPricing
+from costs.pricing import ModelPricing
@dataclass
diff --git a/backend/evals/core.py b/backend/evals/core.py
index 8e59eb9d..b938e1aa 100644
--- a/backend/evals/core.py
+++ b/backend/evals/core.py
@@ -18,7 +18,15 @@ from openai.types.chat import ChatCompletionMessageParam
from typing import Any
-async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> str:
+async def generate_code_for_image(
+ image_url: str,
+ stack: Stack,
+ model: Llm,
+ *,
+ eval_set: str | None = None,
+ eval_session_id: str | None = None,
+ input_file: str | None = None,
+) -> str:
prompt_messages = build_image_prompt_messages(
image_data_urls=[image_url],
stack=stack,
@@ -53,6 +61,9 @@ async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> s
stack=str(stack),
input_mode="image",
generation_type="create",
+ eval_session=eval_session_id,
+ eval_set=eval_set,
+ input_file=input_file,
)
runner = Agent(
send_message=send_message,
diff --git a/backend/evals/runner.py b/backend/evals/runner.py
index aa4b1181..d7673831 100644
--- a/backend/evals/runner.py
+++ b/backend/evals/runner.py
@@ -8,6 +8,7 @@ from llm import Llm
from config import LOCAL_ASSET_BASE_URL
from prompts.prompt_types import Stack
from .core import generate_code_for_image
+from .sets import get_set_inputs_dir
from .utils import image_to_data_url
from .config import EVALS_DIR
@@ -36,8 +37,15 @@ def normalize_local_asset_urls(html: str) -> str:
return html
-def _resolve_eval_filenames(input_files: Optional[List[str]]) -> List[str]:
- input_dir = EVALS_DIR + "/inputs"
+def _get_input_dir(eval_set: Optional[str]) -> str:
+ if eval_set:
+ return get_set_inputs_dir(eval_set)
+ return EVALS_DIR + "/inputs"
+
+
+def _resolve_eval_filenames(
+ input_files: Optional[List[str]], input_dir: str
+) -> List[str]:
if input_files and len(input_files) > 0:
return [os.path.basename(f) for f in input_files if f.endswith(".png")]
return [f for f in os.listdir(input_dir) if f.endswith(".png")]
@@ -59,11 +67,21 @@ def count_pending_eval_tasks(
input_files: Optional[List[str]] = None,
n: int = 1,
diff_mode: bool = False,
+ eval_set: Optional[str] = None,
+ skip_input_files: Optional[set[str]] = None,
) -> Tuple[int, int]:
- evals = _resolve_eval_filenames(input_files)
+ evals = _resolve_eval_filenames(input_files, _get_input_dir(eval_set))
if not diff_mode:
return len(evals) * n, 0
+ # Set runs skip by session history (the caller queries the run index);
+ # day-named output folders are shared across sets, so file existence
+ # would produce false positives there.
+ if eval_set is not None:
+ skip = skip_input_files or set()
+ skipped = sum(n for f in evals if f in skip)
+ return len(evals) * n - skipped, skipped
+
output_subfolder = get_eval_output_subfolder(stack=stack, model=model)
pending_tasks = 0
skipped_existing_tasks = 0
@@ -84,6 +102,8 @@ async def generate_code_and_time(
model: Llm,
original_input_filename: str,
attempt_idx: int,
+ eval_set: Optional[str] = None,
+ eval_session_id: Optional[str] = None,
) -> Tuple[str, int, Optional[str], Optional[float], Optional[Exception], int]:
"""
Generates code for an image, measures the time taken, and returns identifiers
@@ -97,7 +117,12 @@ async def generate_code_and_time(
start_time = time.perf_counter()
try:
content = await generate_code_for_image(
- image_url=image_url, stack=stack, model=model
+ image_url=image_url,
+ stack=stack,
+ model=model,
+ eval_set=eval_set,
+ eval_session_id=eval_session_id,
+ input_file=original_input_filename,
)
end_time = time.perf_counter()
duration = end_time - start_time
@@ -137,9 +162,12 @@ async def run_image_evals(
input_files: Optional[List[str]] = None,
diff_mode: bool = False,
progress_callback: Optional[Callable[[dict[str, Any]], Any | Awaitable[Any]]] = None,
+ eval_set: Optional[str] = None,
+ eval_session_id: Optional[str] = None,
+ skip_input_files: Optional[set[str]] = None,
) -> List[str]:
- INPUT_DIR = EVALS_DIR + "/inputs"
- evals = _resolve_eval_filenames(input_files)
+ INPUT_DIR = _get_input_dir(eval_set)
+ evals = _resolve_eval_filenames(input_files, INPUT_DIR)
if not stack:
raise ValueError("No stack was provided")
@@ -182,9 +210,17 @@ async def run_image_evals(
for n_idx in range(n):
output_filename = _output_html_filename(original_filename, n_idx)
output_path = os.path.join(output_subfolder, output_filename)
- if diff_mode and os.path.exists(output_path):
- skipped_existing_tasks += 1
- continue
+ if diff_mode:
+ # Set runs skip by session history (authoritative — shared
+ # day-named output folders collide across sets); the legacy
+ # path keeps the original file-existence check.
+ if eval_set is not None:
+ if original_filename in (skip_input_files or set()):
+ skipped_existing_tasks += 1
+ continue
+ elif os.path.exists(output_path):
+ skipped_existing_tasks += 1
+ continue
if data_url is None:
data_url = await image_to_data_url(filepath)
@@ -197,6 +233,8 @@ async def run_image_evals(
model=current_model_for_task,
original_input_filename=original_filename,
attempt_idx=n_idx,
+ eval_set=eval_set,
+ eval_session_id=eval_session_id,
)
task_coroutines.append(coro)
diff --git a/backend/evals/sessions.py b/backend/evals/sessions.py
new file mode 100644
index 00000000..2610fb70
--- /dev/null
+++ b/backend/evals/sessions.py
@@ -0,0 +1,178 @@
+"""Eval sessions: a work period pinned to one eval set.
+
+Exactly one session is active at a time; starting a new one supersedes the
+old (sessions are never "ended"). Eval runs launched while a session is
+active attach to it via the agent-run recorder, which makes the session
+matrix (images x models) a plain query over the runs index. Sessions never
+gate the UI generation path — they are pure metadata.
+
+Stored in the agent-runs SQLite index (see ``fs_logging/agent_runs.py`` for
+the schema). Timestamps use the same local-ISO format as ``runs.created_at``
+so SQL string comparison against run timestamps is valid.
+"""
+
+import sqlite3
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+from fs_logging.agent_runs import open_index_db
+
+
+@dataclass
+class EvalSession:
+ session_id: str
+ name: str
+ eval_set: str
+ created_at: str
+ is_active: bool
+
+
+class SessionSetMismatchError(Exception):
+ def __init__(self, active_session: EvalSession, requested_set: str) -> None:
+ self.active_session = active_session
+ self.requested_set = requested_set
+ super().__init__(
+ f"Active session {active_session.name!r} is pinned to set "
+ f"{active_session.eval_set!r}, not {requested_set!r}"
+ )
+
+
+_SESSION_COLUMNS = "session_id, name, eval_set, created_at, is_active"
+
+
+def _row_to_session(row: tuple[str, str, str, str, int]) -> EvalSession:
+ return EvalSession(
+ session_id=row[0],
+ name=row[1],
+ eval_set=row[2],
+ created_at=row[3],
+ is_active=bool(row[4]),
+ )
+
+
+def default_session_name() -> str:
+ now = datetime.now()
+ # %-d is not portable; strip the leading zero manually.
+ return f"{now.strftime('%b')} {now.day} session"
+
+
+def _activate_in_transaction(conn: sqlite3.Connection, session_id: str) -> None:
+ conn.execute("BEGIN IMMEDIATE")
+ conn.execute("UPDATE eval_sessions SET is_active = 0 WHERE is_active = 1")
+ conn.execute(
+ "UPDATE eval_sessions SET is_active = 1 WHERE session_id = ?",
+ (session_id,),
+ )
+ conn.commit()
+
+
+def create_session(eval_set: str, name: Optional[str] = None) -> EvalSession:
+ session = EvalSession(
+ session_id=(
+ f"sess_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
+ ),
+ name=name or default_session_name(),
+ eval_set=eval_set,
+ created_at=datetime.now().isoformat(timespec="seconds"),
+ is_active=True,
+ )
+ conn = open_index_db()
+ try:
+ conn.execute(
+ "INSERT INTO eval_sessions "
+ f"({_SESSION_COLUMNS}) VALUES (?, ?, ?, ?, 0)",
+ (
+ session.session_id,
+ session.name,
+ session.eval_set,
+ session.created_at,
+ ),
+ )
+ conn.commit()
+ try:
+ _activate_in_transaction(conn, session.session_id)
+ except sqlite3.IntegrityError:
+ # Concurrent activation raced the partial unique index; retry once.
+ _activate_in_transaction(conn, session.session_id)
+ finally:
+ conn.close()
+ return session
+
+
+def list_sessions() -> list[EvalSession]:
+ conn = open_index_db()
+ try:
+ rows = conn.execute(
+ f"SELECT {_SESSION_COLUMNS} FROM eval_sessions "
+ "ORDER BY created_at DESC, session_id DESC"
+ ).fetchall()
+ finally:
+ conn.close()
+ return [_row_to_session(row) for row in rows]
+
+
+def get_session(session_id: str) -> Optional[EvalSession]:
+ conn = open_index_db()
+ try:
+ row = conn.execute(
+ f"SELECT {_SESSION_COLUMNS} FROM eval_sessions WHERE session_id = ?",
+ (session_id,),
+ ).fetchone()
+ finally:
+ conn.close()
+ return _row_to_session(row) if row else None
+
+
+def get_active_session() -> Optional[EvalSession]:
+ conn = open_index_db()
+ try:
+ row = conn.execute(
+ f"SELECT {_SESSION_COLUMNS} FROM eval_sessions WHERE is_active = 1"
+ ).fetchone()
+ finally:
+ conn.close()
+ return _row_to_session(row) if row else None
+
+
+def activate_session(session_id: str) -> Optional[EvalSession]:
+ conn = open_index_db()
+ try:
+ try:
+ _activate_in_transaction(conn, session_id)
+ except sqlite3.IntegrityError:
+ _activate_in_transaction(conn, session_id)
+ finally:
+ conn.close()
+ return get_session(session_id)
+
+
+def resolve_session_for_run(set_name: str) -> EvalSession:
+ """The session eval runs should attach to for this set.
+
+ Active session on the same set wins; a mismatched active session is an
+ error (the caller must explicitly start a new session); no session at
+ all auto-creates one so a bare "/run_evals with a set" just works.
+ """
+ active = get_active_session()
+ if active is None:
+ return create_session(set_name)
+ if active.eval_set != set_name:
+ raise SessionSetMismatchError(active, set_name)
+ return active
+
+
+def completed_eval_inputs(session_id: str, model: str, stack: str) -> set[str]:
+ """Input filenames already completed in this session for (model, stack)."""
+ conn = open_index_db()
+ try:
+ rows = conn.execute(
+ "SELECT DISTINCT input_file FROM runs "
+ "WHERE eval_session = ? AND model = ? AND stack = ? "
+ "AND status = 'completed' AND input_file IS NOT NULL",
+ (session_id, model, stack),
+ ).fetchall()
+ finally:
+ conn.close()
+ return {row[0] for row in rows}
diff --git a/backend/evals/sets.py b/backend/evals/sets.py
new file mode 100644
index 00000000..bb3257f6
--- /dev/null
+++ b/backend/evals/sets.py
@@ -0,0 +1,224 @@
+"""Named eval sets: folders of input images under ``{EVALS_DIR}/sets``.
+
+A set is created by dropping PNGs into ``sets/{name}/inputs/`` — there is no
+creation API. ``manifest.json`` (bootstrapped on first listing) carries the
+display name, notes, and a per-image sha256 cache keyed by (size, mtime) so
+content hashes are computed once. The sha256 of an image's bytes is how UI
+generations are matched to matrix rows.
+"""
+
+import hashlib
+import json
+import os
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Optional, cast
+
+import evals.config as evals_config
+
+# Blocks path traversal while allowing human-friendly names like
+# "jun-21-evals" or "Landing Pages v2".
+_SET_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._ -]*$")
+
+
+class InvalidSetNameError(Exception):
+ pass
+
+
+class EvalSetNotFoundError(Exception):
+ pass
+
+
+@dataclass
+class EvalSetImage:
+ filename: str
+ sha256: str
+ size_bytes: int
+ mtime: float
+ tags: list[str]
+
+
+@dataclass
+class EvalSetInfo:
+ name: str
+ display_name: str
+ created_at: Optional[str]
+ notes: str
+ image_count: int
+
+
+def get_sets_dir() -> str:
+ return os.path.join(evals_config.EVALS_DIR, "sets")
+
+
+def _validate_set_name(set_name: str) -> str:
+ if not _SET_NAME_PATTERN.match(set_name):
+ raise InvalidSetNameError(f"Invalid eval set name: {set_name!r}")
+ return set_name
+
+
+def _get_set_dir(set_name: str) -> str:
+ return os.path.join(get_sets_dir(), _validate_set_name(set_name))
+
+
+def get_set_inputs_dir(set_name: str) -> str:
+ return os.path.join(_get_set_dir(set_name), "inputs")
+
+
+def _manifest_path(set_name: str) -> str:
+ return os.path.join(_get_set_dir(set_name), "manifest.json")
+
+
+def _load_manifest(set_name: str) -> dict[str, Any]:
+ try:
+ with open(_manifest_path(set_name), "r", encoding="utf-8") as f:
+ loaded = cast(Any, json.load(f))
+ if isinstance(loaded, dict):
+ return cast(dict[str, Any], loaded)
+ return {}
+ except (OSError, json.JSONDecodeError):
+ return {}
+
+
+def _tags_of(entry: Optional[dict[str, Any]]) -> list[str]:
+ if not entry:
+ return []
+ raw = entry.get("tags")
+ if not isinstance(raw, list):
+ return []
+ return [tag for tag in cast(list[object], raw) if isinstance(tag, str)]
+
+
+def _save_manifest(set_name: str, manifest: dict[str, Any]) -> None:
+ try:
+ with open(_manifest_path(set_name), "w", encoding="utf-8") as f:
+ json.dump(manifest, f, indent=2, ensure_ascii=False)
+ except OSError as exc:
+ # The manifest is a cache/metadata sidecar; failing to persist it
+ # must not fail set operations.
+ print(f"[EVAL SETS] Failed to write manifest for {set_name}: {exc}")
+
+
+def _sha256_of_file(path: str) -> str:
+ digest = hashlib.sha256()
+ with open(path, "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def list_set_images(set_name: str) -> list[EvalSetImage]:
+ """PNGs of the set, hashes served from the manifest cache when fresh."""
+ inputs_dir = get_set_inputs_dir(set_name)
+ if not os.path.isdir(inputs_dir):
+ raise EvalSetNotFoundError(f"Eval set not found: {set_name}")
+
+ manifest = _load_manifest(set_name)
+ raw_images = manifest.get("images")
+ cached_images: dict[str, Any] = (
+ cast(dict[str, Any], raw_images) if isinstance(raw_images, dict) else {}
+ )
+ manifest_changed = not manifest
+
+ filenames = sorted(
+ entry
+ for entry in os.listdir(inputs_dir)
+ if entry.lower().endswith(".png")
+ and os.path.isfile(os.path.join(inputs_dir, entry))
+ )
+
+ images: list[EvalSetImage] = []
+ fresh_entries: dict[str, Any] = {}
+ for filename in filenames:
+ path = os.path.join(inputs_dir, filename)
+ stat = os.stat(path)
+ raw_cached = cached_images.get(filename)
+ cached: Optional[dict[str, Any]] = (
+ cast(dict[str, Any], raw_cached)
+ if isinstance(raw_cached, dict)
+ else None
+ )
+ tags = _tags_of(cached)
+ if (
+ cached is not None
+ and cached.get("size") == stat.st_size
+ and cached.get("mtime") == stat.st_mtime
+ and isinstance(cached.get("sha256"), str)
+ ):
+ sha256 = str(cached["sha256"])
+ else:
+ sha256 = _sha256_of_file(path)
+ manifest_changed = True
+ fresh_entries[filename] = {
+ "sha256": sha256,
+ "size": stat.st_size,
+ "mtime": stat.st_mtime,
+ "tags": tags,
+ }
+ images.append(
+ EvalSetImage(
+ filename=filename,
+ sha256=sha256,
+ size_bytes=stat.st_size,
+ mtime=stat.st_mtime,
+ tags=tags,
+ )
+ )
+
+ if set(fresh_entries) != set(cached_images):
+ manifest_changed = True
+ if manifest_changed:
+ manifest.setdefault("display_name", set_name)
+ manifest.setdefault(
+ "created_at", datetime.now().isoformat(timespec="seconds")
+ )
+ manifest.setdefault("notes", "")
+ manifest["images"] = fresh_entries
+ _save_manifest(set_name, manifest)
+
+ return images
+
+
+def get_set(set_name: str) -> EvalSetInfo:
+ images = list_set_images(set_name)
+ manifest = _load_manifest(set_name)
+ return EvalSetInfo(
+ name=set_name,
+ display_name=str(manifest.get("display_name") or set_name),
+ created_at=(
+ str(manifest["created_at"]) if manifest.get("created_at") else None
+ ),
+ notes=str(manifest.get("notes") or ""),
+ image_count=len(images),
+ )
+
+
+def list_sets() -> list[EvalSetInfo]:
+ sets_dir = get_sets_dir()
+ if not os.path.isdir(sets_dir):
+ return []
+ infos: list[EvalSetInfo] = []
+ for entry in sorted(os.listdir(sets_dir)):
+ if not _SET_NAME_PATTERN.match(entry):
+ continue
+ if not os.path.isdir(os.path.join(sets_dir, entry, "inputs")):
+ continue
+ infos.append(get_set(entry))
+ return infos
+
+
+def resolve_set_image_path(set_name: str, filename: str) -> str:
+ """Absolute path of a set image; raises on traversal or missing file."""
+ inputs_dir = get_set_inputs_dir(set_name)
+ if not os.path.isdir(inputs_dir):
+ raise EvalSetNotFoundError(f"Eval set not found: {set_name}")
+ safe_name = os.path.basename(filename)
+ if not safe_name.lower().endswith(".png"):
+ raise InvalidSetNameError(f"Not a set image: {filename!r}")
+ path = os.path.realpath(os.path.join(inputs_dir, safe_name))
+ if not path.startswith(os.path.realpath(inputs_dir) + os.sep):
+ raise InvalidSetNameError(f"Invalid image path: {filename!r}")
+ if not os.path.isfile(path):
+ raise FileNotFoundError(path)
+ return path
diff --git a/backend/fs_logging/agent_runs.py b/backend/fs_logging/agent_runs.py
index 5b355de7..fbdd3910 100644
--- a/backend/fs_logging/agent_runs.py
+++ b/backend/fs_logging/agent_runs.py
@@ -22,6 +22,7 @@ Recording must never break generation: every public method no-ops when
disabled and swallows its own exceptions.
"""
+import base64
import hashlib
import json
import os
@@ -31,13 +32,15 @@ import sqlite3
import time
import uuid
from datetime import datetime
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, List, Optional, cast
from urllib.parse import unquote, urlparse
+from openai.types.chat import ChatCompletionMessageParam
+
from config import LOCAL_ASSET_DIR, PROMPT_REPORTS_ENABLED
from llm import MODEL_PROVIDER, Llm
-from agent.providers.pricing import MODEL_PRICING
-from agent.providers.token_usage import TokenUsage
+from costs.pricing import MODEL_PRICING
+from costs.token_usage import TokenUsage
from fs_logging.prompt_reports import get_run_logs_directory, to_serializable
if TYPE_CHECKING:
@@ -100,10 +103,23 @@ CREATE TABLE IF NOT EXISTS runs (
total_cost_usd REAL,
has_unpriced_calls INTEGER NOT NULL DEFAULT 0,
run_dir TEXT NOT NULL,
- size_bytes INTEGER
+ size_bytes INTEGER,
+ eval_session TEXT,
+ eval_set TEXT,
+ input_file TEXT,
+ input_image_sha256 TEXT
);
CREATE INDEX IF NOT EXISTS idx_runs_generation ON runs(generation_id);
CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(created_at);
+CREATE TABLE IF NOT EXISTS eval_sessions (
+ session_id TEXT PRIMARY KEY,
+ name TEXT NOT NULL,
+ eval_set TEXT NOT NULL,
+ created_at TEXT NOT NULL,
+ is_active INTEGER NOT NULL DEFAULT 0
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_eval_sessions_active
+ ON eval_sessions(is_active) WHERE is_active = 1;
CREATE TABLE IF NOT EXISTS llm_calls (
run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE,
step INTEGER NOT NULL,
@@ -123,6 +139,27 @@ CREATE TABLE IF NOT EXISTS llm_calls (
"""
+def _migrate_runs_table(conn: sqlite3.Connection) -> None:
+ """Add columns introduced after the first release.
+
+ ``CREATE TABLE IF NOT EXISTS`` never alters an existing table, so a
+ populated index.db from before these columns needs explicit ALTERs. The
+ matching indexes must live here too — creating them in ``_SCHEMA`` would
+ fail against an un-migrated table.
+ """
+ existing = {row[1] for row in conn.execute("PRAGMA table_info(runs)")}
+ for column in ("eval_session", "eval_set", "input_file", "input_image_sha256"):
+ if column not in existing:
+ conn.execute(f"ALTER TABLE runs ADD COLUMN {column} TEXT")
+ conn.execute(
+ "CREATE INDEX IF NOT EXISTS idx_runs_eval_session ON runs(eval_session)"
+ )
+ conn.execute(
+ "CREATE INDEX IF NOT EXISTS idx_runs_input_sha ON runs(input_image_sha256)"
+ )
+ conn.commit()
+
+
def open_index_db() -> sqlite3.Connection:
os.makedirs(get_agent_runs_directory(), exist_ok=True)
conn = sqlite3.connect(get_agent_runs_db_path(), timeout=5)
@@ -131,6 +168,7 @@ def open_index_db() -> sqlite3.Connection:
conn.execute("PRAGMA busy_timeout=5000")
conn.execute("PRAGMA foreign_keys=ON")
conn.executescript(_SCHEMA)
+ _migrate_runs_table(conn)
return conn
@@ -138,6 +176,41 @@ def _now_iso() -> str:
return datetime.now().isoformat(timespec="seconds")
+def _first_input_image_sha256(prompt_messages: Any) -> Optional[str]:
+ """Digest of the first input image's raw bytes, or None.
+
+ Mirrors ``AgentEngine._extract_input_images`` — duplicated because this
+ module must not import ``agent.engine`` (the engine imports us). Since
+ eval inputs are base64-encoded file bytes verbatim, this digest equals
+ the sha256 of the PNG on disk, which is the matrix-matching invariant.
+ """
+ try:
+ messages = cast(List[ChatCompletionMessageParam], prompt_messages or [])
+ for message in messages:
+ if not isinstance(message, dict):
+ continue
+ content = message.get("content")
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if not isinstance(part, dict) or part.get("type") != "image_url":
+ continue
+ image_url = part.get("image_url")
+ if not isinstance(image_url, dict):
+ continue
+ url = cast(object, image_url.get("url"))
+ if (
+ isinstance(url, str)
+ and url.startswith("data:image/")
+ and "," in url
+ ):
+ payload = base64.b64decode(url.split(",", 1)[1])
+ return hashlib.sha256(payload).hexdigest()
+ except Exception:
+ return None
+ return None
+
+
def _guess_extension(url: str, content_type: Optional[str]) -> str:
path_ext = os.path.splitext(urlparse(url).path)[1].lower()
if path_ext in _IMAGE_EXTENSIONS:
@@ -163,6 +236,9 @@ class AgentRunRecorder:
input_mode: Optional[str] = None,
generation_type: Optional[str] = None,
enabled: Optional[bool] = None,
+ eval_session: Optional[str] = None,
+ eval_set: Optional[str] = None,
+ input_file: Optional[str] = None,
) -> None:
self.enabled = PROMPT_REPORTS_ENABLED if enabled is None else enabled
self.generation_id = generation_id
@@ -171,6 +247,10 @@ class AgentRunRecorder:
self.stack = stack
self.input_mode = input_mode
self.generation_type = generation_type
+ self.eval_session = eval_session
+ self.eval_set = eval_set
+ self.input_file = input_file
+ self._input_image_sha256: Optional[str] = None
self.run_id = (
f"run_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
)
@@ -249,6 +329,7 @@ class AgentRunRecorder:
self._run_started_mono = time.perf_counter()
self._model = model.value
self._provider = MODEL_PROVIDER.get(model)
+ self._input_image_sha256 = _first_input_image_sha256(prompt_messages)
self._append_event(
"run_start",
{
@@ -261,6 +342,10 @@ class AgentRunRecorder:
"stack": self.stack,
"input_mode": self.input_mode,
"generation_type": self.generation_type,
+ "eval_session": self.eval_session,
+ "eval_set": self.eval_set,
+ "input_file": self.input_file,
+ "input_image_sha256": self._input_image_sha256,
"prompt_messages": prompt_messages,
},
)
@@ -268,8 +353,9 @@ class AgentRunRecorder:
"""INSERT OR REPLACE INTO runs
(run_id, generation_id, variant_index, entry_point, provider,
model, stack, input_mode, generation_type, status,
- created_at, run_dir)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', ?, ?)""",
+ created_at, run_dir, eval_session, eval_set, input_file,
+ input_image_sha256)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', ?, ?, ?, ?, ?, ?)""",
(
self.run_id,
self.generation_id,
@@ -282,6 +368,10 @@ class AgentRunRecorder:
self.generation_type,
_now_iso(),
self.run_dir,
+ self.eval_session,
+ self.eval_set,
+ self.input_file,
+ self._input_image_sha256,
),
)
except Exception as exc:
@@ -594,6 +684,10 @@ class AgentRunRecorder:
"stack": self.stack,
"input_mode": self.input_mode,
"generation_type": self.generation_type,
+ "eval_session": self.eval_session,
+ "eval_set": self.eval_set,
+ "input_file": self.input_file,
+ "input_image_sha256": self._input_image_sha256,
"status": status,
"error": error,
"completed_at": _now_iso(),
diff --git a/backend/fs_logging/prompt_reports.py b/backend/fs_logging/prompt_reports.py
index 37879b93..cbacd587 100644
--- a/backend/fs_logging/prompt_reports.py
+++ b/backend/fs_logging/prompt_reports.py
@@ -19,8 +19,8 @@ from datetime import datetime
from enum import Enum
from typing import Any
-from agent.providers.pricing import MODEL_PRICING
-from agent.providers.token_usage import TokenUsage
+from costs.pricing import MODEL_PRICING
+from costs.token_usage import TokenUsage
from agent.state import ensure_str
from config import PROMPT_REPORTS_ENABLED
from llm import Llm
diff --git a/backend/main.py b/backend/main.py
index 3a4b8541..aa83aa91 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -16,6 +16,7 @@ from routes import (
design_systems,
prompt_reports,
agent_runs,
+ eval_sets,
)
from uploaded_assets import configure_uploaded_asset_routes
@@ -55,3 +56,4 @@ app.include_router(export.router)
app.include_router(design_systems.router)
app.include_router(prompt_reports.router)
app.include_router(agent_runs.router)
+app.include_router(eval_sets.router)
diff --git a/backend/routes/agent_runs.py b/backend/routes/agent_runs.py
index a1d1dee2..33628c51 100644
--- a/backend/routes/agent_runs.py
+++ b/backend/routes/agent_runs.py
@@ -54,6 +54,10 @@ _RUN_COLUMNS = [
"total_cost_usd",
"has_unpriced_calls",
"size_bytes",
+ "eval_session",
+ "eval_set",
+ "input_file",
+ "input_image_sha256",
]
@@ -85,6 +89,10 @@ class AgentRunSummary(BaseModel):
total_cost_usd: Optional[float]
has_unpriced_calls: bool
size_bytes: Optional[int]
+ eval_session: Optional[str]
+ eval_set: Optional[str]
+ input_file: Optional[str]
+ input_image_sha256: Optional[str]
class AgentRunListResponse(BaseModel):
diff --git a/backend/routes/eval_sets.py b/backend/routes/eval_sets.py
new file mode 100644
index 00000000..39df8eb2
--- /dev/null
+++ b/backend/routes/eval_sets.py
@@ -0,0 +1,304 @@
+"""API for eval sets, eval sessions, and the per-session run matrix."""
+
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+from evals import sessions as eval_sessions
+from evals import sets as eval_sets
+from fs_logging.agent_runs import open_index_db
+
+router = APIRouter()
+
+_STALE_RUNNING_AFTER = timedelta(hours=2)
+
+
+class EvalSetSummaryModel(BaseModel):
+ name: str
+ display_name: str
+ created_at: Optional[str]
+ notes: str
+ image_count: int
+
+
+class EvalSetImageModel(BaseModel):
+ filename: str
+ sha256: str
+ size_bytes: int
+ tags: List[str]
+
+
+class EvalSetDetailModel(EvalSetSummaryModel):
+ images: List[EvalSetImageModel]
+
+
+class EvalSessionModel(BaseModel):
+ session_id: str
+ name: str
+ eval_set: str
+ created_at: str
+ is_active: bool
+
+
+class EvalSessionListResponse(BaseModel):
+ sessions: List[EvalSessionModel]
+ active_session_id: Optional[str]
+
+
+class CreateEvalSessionRequest(BaseModel):
+ eval_set: str
+ name: Optional[str] = None
+
+
+class MatrixRunModel(BaseModel):
+ run_id: str
+ status: str
+ source: str # "eval" | "ui"
+ total_duration_ms: Optional[int]
+ total_cost_usd: Optional[float]
+ created_at: str
+ is_stale: bool
+
+
+class MatrixCellModel(BaseModel):
+ filename: str
+ model: str
+ runs: List[MatrixRunModel] # newest first
+
+
+class MatrixRowModel(BaseModel):
+ filename: str
+ sha256: str
+ image_url: str
+
+
+class SessionMatrixResponse(BaseModel):
+ session: EvalSessionModel
+ eval_set: Optional[EvalSetSummaryModel]
+ set_missing: bool
+ rows: List[MatrixRowModel]
+ models: List[str]
+ cells: List[MatrixCellModel]
+ unmatched_run_count: int
+
+
+def _set_info_to_model(info: eval_sets.EvalSetInfo) -> EvalSetSummaryModel:
+ return EvalSetSummaryModel(
+ name=info.name,
+ display_name=info.display_name,
+ created_at=info.created_at,
+ notes=info.notes,
+ image_count=info.image_count,
+ )
+
+
+def _session_to_model(session: eval_sessions.EvalSession) -> EvalSessionModel:
+ return EvalSessionModel(
+ session_id=session.session_id,
+ name=session.name,
+ eval_set=session.eval_set,
+ created_at=session.created_at,
+ is_active=session.is_active,
+ )
+
+
[email protected]("/eval-sets", response_model=List[EvalSetSummaryModel])
+async def list_eval_sets() -> List[EvalSetSummaryModel]:
+ return [_set_info_to_model(info) for info in eval_sets.list_sets()]
+
+
[email protected]("/eval-sets/{set_name}", response_model=EvalSetDetailModel)
+async def get_eval_set(set_name: str) -> EvalSetDetailModel:
+ try:
+ info = eval_sets.get_set(set_name)
+ images = eval_sets.list_set_images(set_name)
+ except eval_sets.InvalidSetNameError as e:
+ raise HTTPException(status_code=400, detail=str(e))
+ except eval_sets.EvalSetNotFoundError:
+ raise HTTPException(status_code=404, detail=f"Eval set not found: {set_name}")
+ return EvalSetDetailModel(
+ **_set_info_to_model(info).model_dump(),
+ images=[
+ EvalSetImageModel(
+ filename=image.filename,
+ sha256=image.sha256,
+ size_bytes=image.size_bytes,
+ tags=image.tags,
+ )
+ for image in images
+ ],
+ )
+
+
[email protected]("/eval-sets/{set_name}/images/{filename}")
+async def get_eval_set_image(set_name: str, filename: str) -> FileResponse:
+ try:
+ path = eval_sets.resolve_set_image_path(set_name, filename)
+ except eval_sets.InvalidSetNameError as e:
+ raise HTTPException(status_code=400, detail=str(e))
+ except eval_sets.EvalSetNotFoundError:
+ raise HTTPException(status_code=404, detail=f"Eval set not found: {set_name}")
+ except FileNotFoundError:
+ raise HTTPException(status_code=404, detail="Image not found")
+ return FileResponse(path)
+
+
[email protected]("/eval-sessions", response_model=EvalSessionListResponse)
+async def list_eval_sessions() -> EvalSessionListResponse:
+ sessions = eval_sessions.list_sessions()
+ active = next((s for s in sessions if s.is_active), None)
+ return EvalSessionListResponse(
+ sessions=[_session_to_model(s) for s in sessions],
+ active_session_id=active.session_id if active else None,
+ )
+
+
[email protected]("/eval-sessions/active", response_model=Optional[EvalSessionModel])
+async def get_active_eval_session() -> Optional[EvalSessionModel]:
+ active = eval_sessions.get_active_session()
+ return _session_to_model(active) if active else None
+
+
[email protected]("/eval-sessions", response_model=EvalSessionModel)
+async def create_eval_session(request: CreateEvalSessionRequest) -> EvalSessionModel:
+ try:
+ eval_sets.get_set(request.eval_set)
+ except eval_sets.InvalidSetNameError as e:
+ raise HTTPException(status_code=400, detail=str(e))
+ except eval_sets.EvalSetNotFoundError:
+ raise HTTPException(
+ status_code=404, detail=f"Eval set not found: {request.eval_set}"
+ )
+ session = eval_sessions.create_session(request.eval_set, request.name)
+ return _session_to_model(session)
+
+
+ "/eval-sessions/{session_id}/activate", response_model=EvalSessionModel
+)
+async def activate_eval_session(session_id: str) -> EvalSessionModel:
+ if eval_sessions.get_session(session_id) is None:
+ raise HTTPException(status_code=404, detail="Session not found")
+ session = eval_sessions.activate_session(session_id)
+ assert session is not None
+ return _session_to_model(session)
+
+
+def _is_stale_running(status: str, created_at: str) -> bool:
+ if status != "running":
+ return False
+ try:
+ started = datetime.fromisoformat(created_at)
+ except ValueError:
+ return True
+ return datetime.now() - started > _STALE_RUNNING_AFTER
+
+
+ "/eval-sessions/{session_id}/matrix", response_model=SessionMatrixResponse
+)
+async def get_session_matrix(session_id: str) -> SessionMatrixResponse:
+ session = eval_sessions.get_session(session_id)
+ if session is None:
+ raise HTTPException(status_code=404, detail="Session not found")
+
+ set_missing = False
+ set_info: Optional[eval_sets.EvalSetInfo] = None
+ images: List[eval_sets.EvalSetImage] = []
+ try:
+ set_info = eval_sets.get_set(session.eval_set)
+ images = eval_sets.list_set_images(session.eval_set)
+ except (eval_sets.EvalSetNotFoundError, eval_sets.InvalidSetNameError):
+ set_missing = True
+
+ rows = [
+ MatrixRowModel(
+ filename=image.filename,
+ sha256=image.sha256,
+ image_url=f"/eval-sets/{session.eval_set}/images/{image.filename}",
+ )
+ for image in images
+ ]
+ set_filenames = {image.filename for image in images}
+ filenames_by_sha: Dict[str, List[str]] = {}
+ for image in images:
+ filenames_by_sha.setdefault(image.sha256, []).append(image.filename)
+
+ run_query_columns = (
+ "run_id, model, status, total_duration_ms, total_cost_usd, "
+ "created_at, input_file, input_image_sha256"
+ )
+ conn = open_index_db()
+ try:
+ eval_rows = conn.execute(
+ f"SELECT {run_query_columns} FROM runs "
+ "WHERE eval_session = ? ORDER BY created_at DESC, run_id DESC",
+ (session_id,),
+ ).fetchall()
+ ui_rows: List[tuple[Any, ...]] = []
+ if filenames_by_sha:
+ placeholders = ", ".join("?" for _ in filenames_by_sha)
+ ui_rows = conn.execute(
+ f"SELECT {run_query_columns} FROM runs "
+ "WHERE eval_session IS NULL AND entry_point != 'eval' "
+ f"AND created_at >= ? AND input_image_sha256 IN ({placeholders}) "
+ "ORDER BY created_at DESC, run_id DESC",
+ (session.created_at, *filenames_by_sha.keys()),
+ ).fetchall()
+ finally:
+ conn.close()
+
+ # (filename, model) -> runs, insertion-ordered so cells stay newest-first.
+ cell_runs: Dict[tuple[str, str], List[MatrixRunModel]] = {}
+ model_first_seen: Dict[str, str] = {}
+ unmatched_run_count = 0
+
+ def add_run(filename: str, row: tuple[Any, ...], source: str) -> None:
+ run_id, model, status, duration, cost, created_at = row[:6]
+ cell_runs.setdefault((filename, model), []).append(
+ MatrixRunModel(
+ run_id=run_id,
+ status=status,
+ source=source,
+ total_duration_ms=duration,
+ total_cost_usd=cost,
+ created_at=created_at,
+ is_stale=_is_stale_running(status, created_at),
+ )
+ )
+ # Rows are scanned newest-first; keep overwriting so the oldest
+ # occurrence wins and columns order by first use in the session.
+ model_first_seen[model] = created_at
+
+ for row in eval_rows:
+ input_file = row[6]
+ # With a missing set the cells still carry the session's history;
+ # otherwise runs whose file left the set are counted, not shown.
+ if input_file and (set_missing or input_file in set_filenames):
+ add_run(input_file, row, "eval")
+ else:
+ unmatched_run_count += 1
+
+ for row in ui_rows:
+ sha = row[7]
+ for filename in filenames_by_sha.get(sha, []):
+ add_run(filename, row, "ui")
+
+ models = sorted(model_first_seen, key=lambda m: model_first_seen[m])
+ cells = [
+ MatrixCellModel(filename=filename, model=model, runs=runs)
+ for (filename, model), runs in cell_runs.items()
+ ]
+
+ return SessionMatrixResponse(
+ session=_session_to_model(session),
+ eval_set=_set_info_to_model(set_info) if set_info else None,
+ set_missing=set_missing,
+ rows=rows,
+ models=models,
+ cells=cells,
+ unmatched_run_count=unmatched_run_count,
+ )
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index 8fc78646..be9cef1a 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -6,8 +6,10 @@ from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from evals.utils import image_to_data_url
from evals.config import EVALS_DIR
-from typing import Set
+from typing import Optional, Set
from evals.runner import run_image_evals, count_pending_eval_tasks
+from evals import sessions as eval_sessions
+from evals import sets as eval_sets
from typing import List, Dict
from llm import Llm
from prompts.prompt_types import Stack
@@ -107,6 +109,47 @@ class RunEvalsRequest(BaseModel):
stack: Stack
files: List[str] = [] # Optional list of specific file paths to run evals on
diff_mode: bool = False
+ # When set, inputs come from {EVALS_DIR}/sets/{set_name}/inputs and runs
+ # attach to the active eval session (auto-created when none exists).
+ set_name: Optional[str] = None
+
+
+def _resolve_set_run(
+ request: RunEvalsRequest,
+) -> tuple[Optional[str], Optional[eval_sessions.EvalSession], Dict[str, set[str]]]:
+ """Validate the requested set and resolve the session + per-model skips."""
+ if not request.set_name:
+ return None, None, {}
+ try:
+ set_info = eval_sets.get_set(request.set_name)
+ except eval_sets.InvalidSetNameError as e:
+ raise HTTPException(status_code=400, detail=str(e))
+ except eval_sets.EvalSetNotFoundError:
+ raise HTTPException(
+ status_code=404, detail=f"Eval set not found: {request.set_name}"
+ )
+ if set_info.image_count == 0:
+ raise HTTPException(
+ status_code=400, detail=f"Eval set {request.set_name!r} has no images"
+ )
+ try:
+ session = eval_sessions.resolve_session_for_run(request.set_name)
+ except eval_sessions.SessionSetMismatchError as e:
+ raise HTTPException(
+ status_code=400,
+ detail=(
+ f"Active session {e.active_session.name!r} is pinned to set "
+ f"{e.active_session.eval_set!r}. Start a new session for set "
+ f"{request.set_name!r} first (POST /eval-sessions)."
+ ),
+ )
+ skip_per_model: Dict[str, set[str]] = {}
+ if request.diff_mode:
+ for model in request.models:
+ skip_per_model[model] = eval_sessions.completed_eval_inputs(
+ session.session_id, model, str(request.stack)
+ )
+ return request.set_name, session, skip_per_model
class OpenAIInputCompareRequest(BaseModel):
@@ -182,6 +225,7 @@ async def compare_openai_inputs_for_evals(
@router.post("/run_evals", response_model=List[str])
async def run_evals(request: RunEvalsRequest) -> List[str]:
"""Run evaluations on selected images in the inputs directory for multiple models"""
+ eval_set, session, skip_per_model = _resolve_set_run(request)
all_output_files: List[str] = []
for model in request.models:
@@ -190,16 +234,22 @@ async def run_evals(request: RunEvalsRequest) -> List[str]:
stack=request.stack,
input_files=request.files,
diff_mode=request.diff_mode,
+ eval_set=eval_set,
+ eval_session_id=session.session_id if session else None,
+ skip_input_files=skip_per_model.get(model),
)
all_output_files.extend(output_files)
return all_output_files
-def _count_eval_files(selected_files: List[str]) -> int:
+def _count_eval_files(selected_files: List[str], set_name: Optional[str] = None) -> int:
if selected_files:
return len([f for f in selected_files if f.endswith(".png")])
+ if set_name:
+ return len(eval_sets.list_set_images(set_name))
+
input_dir = os.path.join(EVALS_DIR, "inputs")
return len([f for f in os.listdir(input_dir) if f.endswith(".png")])
@@ -210,6 +260,8 @@ async def run_evals_stream(request: RunEvalsRequest):
if not request.models:
raise HTTPException(status_code=400, detail="At least one model is required")
+ eval_set, session, skip_per_model = _resolve_set_run(request)
+
per_model_task_counts: Dict[str, int] = {}
per_model_skipped_existing: Dict[str, int] = {}
if request.diff_mode:
@@ -220,11 +272,13 @@ async def run_evals_stream(request: RunEvalsRequest):
input_files=request.files,
n=N,
diff_mode=True,
+ eval_set=eval_set,
+ skip_input_files=skip_per_model.get(model),
)
per_model_task_counts[model] = pending_tasks
per_model_skipped_existing[model] = skipped_tasks
else:
- per_model_task_count = _count_eval_files(request.files)
+ per_model_task_count = _count_eval_files(request.files, eval_set)
for model in request.models:
per_model_task_counts[model] = per_model_task_count
per_model_skipped_existing[model] = 0
@@ -252,6 +306,9 @@ async def run_evals_stream(request: RunEvalsRequest):
"completed_tasks": 0,
"diff_mode": request.diff_mode,
"total_skipped_existing": total_skipped_existing,
+ "eval_set": eval_set,
+ "eval_session_id": session.session_id if session else None,
+ "eval_session_name": session.name if session else None,
}
)
@@ -288,6 +345,9 @@ async def run_evals_stream(request: RunEvalsRequest):
input_files=request.files,
diff_mode=request.diff_mode,
progress_callback=on_progress,
+ eval_set=eval_set,
+ eval_session_id=session.session_id if session else None,
+ skip_input_files=skip_per_model.get(model),
)
all_output_files.extend(output_files)
completed_offset += model_task_count
diff --git a/backend/tests/test_agent_runs.py b/backend/tests/test_agent_runs.py
index 13284ba2..6ca0d11c 100644
--- a/backend/tests/test_agent_runs.py
+++ b/backend/tests/test_agent_runs.py
@@ -6,8 +6,8 @@ from typing import Any, Optional
import pytest
from agent.providers.base import StreamEvent
-from agent.providers.pricing import MODEL_PRICING
-from agent.providers.token_usage import TokenUsage
+from costs.pricing import MODEL_PRICING
+from costs.token_usage import TokenUsage
from agent.tools.types import ToolCall, ToolExecutionResult
from fs_logging.agent_runs import (
RUN_ID_PATTERN,
diff --git a/backend/tests/test_eval_sessions.py b/backend/tests/test_eval_sessions.py
new file mode 100644
index 00000000..1faad2b0
--- /dev/null
+++ b/backend/tests/test_eval_sessions.py
@@ -0,0 +1,467 @@
+import base64
+import sqlite3
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Optional
+
+import pytest
+from fastapi import HTTPException
+
+import evals.config
+from evals.sessions import (
+ SessionSetMismatchError,
+ activate_session,
+ create_session,
+ get_active_session,
+ list_sessions,
+ resolve_session_for_run,
+ completed_eval_inputs,
+)
+from fs_logging.agent_runs import (
+ AgentRunRecorder,
+ get_agent_runs_db_path,
+ get_agent_runs_directory,
+ open_index_db,
+)
+from llm import Llm
+
+
+def logs_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ monkeypatch.setenv("LOGS_PATH", str(tmp_path))
+ return tmp_path
+
+
+def evals_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ monkeypatch.setattr(evals.config, "EVALS_DIR", str(tmp_path / "evals_data"))
+ return tmp_path / "evals_data"
+
+
+def _make_set(evals_dir: Path, name: str, images: dict[str, bytes]) -> None:
+ inputs = evals_dir / "sets" / name / "inputs"
+ inputs.mkdir(parents=True)
+ for filename, content in images.items():
+ (inputs / filename).write_bytes(content)
+
+
+# The runs DDL exactly as first shipped, before the eval columns existed.
+_LEGACY_RUNS_DDL = """
+CREATE TABLE runs (
+ run_id TEXT PRIMARY KEY,
+ generation_id TEXT NOT NULL,
+ variant_index INTEGER NOT NULL,
+ entry_point TEXT NOT NULL,
+ provider TEXT,
+ model TEXT,
+ api_model_name TEXT,
+ stack TEXT,
+ input_mode TEXT,
+ generation_type TEXT,
+ status TEXT NOT NULL DEFAULT 'running',
+ error TEXT,
+ created_at TEXT NOT NULL,
+ completed_at TEXT,
+ total_duration_ms INTEGER,
+ llm_time_ms INTEGER,
+ tool_time_ms INTEGER,
+ num_llm_calls INTEGER NOT NULL DEFAULT 0,
+ num_tool_calls INTEGER NOT NULL DEFAULT 0,
+ input_tokens INTEGER,
+ output_tokens INTEGER,
+ cache_read_tokens INTEGER,
+ cache_write_tokens INTEGER,
+ total_tokens INTEGER,
+ total_cost_usd REAL,
+ has_unpriced_calls INTEGER NOT NULL DEFAULT 0,
+ run_dir TEXT NOT NULL,
+ size_bytes INTEGER
+)
+"""
+
+
+def test_migration_upgrades_legacy_db(logs_path: Path) -> None:
+ db_dir = Path(get_agent_runs_directory())
+ db_dir.mkdir(parents=True)
+ conn = sqlite3.connect(get_agent_runs_db_path())
+ conn.execute(_LEGACY_RUNS_DDL)
+ conn.execute(
+ "INSERT INTO runs (run_id, generation_id, variant_index, entry_point, "
+ "status, created_at, run_dir) VALUES ('run_legacy', 'gen_x', 0, "
+ "'websocket', 'completed', '2026-07-01T10:00:00', '/tmp/x')"
+ )
+ conn.commit()
+ conn.close()
+
+ migrated = open_index_db()
+ try:
+ columns = {row[1] for row in migrated.execute("PRAGMA table_info(runs)")}
+ assert {
+ "eval_session",
+ "eval_set",
+ "input_file",
+ "input_image_sha256",
+ } <= columns
+ row = migrated.execute(
+ "SELECT status, eval_session, eval_set, input_file FROM runs "
+ "WHERE run_id = 'run_legacy'"
+ ).fetchone()
+ assert row == ("completed", None, None, None)
+ tables = {
+ r[0]
+ for r in migrated.execute(
+ "SELECT name FROM sqlite_master WHERE type = 'table'"
+ )
+ }
+ assert "eval_sessions" in tables
+ finally:
+ migrated.close()
+
+ # Idempotent on reopen.
+ open_index_db().close()
+
+
+def _active_count() -> int:
+ conn = sqlite3.connect(get_agent_runs_db_path())
+ try:
+ return conn.execute(
+ "SELECT COUNT(*) FROM eval_sessions WHERE is_active = 1"
+ ).fetchone()[0]
+ finally:
+ conn.close()
+
+
+def test_create_and_activate_single_active_invariant(logs_path: Path) -> None:
+ first = create_session("set-a", "first")
+ assert first.is_active
+ second = create_session("set-b")
+ assert second.name.endswith("session") # auto-named
+ assert _active_count() == 1
+ active = get_active_session()
+ assert active is not None and active.session_id == second.session_id
+
+ activate_session(first.session_id)
+ assert _active_count() == 1
+ active = get_active_session()
+ assert active is not None and active.session_id == first.session_id
+
+ sessions = list_sessions()
+ assert {s.session_id for s in sessions} == {
+ first.session_id,
+ second.session_id,
+ }
+
+
+def test_resolve_session_for_run(logs_path: Path) -> None:
+ # No session: auto-creates and activates.
+ session = resolve_session_for_run("set-a")
+ assert session.is_active and session.eval_set == "set-a"
+ # Same set: reuses.
+ again = resolve_session_for_run("set-a")
+ assert again.session_id == session.session_id
+ # Different set: refuses.
+ with pytest.raises(SessionSetMismatchError):
+ resolve_session_for_run("set-b")
+
+
+def _record_run(
+ *,
+ model: Llm = Llm.GPT_5_5_HIGH,
+ entry_point: str = "eval",
+ eval_session: Optional[str] = None,
+ eval_set: Optional[str] = None,
+ input_file: Optional[str] = None,
+ image_bytes: Optional[bytes] = None,
+ stack: str = "html_tailwind",
+) -> AgentRunRecorder:
+ recorder = AgentRunRecorder(
+ generation_id="gen_t",
+ variant_index=0,
+ entry_point=entry_point,
+ stack=stack,
+ enabled=True,
+ eval_session=eval_session,
+ eval_set=eval_set,
+ input_file=input_file,
+ )
+ prompt: list[Any] = []
+ if image_bytes is not None:
+ data_url = "data:image/png;base64," + base64.b64encode(
+ image_bytes
+ ).decode("ascii")
+ prompt = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image_url", "image_url": {"url": data_url}}
+ ],
+ }
+ ]
+ recorder.record_run_start(model, prompt)
+ return recorder
+
+
+def _set_run_status(
+ run_id: str, status: str, created_at: Optional[str] = None
+) -> None:
+ conn = sqlite3.connect(get_agent_runs_db_path())
+ try:
+ conn.execute(
+ "UPDATE runs SET status = ? WHERE run_id = ?", (status, run_id)
+ )
+ if created_at is not None:
+ conn.execute(
+ "UPDATE runs SET created_at = ? WHERE run_id = ?",
+ (created_at, run_id),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+
+def test_recorder_eval_columns_and_websocket_sha(logs_path: Path) -> None:
+ eval_recorder = _record_run(
+ eval_session="sess_x",
+ eval_set="set-a",
+ input_file="shop.png",
+ image_bytes=b"shop-image",
+ )
+ ui_recorder = _record_run(entry_point="websocket", image_bytes=b"ui-image")
+ garbage_recorder = AgentRunRecorder(
+ generation_id="g", variant_index=0, entry_point="websocket", enabled=True
+ )
+ garbage_recorder.record_run_start(Llm.GPT_5_5_HIGH, {"not": "a list"})
+
+ conn = sqlite3.connect(get_agent_runs_db_path())
+ try:
+ rows = {
+ row[0]: row
+ for row in conn.execute(
+ "SELECT run_id, eval_session, eval_set, input_file, "
+ "input_image_sha256 FROM runs"
+ )
+ }
+ finally:
+ conn.close()
+
+ import hashlib
+
+ eval_row = rows[eval_recorder.run_id]
+ assert eval_row[1:4] == ("sess_x", "set-a", "shop.png")
+ assert eval_row[4] == hashlib.sha256(b"shop-image").hexdigest()
+
+ ui_row = rows[ui_recorder.run_id]
+ assert ui_row[1:4] == (None, None, None)
+ assert ui_row[4] == hashlib.sha256(b"ui-image").hexdigest()
+
+ assert rows[garbage_recorder.run_id][4] is None
+
+
+def test_completed_eval_inputs(logs_path: Path) -> None:
+ session = create_session("set-a")
+ done = _record_run(
+ eval_session=session.session_id, eval_set="set-a", input_file="a.png"
+ )
+ _set_run_status(done.run_id, "completed")
+ failed = _record_run(
+ eval_session=session.session_id, eval_set="set-a", input_file="b.png"
+ )
+ _set_run_status(failed.run_id, "failed")
+ _record_run(
+ eval_session=session.session_id,
+ eval_set="set-a",
+ input_file="c.png",
+ model=Llm.CLAUDE_OPUS_4_8_HIGH,
+ )
+
+ completed = completed_eval_inputs(
+ session.session_id, Llm.GPT_5_5_HIGH.value, "html_tailwind"
+ )
+ assert completed == {"a.png"}
+
+
+async def test_session_matrix(logs_path: Path, evals_dir: Path) -> None:
+ from routes.eval_sets import get_session_matrix
+
+ _make_set(
+ evals_dir, "set-a", {"one.png": b"image-one", "two.png": b"image-two"}
+ )
+ session = create_session("set-a")
+
+ completed = _record_run(
+ eval_session=session.session_id, eval_set="set-a", input_file="one.png"
+ )
+ _set_run_status(completed.run_id, "completed")
+ failed = _record_run(
+ eval_session=session.session_id,
+ eval_set="set-a",
+ input_file="one.png",
+ model=Llm.CLAUDE_OPUS_4_8_HIGH,
+ )
+ _set_run_status(failed.run_id, "failed")
+ running = _record_run(
+ eval_session=session.session_id, eval_set="set-a", input_file="two.png"
+ )
+ stale_time = (datetime.now() - timedelta(hours=3)).isoformat(
+ timespec="seconds"
+ )
+ stale = _record_run(
+ eval_session=session.session_id,
+ eval_set="set-a",
+ input_file="two.png",
+ model=Llm.CLAUDE_OPUS_4_8_HIGH,
+ )
+ _set_run_status(stale.run_id, "running", created_at=stale_time)
+ orphan = _record_run(
+ eval_session=session.session_id, eval_set="set-a", input_file="gone.png"
+ )
+ _set_run_status(orphan.run_id, "completed")
+
+ # UI run after session start with a matching image hash: included.
+ ui_match = _record_run(entry_point="websocket", image_bytes=b"image-one")
+ _set_run_status(ui_match.run_id, "completed")
+ # UI run BEFORE session start: excluded.
+ ui_before = _record_run(entry_point="websocket", image_bytes=b"image-one")
+ _set_run_status(
+ ui_before.run_id, "completed", created_at="2020-01-01T00:00:00"
+ )
+
+ matrix = await get_session_matrix(session.session_id)
+ assert not matrix.set_missing
+ assert [row.filename for row in matrix.rows] == ["one.png", "two.png"]
+ assert set(matrix.models) == {
+ Llm.GPT_5_5_HIGH.value,
+ Llm.CLAUDE_OPUS_4_8_HIGH.value,
+ }
+ assert matrix.unmatched_run_count == 1
+
+ cells = {(c.filename, c.model): c for c in matrix.cells}
+ one_gpt = cells[("one.png", Llm.GPT_5_5_HIGH.value)]
+ assert {r.run_id for r in one_gpt.runs} == {
+ completed.run_id,
+ ui_match.run_id,
+ }
+ sources = {r.run_id: r.source for r in one_gpt.runs}
+ assert sources[ui_match.run_id] == "ui"
+ assert sources[completed.run_id] == "eval"
+ assert ui_before.run_id not in sources
+
+ one_claude = cells[("one.png", Llm.CLAUDE_OPUS_4_8_HIGH.value)]
+ assert one_claude.runs[0].status == "failed"
+
+ two_gpt = cells[("two.png", Llm.GPT_5_5_HIGH.value)]
+ assert two_gpt.runs[0].status == "running"
+ assert two_gpt.runs[0].is_stale is False
+ two_claude = cells[("two.png", Llm.CLAUDE_OPUS_4_8_HIGH.value)]
+ assert two_claude.runs[0].is_stale is True
+
+
+async def test_session_matrix_missing_set(
+ logs_path: Path, evals_dir: Path
+) -> None:
+ from routes.eval_sets import get_session_matrix
+
+ session = create_session("never-existed")
+ run = _record_run(
+ eval_session=session.session_id,
+ eval_set="never-existed",
+ input_file="x.png",
+ )
+ _set_run_status(run.run_id, "completed")
+
+ matrix = await get_session_matrix(session.session_id)
+ assert matrix.set_missing is True
+ assert matrix.rows == []
+ # History still browsable through cells.
+ assert any(c.filename == "x.png" for c in matrix.cells)
+
+
+async def test_run_evals_threads_set_and_session(
+ logs_path: Path, evals_dir: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ import routes.evals as evals_routes
+ from routes.evals import RunEvalsRequest, run_evals
+
+ _make_set(evals_dir, "set-a", {"a.png": b"a", "b.png": b"b"})
+ _make_set(evals_dir, "set-b", {"x.png": b"x"})
+
+ captured: list[dict[str, Any]] = []
+
+ async def fake_run_image_evals(**kwargs: Any) -> list[str]:
+ captured.append(kwargs)
+ return []
+
+ monkeypatch.setattr(evals_routes, "run_image_evals", fake_run_image_evals)
+
+ # Seed a completed combo so diff mode skips it.
+ session = create_session("set-a")
+ done = _record_run(
+ eval_session=session.session_id, eval_set="set-a", input_file="a.png"
+ )
+ _set_run_status(done.run_id, "completed")
+
+ await run_evals(
+ RunEvalsRequest(
+ models=[Llm.GPT_5_5_HIGH.value],
+ stack="html_tailwind",
+ set_name="set-a",
+ diff_mode=True,
+ )
+ )
+ assert len(captured) == 1
+ kwargs = captured[0]
+ assert kwargs["eval_set"] == "set-a"
+ assert kwargs["eval_session_id"] == session.session_id
+ assert kwargs["skip_input_files"] == {"a.png"}
+
+ # Mismatched active session -> 400 with guidance.
+ with pytest.raises(HTTPException) as excinfo:
+ await run_evals(
+ RunEvalsRequest(
+ models=[Llm.GPT_5_5_HIGH.value],
+ stack="html_tailwind",
+ set_name="set-b",
+ )
+ )
+ assert excinfo.value.status_code == 400
+ assert "Start a new session" in str(excinfo.value.detail)
+
+ # Unknown set -> 404.
+ create_session("set-a") # keep active pinned to an existing set
+ with pytest.raises(HTTPException) as excinfo:
+ await run_evals(
+ RunEvalsRequest(
+ models=[Llm.GPT_5_5_HIGH.value],
+ stack="html_tailwind",
+ set_name="missing-set",
+ )
+ )
+ assert excinfo.value.status_code == 404
+
+
+def test_count_pending_eval_tasks_with_set(
+ logs_path: Path, evals_dir: Path
+) -> None:
+ from evals.runner import count_pending_eval_tasks
+
+ _make_set(evals_dir, "set-a", {"a.png": b"a", "b.png": b"b", "c.png": b"c"})
+
+ pending, skipped = count_pending_eval_tasks(
+ stack="html_tailwind",
+ model=Llm.GPT_5_5_HIGH.value,
+ diff_mode=True,
+ eval_set="set-a",
+ skip_input_files={"a.png"},
+ )
+ assert (pending, skipped) == (2, 1)
+
+ pending, skipped = count_pending_eval_tasks(
+ stack="html_tailwind",
+ model=Llm.GPT_5_5_HIGH.value,
+ diff_mode=False,
+ eval_set="set-a",
+ )
+ assert (pending, skipped) == (3, 0)
diff --git a/backend/tests/test_eval_sets.py b/backend/tests/test_eval_sets.py
new file mode 100644
index 00000000..17952abd
--- /dev/null
+++ b/backend/tests/test_eval_sets.py
@@ -0,0 +1,123 @@
+import hashlib
+import json
+import os
+import time
+from pathlib import Path
+
+import pytest
+from fastapi import HTTPException
+
+import evals.config
+from evals.sets import (
+ EvalSetNotFoundError,
+ InvalidSetNameError,
+ get_set,
+ get_set_inputs_dir,
+ list_set_images,
+ list_sets,
+ resolve_set_image_path,
+)
+from routes.eval_sets import get_eval_set_image
+
+
+def evals_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ monkeypatch.setattr(evals.config, "EVALS_DIR", str(tmp_path))
+ return tmp_path
+
+
+def _make_set(evals_dir: Path, name: str, images: dict[str, bytes]) -> Path:
+ inputs = evals_dir / "sets" / name / "inputs"
+ inputs.mkdir(parents=True)
+ for filename, content in images.items():
+ (inputs / filename).write_bytes(content)
+ return inputs
+
+
+def test_list_set_images_bootstraps_manifest_with_hashes(evals_dir: Path) -> None:
+ _make_set(evals_dir, "jun-21-evals", {"a.png": b"aaa", "b.png": b"bbb"})
+
+ images = list_set_images("jun-21-evals")
+ assert [i.filename for i in images] == ["a.png", "b.png"]
+ assert images[0].sha256 == hashlib.sha256(b"aaa").hexdigest()
+
+ manifest_path = evals_dir / "sets" / "jun-21-evals" / "manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ assert manifest["display_name"] == "jun-21-evals"
+ assert manifest["images"]["b.png"]["sha256"] == (
+ hashlib.sha256(b"bbb").hexdigest()
+ )
+
+
+def test_hash_cache_reused_and_invalidated(evals_dir: Path) -> None:
+ inputs = _make_set(evals_dir, "s1", {"a.png": b"original"})
+ list_set_images("s1")
+
+ # Tamper with the cached sha; with unchanged size+mtime it must be
+ # served from cache (i.e. NOT recomputed).
+ manifest_path = evals_dir / "sets" / "s1" / "manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["images"]["a.png"]["sha256"] = "cached-sentinel"
+ manifest_path.write_text(json.dumps(manifest))
+ assert list_set_images("s1")[0].sha256 == "cached-sentinel"
+
+ # Rewriting the file (new mtime) invalidates the cache entry.
+ image_path = inputs / "a.png"
+ image_path.write_bytes(b"rewritten")
+ os.utime(image_path, (time.time() + 5, time.time() + 5))
+ images = list_set_images("s1")
+ assert images[0].sha256 == hashlib.sha256(b"rewritten").hexdigest()
+ manifest = json.loads(manifest_path.read_text())
+ assert manifest["images"]["a.png"]["sha256"] == images[0].sha256
+
+
+def test_deleted_files_pruned_from_manifest(evals_dir: Path) -> None:
+ inputs = _make_set(evals_dir, "s1", {"a.png": b"a", "b.png": b"b"})
+ list_set_images("s1")
+ (inputs / "b.png").unlink()
+
+ images = list_set_images("s1")
+ assert [i.filename for i in images] == ["a.png"]
+ manifest = json.loads(
+ (evals_dir / "sets" / "s1" / "manifest.json").read_text()
+ )
+ assert "b.png" not in manifest["images"]
+
+
+def test_list_sets_and_get_set(evals_dir: Path) -> None:
+ _make_set(evals_dir, "alpha", {"a.png": b"a"})
+ _make_set(evals_dir, "beta", {"b.png": b"b", "c.png": b"c"})
+
+ sets = list_sets()
+ assert [(s.name, s.image_count) for s in sets] == [("alpha", 1), ("beta", 2)]
+ assert get_set("beta").image_count == 2
+ with pytest.raises(EvalSetNotFoundError):
+ get_set("missing")
+
+
+def test_set_name_validation(evals_dir: Path) -> None:
+ for bad_name in ("../etc", "/abs", "", ".hidden"):
+ with pytest.raises(InvalidSetNameError):
+ get_set_inputs_dir(bad_name)
+
+
+async def test_image_path_resolution_and_route_guards(evals_dir: Path) -> None:
+ _make_set(evals_dir, "s1", {"a.png": b"png-bytes"})
+
+ assert resolve_set_image_path("s1", "a.png").endswith("a.png")
+ # Traversal components are stripped to a basename.
+ assert resolve_set_image_path("s1", "../a.png").endswith("a.png")
+ with pytest.raises(InvalidSetNameError):
+ resolve_set_image_path("s1", "notes.txt")
+ with pytest.raises(FileNotFoundError):
+ resolve_set_image_path("s1", "missing.png")
+
+ response = await get_eval_set_image("s1", "a.png")
+ assert response.path == str(evals_dir / "sets" / "s1" / "inputs" / "a.png")
+ with pytest.raises(HTTPException) as excinfo:
+ await get_eval_set_image("s1", "missing.png")
+ assert excinfo.value.status_code == 404
+ with pytest.raises(HTTPException) as excinfo:
+ await get_eval_set_image("nope", "a.png")
+ assert excinfo.value.status_code == 404
diff --git a/backend/tests/test_prompt_reports.py b/backend/tests/test_prompt_reports.py
index fe23efb5..1108fcc8 100644
--- a/backend/tests/test_prompt_reports.py
+++ b/backend/tests/test_prompt_reports.py
@@ -7,7 +7,7 @@ from pathlib import Path
import pytest
from fastapi import HTTPException
-from agent.providers.token_usage import TokenUsage
+from costs.token_usage import TokenUsage
from fs_logging.prompt_reports import (
PROMPT_REPORT_FILENAME_PATTERN,
PromptReportLogger,
diff --git a/backend/tests/test_token_usage.py b/backend/tests/test_token_usage.py
index c25bd181..136a0808 100644
--- a/backend/tests/test_token_usage.py
+++ b/backend/tests/test_token_usage.py
@@ -2,8 +2,8 @@
from types import SimpleNamespace
-from agent.providers.pricing import MODEL_PRICING, ModelPricing
-from agent.providers.token_usage import TokenUsage
+from costs.pricing import MODEL_PRICING, ModelPricing
+from costs.token_usage import TokenUsage
from agent.providers.anthropic import _extract_anthropic_usage
from agent.providers.gemini import _extract_usage as _extract_gemini_usage
from agent.providers.openai import _extract_openai_usage
diff --git a/frontend/src/components/evals/AgentRunsPage.tsx b/frontend/src/components/evals/AgentRunsPage.tsx
index d035cfcd..c306f026 100644
--- a/frontend/src/components/evals/AgentRunsPage.tsx
+++ b/frontend/src/components/evals/AgentRunsPage.tsx
@@ -1,6 +1,7 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import toast from "react-hot-toast";
import { BsBoxArrowUpRight } from "react-icons/bs";
+import { useSearchParams } from "react-router-dom";
import { HTTP_BACKEND_URL } from "../../config";
import EvalNavigation from "./EvalNavigation";
@@ -47,6 +48,10 @@ interface AgentRunSummary {
total_cost_usd: number | null;
has_unpriced_calls: boolean;
size_bytes: number | null;
+ eval_session: string | null;
+ eval_set: string | null;
+ input_file: string | null;
+ input_image_sha256: string | null;
}
interface AgentRunListResponse {
@@ -148,6 +153,7 @@ function SummaryStats({ run }: { run: AgentRunSummary }) {
}
function AgentRunsPage() {
+ const [searchParams, setSearchParams] = useSearchParams();
const [runs, setRuns] = useState<AgentRunSummary[]>([]);
const [totalSizeBytes, setTotalSizeBytes] = useState(0);
const [runsDirectory, setRunsDirectory] = useState("");
@@ -192,6 +198,7 @@ function AgentRunsPage() {
setActiveTab("timeline");
setIncludeDeltas(withDeltas);
setExpandSignal({ mode: "best", version: 0 });
+ setSearchParams({ run: runId }, { replace: true });
setIsLoadingDetail(true);
try {
const response = await fetch(
@@ -212,16 +219,22 @@ function AgentRunsPage() {
setIsLoadingDetail(false);
}
},
- []
+ [setSearchParams]
);
useEffect(() => {
fetchRuns().then((loadedRuns) => {
- if (loadedRuns.length > 0) {
+ // Deep link (?run=...) wins over the newest-run default so matrix
+ // cells and shared URLs land on the intended run.
+ const requestedRun = searchParams.get("run");
+ if (requestedRun) {
+ openRun(requestedRun);
+ } else if (loadedRuns.length > 0) {
const groups = groupByGeneration(loadedRuns);
openRun(groups[0].runs[0].run_id);
}
});
+ // eslint-disable-next-line react-hooks/exhaustive-deps
}, [fetchRuns, openRun]);
const handlePrune = async () => {
diff --git a/frontend/src/components/evals/EvalNavigation.tsx b/frontend/src/components/evals/EvalNavigation.tsx
index 635f2b5e..3f7c00f0 100644
--- a/frontend/src/components/evals/EvalNavigation.tsx
+++ b/frontend/src/components/evals/EvalNavigation.tsx
@@ -47,6 +47,13 @@ function EvalNavigation() {
>
Agent Runs
</Link>
+
+ <Link
+ to="/evals/sessions"
+ className="hover:text-blue-300 transition-colors"
+ >
+ Sessions
+ </Link>
</div>
<Link
diff --git a/frontend/src/components/evals/EvalSessionsPage.tsx b/frontend/src/components/evals/EvalSessionsPage.tsx
new file mode 100644
index 00000000..9fc78283
--- /dev/null
+++ b/frontend/src/components/evals/EvalSessionsPage.tsx
@@ -0,0 +1,513 @@
+import { useCallback, useEffect, useMemo, useState } from "react";
+import toast from "react-hot-toast";
+import { useNavigate } from "react-router-dom";
+import { BsBoxArrowUpRight } from "react-icons/bs";
+
+import { HTTP_BACKEND_URL } from "../../config";
+import EvalNavigation from "./EvalNavigation";
+import { formatCost, formatMs, formatTimestamp } from "./report-format";
+
+interface EvalSetSummary {
+ name: string;
+ display_name: string;
+ created_at: string | null;
+ notes: string;
+ image_count: number;
+}
+
+interface EvalSession {
+ session_id: string;
+ name: string;
+ eval_set: string;
+ created_at: string;
+ is_active: boolean;
+}
+
+interface MatrixRun {
+ run_id: string;
+ status: string;
+ source: "eval" | "ui";
+ total_duration_ms: number | null;
+ total_cost_usd: number | null;
+ created_at: string;
+ is_stale: boolean;
+}
+
+interface MatrixCell {
+ filename: string;
+ model: string;
+ runs: MatrixRun[];
+}
+
+interface MatrixRow {
+ filename: string;
+ sha256: string;
+ image_url: string;
+}
+
+interface SessionMatrix {
+ session: EvalSession;
+ eval_set: EvalSetSummary | null;
+ set_missing: boolean;
+ rows: MatrixRow[];
+ models: string[];
+ cells: MatrixCell[];
+ unmatched_run_count: number;
+}
+
+function statusDotClass(status: string, isStale: boolean): string {
+ if (status === "completed") return "bg-emerald-400";
+ if (status === "failed") return "bg-red-400";
+ return isStale ? "bg-zinc-500" : "bg-amber-400 animate-pulse";
+}
+
+function CellFace({
+ runs,
+ onOpenRun,
+}: {
+ runs: MatrixRun[];
+ onOpenRun: (runId: string) => void;
+}) {
+ const [expanded, setExpanded] = useState(false);
+ const newest = runs[0];
+ if (!newest) return null;
+
+ return (
+ <div className="flex flex-col gap-1">
+ <div
+ onClick={() => onOpenRun(newest.run_id)}
+ className={`group flex cursor-pointer items-center gap-1.5 rounded-md border px-1.5 py-1 transition-colors hover:bg-zinc-800 ${
+ newest.status === "failed"
+ ? "border-red-900 bg-red-950/30"
+ : "border-zinc-700 bg-zinc-950/60"
+ } ${newest.is_stale ? "opacity-50" : ""}`}
+ title={`${newest.status} · ${formatTimestamp(newest.created_at)}`}
+ >
+ <span
+ className={`h-2 w-2 shrink-0 rounded-full ${statusDotClass(
+ newest.status,
+ newest.is_stale
+ )}`}
+ />
+ <span className="font-mono text-[11px] text-zinc-300">
+ {formatMs(newest.total_duration_ms)}
+ </span>
+ <span className="font-mono text-[11px] text-emerald-400">
+ {formatCost(newest.total_cost_usd)}
+ </span>
+ {newest.source === "ui" && (
+ <span className="rounded bg-amber-900/60 px-1 text-[10px] font-medium text-amber-200">
+ UI
+ </span>
+ )}
+ {runs.length > 1 && (
+ <button
+ onClick={(e) => {
+ e.stopPropagation();
+ setExpanded(!expanded);
+ }}
+ className="rounded bg-zinc-800 px-1 text-[10px] text-zinc-400 hover:text-zinc-100"
+ title="Show all attempts"
+ >
+ ×{runs.length}
+ </button>
+ )}
+ <a
+ href={`${HTTP_BACKEND_URL}/agent-runs/${encodeURIComponent(
+ newest.run_id
+ )}/output`}
+ target="_blank"
+ rel="noreferrer"
+ onClick={(e) => e.stopPropagation()}
+ title="Open captured HTML"
+ className="ml-auto shrink-0 rounded p-0.5 text-zinc-600 opacity-0 transition-opacity hover:text-zinc-200 group-hover:opacity-100"
+ >
+ <BsBoxArrowUpRight className="text-[10px]" />
+ </a>
+ </div>
+ {expanded &&
+ runs.slice(1).map((run) => (
+ <div
+ key={run.run_id}
+ onClick={() => onOpenRun(run.run_id)}
+ className="flex cursor-pointer items-center gap-1.5 rounded-md border border-zinc-800 px-1.5 py-0.5 opacity-70 hover:bg-zinc-800"
+ >
+ <span
+ className={`h-1.5 w-1.5 shrink-0 rounded-full ${statusDotClass(
+ run.status,
+ run.is_stale
+ )}`}
+ />
+ <span className="font-mono text-[10px] text-zinc-400">
+ {formatMs(run.total_duration_ms)} ·{" "}
+ {formatCost(run.total_cost_usd)}
+ {run.source === "ui" ? " · UI" : ""}
+ </span>
+ </div>
+ ))}
+ </div>
+ );
+}
+
+function EvalSessionsPage() {
+ const navigate = useNavigate();
+ const [sessions, setSessions] = useState<EvalSession[]>([]);
+ const [sets, setSets] = useState<EvalSetSummary[]>([]);
+ const [selectedSessionId, setSelectedSessionId] = useState<string | null>(
+ null
+ );
+ const [matrix, setMatrix] = useState<SessionMatrix | null>(null);
+ const [isLoadingMatrix, setIsLoadingMatrix] = useState(false);
+ const [newSessionSet, setNewSessionSet] = useState("");
+ const [newSessionName, setNewSessionName] = useState("");
+ const [isCreating, setIsCreating] = useState(false);
+
+ const cellsByKey = useMemo(() => {
+ const map = new Map<string, MatrixCell>();
+ for (const cell of matrix?.cells ?? []) {
+ map.set(`${cell.filename}|${cell.model}`, cell);
+ }
+ return map;
+ }, [matrix]);
+
+ // Images with outputs float to the top: most models covered first, then
+ // most total runs, then filename for a stable order.
+ const sortedRows = useMemo(() => {
+ if (!matrix) return [];
+ const coverage = new Map<string, { models: number; runs: number }>();
+ for (const cell of matrix.cells) {
+ const entry = coverage.get(cell.filename) ?? { models: 0, runs: 0 };
+ if (cell.runs.length > 0) entry.models += 1;
+ entry.runs += cell.runs.length;
+ coverage.set(cell.filename, entry);
+ }
+ return [...matrix.rows].sort((a, b) => {
+ const ca = coverage.get(a.filename) ?? { models: 0, runs: 0 };
+ const cb = coverage.get(b.filename) ?? { models: 0, runs: 0 };
+ if (cb.models !== ca.models) return cb.models - ca.models;
+ if (cb.runs !== ca.runs) return cb.runs - ca.runs;
+ return a.filename.localeCompare(b.filename);
+ });
+ }, [matrix]);
+
+ const fetchSessions = useCallback(async (): Promise<EvalSession[]> => {
+ try {
+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`);
+ const data = await response.json();
+ setSessions(data.sessions);
+ return data.sessions;
+ } catch (error) {
+ console.error("Error fetching sessions", error);
+ toast.error("Failed to load sessions.");
+ return [];
+ }
+ }, []);
+
+ const fetchSets = useCallback(async () => {
+ try {
+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sets`);
+ const data: EvalSetSummary[] = await response.json();
+ setSets(data);
+ if (data.length > 0) {
+ setNewSessionSet((current) => current || data[0].name);
+ }
+ } catch (error) {
+ console.error("Error fetching eval sets", error);
+ }
+ }, []);
+
+ const loadMatrix = useCallback(async (sessionId: string) => {
+ setSelectedSessionId(sessionId);
+ setIsLoadingMatrix(true);
+ try {
+ const response = await fetch(
+ `${HTTP_BACKEND_URL}/eval-sessions/${encodeURIComponent(
+ sessionId
+ )}/matrix`
+ );
+ if (!response.ok) throw new Error("Request failed");
+ setMatrix(await response.json());
+ } catch (error) {
+ console.error("Error loading matrix", error);
+ setMatrix(null);
+ toast.error("Failed to load session matrix.");
+ } finally {
+ setIsLoadingMatrix(false);
+ }
+ }, []);
+
+ useEffect(() => {
+ fetchSets();
+ fetchSessions().then((loaded) => {
+ const active = loaded.find((s) => s.is_active) ?? loaded[0];
+ if (active) loadMatrix(active.session_id);
+ });
+ }, [fetchSets, fetchSessions, loadMatrix]);
+
+ // Live refresh while any cell is actively running.
+ useEffect(() => {
+ if (!matrix || !selectedSessionId) return;
+ const hasLiveRuns = matrix.cells.some((cell) =>
+ cell.runs.some((run) => run.status === "running" && !run.is_stale)
+ );
+ if (!hasLiveRuns) return;
+ const interval = window.setInterval(
+ () => loadMatrix(selectedSessionId),
+ 10_000
+ );
+ return () => window.clearInterval(interval);
+ }, [matrix, selectedSessionId, loadMatrix]);
+
+ const handleActivate = async (sessionId: string) => {
+ try {
+ const response = await fetch(
+ `${HTTP_BACKEND_URL}/eval-sessions/${encodeURIComponent(
+ sessionId
+ )}/activate`,
+ { method: "POST" }
+ );
+ if (!response.ok) throw new Error("Request failed");
+ toast.success("Session activated.");
+ fetchSessions();
+ } catch (error) {
+ console.error("Error activating session", error);
+ toast.error("Failed to activate session.");
+ }
+ };
+
+ const handleCreate = async () => {
+ if (!newSessionSet) return;
+ setIsCreating(true);
+ try {
+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ eval_set: newSessionSet,
+ name: newSessionName.trim() || null,
+ }),
+ });
+ if (!response.ok) {
+ const data = await response.json();
+ throw new Error(data.detail || "Request failed");
+ }
+ const created: EvalSession = await response.json();
+ setNewSessionName("");
+ toast.success(`Session "${created.name}" started.`);
+ await fetchSessions();
+ loadMatrix(created.session_id);
+ } catch (error) {
+ console.error("Error creating session", error);
+ toast.error(String(error));
+ } finally {
+ setIsCreating(false);
+ }
+ };
+
+ const openRun = (runId: string) => {
+ navigate(`/evals/agent-runs?run=${encodeURIComponent(runId)}`);
+ };
+
+ return (
+ <div className="flex h-screen flex-col overflow-hidden bg-zinc-950 text-white">
+ <div className="shrink-0">
+ <EvalNavigation />
+ </div>
+ <div className="mx-auto flex min-h-0 w-full max-w-[1600px] flex-1 flex-col gap-2 px-4 py-2">
+ <div className="flex shrink-0 flex-wrap items-center justify-between gap-2 rounded-xl border border-zinc-800 bg-zinc-900/80 px-4 py-2">
+ <h1 className="text-base font-semibold tracking-tight">
+ Eval Sessions
+ </h1>
+ <div className="flex items-center gap-2 text-xs">
+ <select
+ value={newSessionSet}
+ onChange={(e) => setNewSessionSet(e.target.value)}
+ className="rounded-md border border-zinc-700 bg-zinc-950 px-2 py-1 text-zinc-200"
+ >
+ {sets.length === 0 && <option value="">No sets found</option>}
+ {sets.map((set) => (
+ <option key={set.name} value={set.name}>
+ {set.display_name} ({set.image_count})
+ </option>
+ ))}
+ </select>
+ <input
+ value={newSessionName}
+ onChange={(e) => setNewSessionName(e.target.value)}
+ placeholder="Session name (optional)"
+ className="w-44 rounded-md border border-zinc-700 bg-zinc-950 px-2 py-1 text-zinc-200 placeholder:text-zinc-600"
+ />
+ <button
+ onClick={handleCreate}
+ disabled={isCreating || !newSessionSet}
+ className="rounded-md border border-emerald-800 bg-emerald-950/60 px-2.5 py-1 text-emerald-200 transition-colors hover:bg-emerald-900/60 disabled:opacity-50"
+ >
+ {isCreating ? "Starting…" : "New session"}
+ </button>
+ </div>
+ </div>
+
+ <div className="grid min-h-0 flex-1 gap-3 lg:grid-cols-[300px_minmax(0,1fr)]">
+ <section className="min-h-0 overflow-y-auto rounded-xl border border-zinc-800 bg-zinc-900/60 p-2">
+ <div className="px-2 py-1 text-xs uppercase tracking-wide text-zinc-500">
+ {sessions.length} session{sessions.length === 1 ? "" : "s"}
+ </div>
+ {sessions.length === 0 && (
+ <p className="px-2 py-4 text-sm text-zinc-400">
+ No sessions yet. Pick a set above and start one, or just run
+ evals with a set — a session is created automatically.
+ </p>
+ )}
+ {sessions.map((session) => (
+ <div
+ key={session.session_id}
+ onClick={() => loadMatrix(session.session_id)}
+ className={`mb-1.5 cursor-pointer rounded-lg border px-2.5 py-2 transition-colors ${
+ session.session_id === selectedSessionId
+ ? "border-blue-600 bg-blue-950/50"
+ : "border-zinc-800 bg-zinc-950/50 hover:bg-zinc-800/60"
+ }`}
+ >
+ <div className="flex items-center gap-2">
+ {session.is_active && (
+ <span
+ className="h-2 w-2 shrink-0 rounded-full bg-emerald-400"
+ title="Active session"
+ />
+ )}
+ <span className="min-w-0 flex-1 truncate text-xs text-zinc-100">
+ {session.name}
+ </span>
+ {!session.is_active && (
+ <button
+ onClick={(e) => {
+ e.stopPropagation();
+ handleActivate(session.session_id);
+ }}
+ className="rounded border border-zinc-700 px-1.5 py-0.5 text-[10px] text-zinc-400 hover:bg-zinc-700 hover:text-zinc-100"
+ >
+ Activate
+ </button>
+ )}
+ </div>
+ <div className="mt-1 flex items-center justify-between gap-2 text-[11px] text-zinc-500">
+ <span className="rounded-full border border-zinc-700 bg-zinc-800 px-1.5 py-0.5 text-zinc-300">
+ {session.eval_set}
+ </span>
+ <span>{formatTimestamp(session.created_at)}</span>
+ </div>
+ </div>
+ ))}
+ </section>
+
+ <section className="min-h-0 overflow-auto rounded-xl border border-zinc-800 bg-zinc-900/60 p-3">
+ {!selectedSessionId && (
+ <p className="py-8 text-center text-sm text-zinc-400">
+ Select a session to see its matrix.
+ </p>
+ )}
+ {isLoadingMatrix && (
+ <p className="py-8 text-center text-sm text-zinc-400">
+ Loading matrix…
+ </p>
+ )}
+ {!isLoadingMatrix && matrix && (
+ <div className="flex flex-col gap-2">
+ {matrix.set_missing && (
+ <div className="rounded-lg border border-amber-900 bg-amber-950/40 px-3 py-2 text-xs text-amber-200">
+ The set "{matrix.session.eval_set}" no longer exists on
+ disk — showing recorded history only.
+ </div>
+ )}
+ {matrix.unmatched_run_count > 0 && (
+ <div className="rounded-lg border border-zinc-700 bg-zinc-950/60 px-3 py-2 text-xs text-zinc-400">
+ {matrix.unmatched_run_count} run
+ {matrix.unmatched_run_count === 1 ? "" : "s"} reference
+ images no longer in the set.
+ </div>
+ )}
+ {matrix.rows.length === 0 && !matrix.set_missing && (
+ <p className="py-8 text-center text-sm text-zinc-400">
+ This set has no images. Drop PNGs into the set's inputs
+ folder.
+ </p>
+ )}
+ {matrix.models.length === 0 && matrix.rows.length > 0 && (
+ <p className="py-2 text-center text-xs text-zinc-500">
+ No runs in this session yet — run evals with this set and
+ cells will appear here.
+ </p>
+ )}
+ {matrix.rows.length > 0 && (
+ <div className="overflow-x-auto">
+ <table className="w-full border-separate border-spacing-1">
+ <thead>
+ <tr>
+ <th className="sticky left-0 z-10 bg-zinc-900/95 px-2 py-1 text-left text-[11px] font-medium uppercase tracking-wide text-zinc-500">
+ Image
+ </th>
+ {matrix.models.map((model) => (
+ <th
+ key={model}
+ className="min-w-[140px] px-2 py-1 text-left font-mono text-[11px] font-normal text-zinc-400"
+ title={model}
+ >
+ {model}
+ </th>
+ ))}
+ </tr>
+ </thead>
+ <tbody>
+ {sortedRows.map((row) => (
+ <tr key={row.filename}>
+ <td className="sticky left-0 z-10 bg-zinc-900/95 px-2 py-1">
+ <div className="flex items-center gap-2">
+ <img
+ src={`${HTTP_BACKEND_URL}${row.image_url}`}
+ alt={row.filename}
+ className="h-12 w-16 rounded border border-zinc-800 bg-zinc-800 object-cover"
+ loading="lazy"
+ />
+ <span className="max-w-[160px] truncate font-mono text-[11px] text-zinc-300">
+ {row.filename}
+ </span>
+ </div>
+ </td>
+ {matrix.models.map((model) => {
+ const cell = cellsByKey.get(
+ `${row.filename}|${model}`
+ );
+ return (
+ <td
+ key={model}
+ className="min-w-[140px] px-1 py-1 align-top"
+ >
+ {cell ? (
+ <CellFace
+ runs={cell.runs}
+ onOpenRun={openRun}
+ />
+ ) : (
+ <div className="rounded-md border border-dashed border-zinc-800 px-1.5 py-1.5 text-center text-[10px] text-zinc-700">
+ —
+ </div>
+ )}
+ </td>
+ );
+ })}
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </div>
+ )}
+ </div>
+ )}
+ </section>
+ </div>
+ </div>
+ </div>
+ );
+}
+
+export default EvalSessionsPage;
diff --git a/frontend/src/components/evals/RunEvalsPage.tsx b/frontend/src/components/evals/RunEvalsPage.tsx
index 41403d36..2923cc62 100644
--- a/frontend/src/components/evals/RunEvalsPage.tsx
+++ b/frontend/src/components/evals/RunEvalsPage.tsx
@@ -37,6 +37,22 @@ interface FailedTask {
error: string;
}
+interface EvalSetSummary {
+ name: string;
+ display_name: string;
+ image_count: number;
+}
+
+interface EvalSetImage {
+ filename: string;
+}
+
+interface ActiveEvalSession {
+ session_id: string;
+ name: string;
+ eval_set: string;
+}
+
function RunEvalsPage() {
const faviconFlashIntervalRef = useRef<number | null>(null);
const [isRunning, setIsRunning] = useState(false);
@@ -46,6 +62,14 @@ function RunEvalsPage() {
const [selectedStack, setSelectedStack] = useState<string>("html_tailwind");
const [diffMode, setDiffMode] = useState(true);
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
+ const [inputSource, setInputSource] = useState<"legacy" | "set">("legacy");
+ const [evalSets, setEvalSets] = useState<EvalSetSummary[]>([]);
+ const [selectedSet, setSelectedSet] = useState("");
+ const [setImages, setSetImages] = useState<EvalSetImage[]>([]);
+ const [selectedSetFiles, setSelectedSetFiles] = useState<string[]>([]);
+ const [activeSession, setActiveSession] = useState<ActiveEvalSession | null>(
+ null
+ );
const [showPaths, setShowPaths] = useState<boolean>(false);
const [completedTasks, setCompletedTasks] = useState(0);
const [totalTasks, setTotalTasks] = useState(0);
@@ -66,6 +90,57 @@ function RunEvalsPage() {
fetchModels();
}, []);
+ useEffect(() => {
+ const fetchEvalSets = async () => {
+ try {
+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sets`);
+ const data: EvalSetSummary[] = await response.json();
+ setEvalSets(data);
+ if (data.length > 0) {
+ setSelectedSet((current) => current || data[0].name);
+ }
+ } catch (error) {
+ console.error("Error fetching eval sets", error);
+ }
+ };
+ fetchEvalSets();
+ }, []);
+
+ const refreshActiveSession = useCallback(async () => {
+ try {
+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions/active`);
+ setActiveSession(await response.json());
+ } catch (error) {
+ console.error("Error fetching active session", error);
+ }
+ }, []);
+
+ useEffect(() => {
+ if (inputSource === "set") refreshActiveSession();
+ }, [inputSource, refreshActiveSession]);
+
+ useEffect(() => {
+ if (inputSource !== "set" || !selectedSet) return;
+ const fetchSetImages = async () => {
+ try {
+ const response = await fetch(
+ `${HTTP_BACKEND_URL}/eval-sets/${encodeURIComponent(selectedSet)}`
+ );
+ if (!response.ok) throw new Error("Request failed");
+ const data = await response.json();
+ setSetImages(data.images);
+ setSelectedSetFiles(
+ data.images.map((image: EvalSetImage) => image.filename)
+ );
+ } catch (error) {
+ console.error("Error fetching set images", error);
+ setSetImages([]);
+ setSelectedSetFiles([]);
+ }
+ };
+ fetchSetImages();
+ }, [inputSource, selectedSet]);
+
useEffect(() => {
return () => {
document.title = "Screenshot to Code";
@@ -134,7 +209,15 @@ function RunEvalsPage() {
setSkippedExistingTasks(0);
stopFaviconFlash();
- const runFiles = filesToRun ?? selectedFiles;
+ const isSetRun = inputSource === "set" && selectedSet;
+ const runFiles =
+ filesToRun ??
+ (isSetRun
+ ? // Whole set = empty list so the backend resolves every image.
+ selectedSetFiles.length === setImages.length
+ ? []
+ : selectedSetFiles
+ : selectedFiles);
const response = await fetch(`${HTTP_BACKEND_URL}/run_evals_stream`, {
method: "POST",
@@ -146,6 +229,7 @@ function RunEvalsPage() {
stack: selectedStack,
files: runFiles,
diff_mode: diffMode,
+ set_name: isSetRun ? selectedSet : null,
}),
});
@@ -273,7 +357,30 @@ function RunEvalsPage() {
return `${selectedModels.slice(0, 2).join(", ")} +${selectedModels.length - 2} more`;
};
- const canRunEvals = selectedModels.length > 0 && selectedFiles.length > 0;
+ const canRunEvals =
+ selectedModels.length > 0 &&
+ (inputSource === "set"
+ ? Boolean(selectedSet) && selectedSetFiles.length > 0
+ : selectedFiles.length > 0);
+ const sessionSetMismatch =
+ inputSource === "set" &&
+ activeSession !== null &&
+ Boolean(selectedSet) &&
+ activeSession.eval_set !== selectedSet;
+
+ const startSessionForSelectedSet = async () => {
+ try {
+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ eval_set: selectedSet }),
+ });
+ if (!response.ok) throw new Error("Request failed");
+ await refreshActiveSession();
+ } catch (error) {
+ console.error("Error starting session", error);
+ }
+ };
const failedFilesForRerun = Array.from(
new Set(failedTaskDetails.map((task) => task.inputFile).filter(Boolean))
);
@@ -510,11 +617,113 @@ function RunEvalsPage() {
{/* Input Files Section */}
<div className="bg-white rounded-lg border border-gray-200 shadow-sm lg:col-span-1 md:col-span-2">
- <div className="border-b border-gray-200 px-4 py-3 bg-gray-50 rounded-t-lg">
+ <div className="border-b border-gray-200 px-4 py-3 bg-gray-50 rounded-t-lg flex items-center justify-between">
<h2 className="font-medium">Select Input Files</h2>
+ <div className="flex rounded-md border border-gray-300 overflow-hidden text-xs">
+ {(
+ [
+ ["legacy", "Legacy inputs"],
+ ["set", "Eval set"],
+ ] as const
+ ).map(([source, label]) => (
+ <button
+ key={source}
+ onClick={() => setInputSource(source)}
+ className={`px-2.5 py-1 transition-colors ${
+ inputSource === source
+ ? "bg-blue-600 text-white"
+ : "bg-white text-gray-600 hover:bg-gray-100"
+ }`}
+ >
+ {label}
+ </button>
+ ))}
+ </div>
</div>
<div className="p-3">
- <InputFileSelector onFilesSelected={handleFilesSelected} />
+ {inputSource === "legacy" ? (
+ <InputFileSelector onFilesSelected={handleFilesSelected} />
+ ) : (
+ <div className="flex flex-col gap-3">
+ <select
+ value={selectedSet}
+ onChange={(e) => setSelectedSet(e.target.value)}
+ className="w-full py-2 px-3 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500"
+ >
+ {evalSets.length === 0 && (
+ <option value="">No eval sets found</option>
+ )}
+ {evalSets.map((set) => (
+ <option key={set.name} value={set.name}>
+ {set.display_name} ({set.image_count} images)
+ </option>
+ ))}
+ </select>
+
+ {sessionSetMismatch && activeSession && (
+ <div className="rounded-md border border-amber-300 bg-amber-50 px-3 py-2 text-xs text-amber-800">
+ Active session "{activeSession.name}" is pinned to set "
+ {activeSession.eval_set}".{" "}
+ <button
+ onClick={() => void startSessionForSelectedSet()}
+ className="font-medium underline hover:text-amber-900"
+ >
+ Start a new session for "{selectedSet}"
+ </button>{" "}
+ before running.
+ </div>
+ )}
+ {!sessionSetMismatch && inputSource === "set" && (
+ <div className="rounded-md border border-blue-200 bg-blue-50 px-3 py-2 text-xs text-blue-800">
+ {activeSession
+ ? `Runs will attach to session "${activeSession.name}" (set "${activeSession.eval_set}").`
+ : "No active session — one will be created automatically."}
+ </div>
+ )}
+
+ <div className="border rounded-md max-h-[220px] overflow-y-auto">
+ <div className="grid grid-cols-1 divide-y divide-gray-100">
+ {setImages.map((image) => {
+ const checked = selectedSetFiles.includes(
+ image.filename
+ );
+ return (
+ <div
+ key={image.filename}
+ className={`flex items-center justify-between px-3 py-1.5 cursor-pointer hover:bg-gray-50 ${
+ checked ? "bg-blue-50" : ""
+ }`}
+ onClick={() =>
+ setSelectedSetFiles((previous) =>
+ checked
+ ? previous.filter(
+ (f) => f !== image.filename
+ )
+ : [...previous, image.filename]
+ )
+ }
+ >
+ <span
+ className="text-sm truncate"
+ title={image.filename}
+ >
+ {image.filename}
+ </span>
+ {checked ? (
+ <BsCheckLg className="h-3.5 w-3.5 text-blue-500 flex-shrink-0" />
+ ) : (
+ <div className="h-3.5 w-3.5 border rounded-sm flex-shrink-0" />
+ )}
+ </div>
+ );
+ })}
+ </div>
+ </div>
+ <p className="text-xs text-gray-500">
+ Selected: {selectedSetFiles.length} / {setImages.length}
+ </p>
+ </div>
+ )}
</div>
</div>
</div>
diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx
index d2f2f065..38900450 100644
--- a/frontend/src/main.tsx
+++ b/frontend/src/main.tsx
@@ -10,6 +10,7 @@ import AllEvalsPage from "./components/evals/AllEvalsPage.tsx";
import OpenAIInputComparePage from "./components/evals/OpenAIInputComparePage.tsx";
import PromptReportsPage from "./components/evals/PromptReportsPage.tsx";
import AgentRunsPage from "./components/evals/AgentRunsPage.tsx";
+import EvalSessionsPage from "./components/evals/EvalSessionsPage.tsx";
ReactDOM.createRoot(document.getElementById("root")!).render(
<React.StrictMode>
@@ -25,6 +26,7 @@ ReactDOM.createRoot(document.getElementById("root")!).render(
/>
<Route path="/evals/prompt-reports" element={<PromptReportsPage />} />
<Route path="/evals/agent-runs" element={<AgentRunsPage />} />
+ <Route path="/evals/sessions" element={<EvalSessionsPage />} />
</Routes>
</Router>
<Toaster toastOptions={{ className: "dark:bg-zinc-950 dark:text-white" }} />