Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Add eval sets, sessions, and per-session matrix view
Abi Raja18de719bb0ba
diff --git a/backend/agent/providers/anthropic/provider.py b/backend/agent/providers/anthropic/provider.pyindex 7b5f0f31..5c48ebcd 100644--- a/backend/agent/providers/anthropic/provider.py+++ b/backend/agent/providers/anthropic/provider.py@@ -22,8 +22,8 @@ from agent.providers.anthropic.image import (process_image,process_image_bytes,)-from agent.providers.pricing import MODEL_PRICING-from agent.providers.token_usage import TokenUsage+from costs.pricing import MODEL_PRICING+from costs.token_usage import TokenUsagefrom agent.tools import CanonicalToolDefinition, ToolCall, parse_json_argumentsfrom fs_logging.agent_runs import AgentRunRecorderfrom fs_logging.prompt_reports import PromptReportLoggerdiff --git a/backend/agent/providers/gemini.py b/backend/agent/providers/gemini.pyindex 60a53593..354c7589 100644--- a/backend/agent/providers/gemini.py+++ b/backend/agent/providers/gemini.py@@ -18,8 +18,8 @@ from agent.providers.base import (ProviderTurn,StreamEvent,)-from agent.providers.pricing import MODEL_PRICING-from agent.providers.token_usage import TokenUsage+from costs.pricing import MODEL_PRICING+from costs.token_usage import TokenUsagefrom agent.tools import CanonicalToolDefinition, ToolCallfrom fs_logging.agent_runs import AgentRunRecorderfrom fs_logging.prompt_reports import PromptReportLoggerdiff --git a/backend/agent/providers/openai.py b/backend/agent/providers/openai.pyindex 2214c46b..340a693e 100644--- a/backend/agent/providers/openai.py+++ b/backend/agent/providers/openai.py@@ -16,8 +16,8 @@ from agent.providers.base import (ProviderTurn,StreamEvent,)-from agent.providers.pricing import MODEL_PRICING-from agent.providers.token_usage import TokenUsage+from costs.pricing import MODEL_PRICING+from costs.token_usage import TokenUsagefrom agent.state import ensure_strfrom agent.tools import CanonicalToolDefinition, ToolCall, parse_json_argumentsfrom fs_logging.agent_runs import AgentRunRecorder@@ -423,7 +423,7 @@ class OpenAIProviderSession(ProviderSession):model: Llm,prompt_messages: List[ChatCompletionMessageParam],tools: List[Dict[str, Any]],- recorder: Optional["AgentRunRecorder"] = None,+ recorder: Optional[AgentRunRecorder] = None,):self._client = clientself._model = modeldiff --git a/backend/costs/__init__.py b/backend/costs/__init__.pynew file mode 100644index 00000000..e86536ac--- /dev/null+++ b/backend/costs/__init__.py@@ -0,0 +1,6 @@+"""Model pricing and token accounting.++Leaf package: it must not import from agent/, fs_logging/, or routes/ so+that any layer (providers, logging, evals) can depend on it without+creating import cycles. Keep this __init__ empty for the same reason.+"""diff --git a/backend/agent/providers/pricing.py b/backend/costs/pricing.pysimilarity index 100%rename from backend/agent/providers/pricing.pyrename to backend/costs/pricing.pydiff --git a/backend/agent/providers/token_usage.py b/backend/costs/token_usage.pysimilarity index 98%rename from backend/agent/providers/token_usage.pyrename to backend/costs/token_usage.pyindex 2141a3c4..a9b0619e 100644--- a/backend/agent/providers/token_usage.py+++ b/backend/costs/token_usage.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom dataclasses import dataclass-from agent.providers.pricing import ModelPricing+from costs.pricing import ModelPricing@dataclassdiff --git a/backend/evals/core.py b/backend/evals/core.pyindex 8e59eb9d..b938e1aa 100644--- a/backend/evals/core.py+++ b/backend/evals/core.py@@ -18,7 +18,15 @@ from openai.types.chat import ChatCompletionMessageParamfrom typing import Any-async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> str:+async def generate_code_for_image(+ image_url: str,+ stack: Stack,+ model: Llm,+ *,+ eval_set: str | None = None,+ eval_session_id: str | None = None,+ input_file: str | None = None,+) -> str:prompt_messages = build_image_prompt_messages(image_data_urls=[image_url],stack=stack,@@ -53,6 +61,9 @@ async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> sstack=str(stack),input_mode="image",generation_type="create",+ eval_session=eval_session_id,+ eval_set=eval_set,+ input_file=input_file,)runner = Agent(send_message=send_message,diff --git a/backend/evals/runner.py b/backend/evals/runner.pyindex aa4b1181..d7673831 100644--- a/backend/evals/runner.py+++ b/backend/evals/runner.py@@ -8,6 +8,7 @@ from llm import Llmfrom config import LOCAL_ASSET_BASE_URLfrom prompts.prompt_types import Stackfrom .core import generate_code_for_image+from .sets import get_set_inputs_dirfrom .utils import image_to_data_urlfrom .config import EVALS_DIR@@ -36,8 +37,15 @@ def normalize_local_asset_urls(html: str) -> str:return html-def _resolve_eval_filenames(input_files: Optional[List[str]]) -> List[str]:- input_dir = EVALS_DIR + "/inputs"+def _get_input_dir(eval_set: Optional[str]) -> str:+ if eval_set:+ return get_set_inputs_dir(eval_set)+ return EVALS_DIR + "/inputs"+++def _resolve_eval_filenames(+ input_files: Optional[List[str]], input_dir: str+) -> List[str]:if input_files and len(input_files) > 0:return [os.path.basename(f) for f in input_files if f.endswith(".png")]return [f for f in os.listdir(input_dir) if f.endswith(".png")]@@ -59,11 +67,21 @@ def count_pending_eval_tasks(input_files: Optional[List[str]] = None,n: int = 1,diff_mode: bool = False,+ eval_set: Optional[str] = None,+ skip_input_files: Optional[set[str]] = None,) -> Tuple[int, int]:- evals = _resolve_eval_filenames(input_files)+ evals = _resolve_eval_filenames(input_files, _get_input_dir(eval_set))if not diff_mode:return len(evals) * n, 0+ # Set runs skip by session history (the caller queries the run index);+ # day-named output folders are shared across sets, so file existence+ # would produce false positives there.+ if eval_set is not None:+ skip = skip_input_files or set()+ skipped = sum(n for f in evals if f in skip)+ return len(evals) * n - skipped, skipped+output_subfolder = get_eval_output_subfolder(stack=stack, model=model)pending_tasks = 0skipped_existing_tasks = 0@@ -84,6 +102,8 @@ async def generate_code_and_time(model: Llm,original_input_filename: str,attempt_idx: int,+ eval_set: Optional[str] = None,+ eval_session_id: Optional[str] = None,) -> Tuple[str, int, Optional[str], Optional[float], Optional[Exception], int]:"""Generates code for an image, measures the time taken, and returns identifiers@@ -97,7 +117,12 @@ async def generate_code_and_time(start_time = time.perf_counter()try:content = await generate_code_for_image(- image_url=image_url, stack=stack, model=model+ image_url=image_url,+ stack=stack,+ model=model,+ eval_set=eval_set,+ eval_session_id=eval_session_id,+ input_file=original_input_filename,)end_time = time.perf_counter()duration = end_time - start_time@@ -137,9 +162,12 @@ async def run_image_evals(input_files: Optional[List[str]] = None,diff_mode: bool = False,progress_callback: Optional[Callable[[dict[str, Any]], Any | Awaitable[Any]]] = None,+ eval_set: Optional[str] = None,+ eval_session_id: Optional[str] = None,+ skip_input_files: Optional[set[str]] = None,) -> List[str]:- INPUT_DIR = EVALS_DIR + "/inputs"- evals = _resolve_eval_filenames(input_files)+ INPUT_DIR = _get_input_dir(eval_set)+ evals = _resolve_eval_filenames(input_files, INPUT_DIR)if not stack:raise ValueError("No stack was provided")@@ -182,9 +210,17 @@ async def run_image_evals(for n_idx in range(n):output_filename = _output_html_filename(original_filename, n_idx)output_path = os.path.join(output_subfolder, output_filename)- if diff_mode and os.path.exists(output_path):- skipped_existing_tasks += 1- continue+ if diff_mode:+ # Set runs skip by session history (authoritative — shared+ # day-named output folders collide across sets); the legacy+ # path keeps the original file-existence check.+ if eval_set is not None:+ if original_filename in (skip_input_files or set()):+ skipped_existing_tasks += 1+ continue+ elif os.path.exists(output_path):+ skipped_existing_tasks += 1+ continueif data_url is None:data_url = await image_to_data_url(filepath)@@ -197,6 +233,8 @@ async def run_image_evals(model=current_model_for_task,original_input_filename=original_filename,attempt_idx=n_idx,+ eval_set=eval_set,+ eval_session_id=eval_session_id,)task_coroutines.append(coro)diff --git a/backend/evals/sessions.py b/backend/evals/sessions.pynew file mode 100644index 00000000..2610fb70--- /dev/null+++ b/backend/evals/sessions.py@@ -0,0 +1,178 @@+"""Eval sessions: a work period pinned to one eval set.++Exactly one session is active at a time; starting a new one supersedes the+old (sessions are never "ended"). Eval runs launched while a session is+active attach to it via the agent-run recorder, which makes the session+matrix (images x models) a plain query over the runs index. Sessions never+gate the UI generation path — they are pure metadata.++Stored in the agent-runs SQLite index (see ``fs_logging/agent_runs.py`` for+the schema). Timestamps use the same local-ISO format as ``runs.created_at``+so SQL string comparison against run timestamps is valid.+"""++import sqlite3+import uuid+from dataclasses import dataclass+from datetime import datetime+from typing import Optional++from fs_logging.agent_runs import open_index_db+++@dataclass+class EvalSession:+ session_id: str+ name: str+ eval_set: str+ created_at: str+ is_active: bool+++class SessionSetMismatchError(Exception):+ def __init__(self, active_session: EvalSession, requested_set: str) -> None:+ self.active_session = active_session+ self.requested_set = requested_set+ super().__init__(+ f"Active session {active_session.name!r} is pinned to set "+ f"{active_session.eval_set!r}, not {requested_set!r}"+ )+++_SESSION_COLUMNS = "session_id, name, eval_set, created_at, is_active"+++def _row_to_session(row: tuple[str, str, str, str, int]) -> EvalSession:+ return EvalSession(+ session_id=row[0],+ name=row[1],+ eval_set=row[2],+ created_at=row[3],+ is_active=bool(row[4]),+ )+++def default_session_name() -> str:+ now = datetime.now()+ # %-d is not portable; strip the leading zero manually.+ return f"{now.strftime('%b')} {now.day} session"+++def _activate_in_transaction(conn: sqlite3.Connection, session_id: str) -> None:+ conn.execute("BEGIN IMMEDIATE")+ conn.execute("UPDATE eval_sessions SET is_active = 0 WHERE is_active = 1")+ conn.execute(+ "UPDATE eval_sessions SET is_active = 1 WHERE session_id = ?",+ (session_id,),+ )+ conn.commit()+++def create_session(eval_set: str, name: Optional[str] = None) -> EvalSession:+ session = EvalSession(+ session_id=(+ f"sess_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"+ ),+ name=name or default_session_name(),+ eval_set=eval_set,+ created_at=datetime.now().isoformat(timespec="seconds"),+ is_active=True,+ )+ conn = open_index_db()+ try:+ conn.execute(+ "INSERT INTO eval_sessions "+ f"({_SESSION_COLUMNS}) VALUES (?, ?, ?, ?, 0)",+ (+ session.session_id,+ session.name,+ session.eval_set,+ session.created_at,+ ),+ )+ conn.commit()+ try:+ _activate_in_transaction(conn, session.session_id)+ except sqlite3.IntegrityError:+ # Concurrent activation raced the partial unique index; retry once.+ _activate_in_transaction(conn, session.session_id)+ finally:+ conn.close()+ return session+++def list_sessions() -> list[EvalSession]:+ conn = open_index_db()+ try:+ rows = conn.execute(+ f"SELECT {_SESSION_COLUMNS} FROM eval_sessions "+ "ORDER BY created_at DESC, session_id DESC"+ ).fetchall()+ finally:+ conn.close()+ return [_row_to_session(row) for row in rows]+++def get_session(session_id: str) -> Optional[EvalSession]:+ conn = open_index_db()+ try:+ row = conn.execute(+ f"SELECT {_SESSION_COLUMNS} FROM eval_sessions WHERE session_id = ?",+ (session_id,),+ ).fetchone()+ finally:+ conn.close()+ return _row_to_session(row) if row else None+++def get_active_session() -> Optional[EvalSession]:+ conn = open_index_db()+ try:+ row = conn.execute(+ f"SELECT {_SESSION_COLUMNS} FROM eval_sessions WHERE is_active = 1"+ ).fetchone()+ finally:+ conn.close()+ return _row_to_session(row) if row else None+++def activate_session(session_id: str) -> Optional[EvalSession]:+ conn = open_index_db()+ try:+ try:+ _activate_in_transaction(conn, session_id)+ except sqlite3.IntegrityError:+ _activate_in_transaction(conn, session_id)+ finally:+ conn.close()+ return get_session(session_id)+++def resolve_session_for_run(set_name: str) -> EvalSession:+ """The session eval runs should attach to for this set.++ Active session on the same set wins; a mismatched active session is an+ error (the caller must explicitly start a new session); no session at+ all auto-creates one so a bare "/run_evals with a set" just works.+ """+ active = get_active_session()+ if active is None:+ return create_session(set_name)+ if active.eval_set != set_name:+ raise SessionSetMismatchError(active, set_name)+ return active+++def completed_eval_inputs(session_id: str, model: str, stack: str) -> set[str]:+ """Input filenames already completed in this session for (model, stack)."""+ conn = open_index_db()+ try:+ rows = conn.execute(+ "SELECT DISTINCT input_file FROM runs "+ "WHERE eval_session = ? AND model = ? AND stack = ? "+ "AND status = 'completed' AND input_file IS NOT NULL",+ (session_id, model, stack),+ ).fetchall()+ finally:+ conn.close()+ return {row[0] for row in rows}diff --git a/backend/evals/sets.py b/backend/evals/sets.pynew file mode 100644index 00000000..bb3257f6--- /dev/null+++ b/backend/evals/sets.py@@ -0,0 +1,224 @@+"""Named eval sets: folders of input images under ``{EVALS_DIR}/sets``.++A set is created by dropping PNGs into ``sets/{name}/inputs/`` — there is no+creation API. ``manifest.json`` (bootstrapped on first listing) carries the+display name, notes, and a per-image sha256 cache keyed by (size, mtime) so+content hashes are computed once. The sha256 of an image's bytes is how UI+generations are matched to matrix rows.+"""++import hashlib+import json+import os+import re+from dataclasses import dataclass+from datetime import datetime+from typing import Any, Optional, cast++import evals.config as evals_config++# Blocks path traversal while allowing human-friendly names like+# "jun-21-evals" or "Landing Pages v2".+_SET_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._ -]*$")+++class InvalidSetNameError(Exception):+ pass+++class EvalSetNotFoundError(Exception):+ pass+++@dataclass+class EvalSetImage:+ filename: str+ sha256: str+ size_bytes: int+ mtime: float+ tags: list[str]+++@dataclass+class EvalSetInfo:+ name: str+ display_name: str+ created_at: Optional[str]+ notes: str+ image_count: int+++def get_sets_dir() -> str:+ return os.path.join(evals_config.EVALS_DIR, "sets")+++def _validate_set_name(set_name: str) -> str:+ if not _SET_NAME_PATTERN.match(set_name):+ raise InvalidSetNameError(f"Invalid eval set name: {set_name!r}")+ return set_name+++def _get_set_dir(set_name: str) -> str:+ return os.path.join(get_sets_dir(), _validate_set_name(set_name))+++def get_set_inputs_dir(set_name: str) -> str:+ return os.path.join(_get_set_dir(set_name), "inputs")+++def _manifest_path(set_name: str) -> str:+ return os.path.join(_get_set_dir(set_name), "manifest.json")+++def _load_manifest(set_name: str) -> dict[str, Any]:+ try:+ with open(_manifest_path(set_name), "r", encoding="utf-8") as f:+ loaded = cast(Any, json.load(f))+ if isinstance(loaded, dict):+ return cast(dict[str, Any], loaded)+ return {}+ except (OSError, json.JSONDecodeError):+ return {}+++def _tags_of(entry: Optional[dict[str, Any]]) -> list[str]:+ if not entry:+ return []+ raw = entry.get("tags")+ if not isinstance(raw, list):+ return []+ return [tag for tag in cast(list[object], raw) if isinstance(tag, str)]+++def _save_manifest(set_name: str, manifest: dict[str, Any]) -> None:+ try:+ with open(_manifest_path(set_name), "w", encoding="utf-8") as f:+ json.dump(manifest, f, indent=2, ensure_ascii=False)+ except OSError as exc:+ # The manifest is a cache/metadata sidecar; failing to persist it+ # must not fail set operations.+ print(f"[EVAL SETS] Failed to write manifest for {set_name}: {exc}")+++def _sha256_of_file(path: str) -> str:+ digest = hashlib.sha256()+ with open(path, "rb") as f:+ for chunk in iter(lambda: f.read(1 << 20), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def list_set_images(set_name: str) -> list[EvalSetImage]:+ """PNGs of the set, hashes served from the manifest cache when fresh."""+ inputs_dir = get_set_inputs_dir(set_name)+ if not os.path.isdir(inputs_dir):+ raise EvalSetNotFoundError(f"Eval set not found: {set_name}")++ manifest = _load_manifest(set_name)+ raw_images = manifest.get("images")+ cached_images: dict[str, Any] = (+ cast(dict[str, Any], raw_images) if isinstance(raw_images, dict) else {}+ )+ manifest_changed = not manifest++ filenames = sorted(+ entry+ for entry in os.listdir(inputs_dir)+ if entry.lower().endswith(".png")+ and os.path.isfile(os.path.join(inputs_dir, entry))+ )++ images: list[EvalSetImage] = []+ fresh_entries: dict[str, Any] = {}+ for filename in filenames:+ path = os.path.join(inputs_dir, filename)+ stat = os.stat(path)+ raw_cached = cached_images.get(filename)+ cached: Optional[dict[str, Any]] = (+ cast(dict[str, Any], raw_cached)+ if isinstance(raw_cached, dict)+ else None+ )+ tags = _tags_of(cached)+ if (+ cached is not None+ and cached.get("size") == stat.st_size+ and cached.get("mtime") == stat.st_mtime+ and isinstance(cached.get("sha256"), str)+ ):+ sha256 = str(cached["sha256"])+ else:+ sha256 = _sha256_of_file(path)+ manifest_changed = True+ fresh_entries[filename] = {+ "sha256": sha256,+ "size": stat.st_size,+ "mtime": stat.st_mtime,+ "tags": tags,+ }+ images.append(+ EvalSetImage(+ filename=filename,+ sha256=sha256,+ size_bytes=stat.st_size,+ mtime=stat.st_mtime,+ tags=tags,+ )+ )++ if set(fresh_entries) != set(cached_images):+ manifest_changed = True+ if manifest_changed:+ manifest.setdefault("display_name", set_name)+ manifest.setdefault(+ "created_at", datetime.now().isoformat(timespec="seconds")+ )+ manifest.setdefault("notes", "")+ manifest["images"] = fresh_entries+ _save_manifest(set_name, manifest)++ return images+++def get_set(set_name: str) -> EvalSetInfo:+ images = list_set_images(set_name)+ manifest = _load_manifest(set_name)+ return EvalSetInfo(+ name=set_name,+ display_name=str(manifest.get("display_name") or set_name),+ created_at=(+ str(manifest["created_at"]) if manifest.get("created_at") else None+ ),+ notes=str(manifest.get("notes") or ""),+ image_count=len(images),+ )+++def list_sets() -> list[EvalSetInfo]:+ sets_dir = get_sets_dir()+ if not os.path.isdir(sets_dir):+ return []+ infos: list[EvalSetInfo] = []+ for entry in sorted(os.listdir(sets_dir)):+ if not _SET_NAME_PATTERN.match(entry):+ continue+ if not os.path.isdir(os.path.join(sets_dir, entry, "inputs")):+ continue+ infos.append(get_set(entry))+ return infos+++def resolve_set_image_path(set_name: str, filename: str) -> str:+ """Absolute path of a set image; raises on traversal or missing file."""+ inputs_dir = get_set_inputs_dir(set_name)+ if not os.path.isdir(inputs_dir):+ raise EvalSetNotFoundError(f"Eval set not found: {set_name}")+ safe_name = os.path.basename(filename)+ if not safe_name.lower().endswith(".png"):+ raise InvalidSetNameError(f"Not a set image: {filename!r}")+ path = os.path.realpath(os.path.join(inputs_dir, safe_name))+ if not path.startswith(os.path.realpath(inputs_dir) + os.sep):+ raise InvalidSetNameError(f"Invalid image path: {filename!r}")+ if not os.path.isfile(path):+ raise FileNotFoundError(path)+ return pathdiff --git a/backend/fs_logging/agent_runs.py b/backend/fs_logging/agent_runs.pyindex 5b355de7..fbdd3910 100644--- a/backend/fs_logging/agent_runs.py+++ b/backend/fs_logging/agent_runs.py@@ -22,6 +22,7 @@ Recording must never break generation: every public method no-ops whendisabled and swallows its own exceptions."""+import base64import hashlibimport jsonimport os@@ -31,13 +32,15 @@ import sqlite3import timeimport uuidfrom datetime import datetime-from typing import TYPE_CHECKING, Any, Optional+from typing import TYPE_CHECKING, Any, List, Optional, castfrom urllib.parse import unquote, urlparse+from openai.types.chat import ChatCompletionMessageParam+from config import LOCAL_ASSET_DIR, PROMPT_REPORTS_ENABLEDfrom llm import MODEL_PROVIDER, Llm-from agent.providers.pricing import MODEL_PRICING-from agent.providers.token_usage import TokenUsage+from costs.pricing import MODEL_PRICING+from costs.token_usage import TokenUsagefrom fs_logging.prompt_reports import get_run_logs_directory, to_serializableif TYPE_CHECKING:@@ -100,10 +103,23 @@ CREATE TABLE IF NOT EXISTS runs (total_cost_usd REAL,has_unpriced_calls INTEGER NOT NULL DEFAULT 0,run_dir TEXT NOT NULL,- size_bytes INTEGER+ size_bytes INTEGER,+ eval_session TEXT,+ eval_set TEXT,+ input_file TEXT,+ input_image_sha256 TEXT);CREATE INDEX IF NOT EXISTS idx_runs_generation ON runs(generation_id);CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(created_at);+CREATE TABLE IF NOT EXISTS eval_sessions (+ session_id TEXT PRIMARY KEY,+ name TEXT NOT NULL,+ eval_set TEXT NOT NULL,+ created_at TEXT NOT NULL,+ is_active INTEGER NOT NULL DEFAULT 0+);+CREATE UNIQUE INDEX IF NOT EXISTS idx_eval_sessions_active+ ON eval_sessions(is_active) WHERE is_active = 1;CREATE TABLE IF NOT EXISTS llm_calls (run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE,step INTEGER NOT NULL,@@ -123,6 +139,27 @@ CREATE TABLE IF NOT EXISTS llm_calls ("""+def _migrate_runs_table(conn: sqlite3.Connection) -> None:+ """Add columns introduced after the first release.++ ``CREATE TABLE IF NOT EXISTS`` never alters an existing table, so a+ populated index.db from before these columns needs explicit ALTERs. The+ matching indexes must live here too — creating them in ``_SCHEMA`` would+ fail against an un-migrated table.+ """+ existing = {row[1] for row in conn.execute("PRAGMA table_info(runs)")}+ for column in ("eval_session", "eval_set", "input_file", "input_image_sha256"):+ if column not in existing:+ conn.execute(f"ALTER TABLE runs ADD COLUMN {column} TEXT")+ conn.execute(+ "CREATE INDEX IF NOT EXISTS idx_runs_eval_session ON runs(eval_session)"+ )+ conn.execute(+ "CREATE INDEX IF NOT EXISTS idx_runs_input_sha ON runs(input_image_sha256)"+ )+ conn.commit()++def open_index_db() -> sqlite3.Connection:os.makedirs(get_agent_runs_directory(), exist_ok=True)conn = sqlite3.connect(get_agent_runs_db_path(), timeout=5)@@ -131,6 +168,7 @@ def open_index_db() -> sqlite3.Connection:conn.execute("PRAGMA busy_timeout=5000")conn.execute("PRAGMA foreign_keys=ON")conn.executescript(_SCHEMA)+ _migrate_runs_table(conn)return conn@@ -138,6 +176,41 @@ def _now_iso() -> str:return datetime.now().isoformat(timespec="seconds")+def _first_input_image_sha256(prompt_messages: Any) -> Optional[str]:+ """Digest of the first input image's raw bytes, or None.++ Mirrors ``AgentEngine._extract_input_images`` — duplicated because this+ module must not import ``agent.engine`` (the engine imports us). Since+ eval inputs are base64-encoded file bytes verbatim, this digest equals+ the sha256 of the PNG on disk, which is the matrix-matching invariant.+ """+ try:+ messages = cast(List[ChatCompletionMessageParam], prompt_messages or [])+ for message in messages:+ if not isinstance(message, dict):+ continue+ content = message.get("content")+ if not isinstance(content, list):+ continue+ for part in content:+ if not isinstance(part, dict) or part.get("type") != "image_url":+ continue+ image_url = part.get("image_url")+ if not isinstance(image_url, dict):+ continue+ url = cast(object, image_url.get("url"))+ if (+ isinstance(url, str)+ and url.startswith("data:image/")+ and "," in url+ ):+ payload = base64.b64decode(url.split(",", 1)[1])+ return hashlib.sha256(payload).hexdigest()+ except Exception:+ return None+ return None++def _guess_extension(url: str, content_type: Optional[str]) -> str:path_ext = os.path.splitext(urlparse(url).path)[1].lower()if path_ext in _IMAGE_EXTENSIONS:@@ -163,6 +236,9 @@ class AgentRunRecorder:input_mode: Optional[str] = None,generation_type: Optional[str] = None,enabled: Optional[bool] = None,+ eval_session: Optional[str] = None,+ eval_set: Optional[str] = None,+ input_file: Optional[str] = None,) -> None:self.enabled = PROMPT_REPORTS_ENABLED if enabled is None else enabledself.generation_id = generation_id@@ -171,6 +247,10 @@ class AgentRunRecorder:self.stack = stackself.input_mode = input_modeself.generation_type = generation_type+ self.eval_session = eval_session+ self.eval_set = eval_set+ self.input_file = input_file+ self._input_image_sha256: Optional[str] = Noneself.run_id = (f"run_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}")@@ -249,6 +329,7 @@ class AgentRunRecorder:self._run_started_mono = time.perf_counter()self._model = model.valueself._provider = MODEL_PROVIDER.get(model)+ self._input_image_sha256 = _first_input_image_sha256(prompt_messages)self._append_event("run_start",{@@ -261,6 +342,10 @@ class AgentRunRecorder:"stack": self.stack,"input_mode": self.input_mode,"generation_type": self.generation_type,+ "eval_session": self.eval_session,+ "eval_set": self.eval_set,+ "input_file": self.input_file,+ "input_image_sha256": self._input_image_sha256,"prompt_messages": prompt_messages,},)@@ -268,8 +353,9 @@ class AgentRunRecorder:"""INSERT OR REPLACE INTO runs(run_id, generation_id, variant_index, entry_point, provider,model, stack, input_mode, generation_type, status,- created_at, run_dir)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', ?, ?)""",+ created_at, run_dir, eval_session, eval_set, input_file,+ input_image_sha256)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', ?, ?, ?, ?, ?, ?)""",(self.run_id,self.generation_id,@@ -282,6 +368,10 @@ class AgentRunRecorder:self.generation_type,_now_iso(),self.run_dir,+ self.eval_session,+ self.eval_set,+ self.input_file,+ self._input_image_sha256,),)except Exception as exc:@@ -594,6 +684,10 @@ class AgentRunRecorder:"stack": self.stack,"input_mode": self.input_mode,"generation_type": self.generation_type,+ "eval_session": self.eval_session,+ "eval_set": self.eval_set,+ "input_file": self.input_file,+ "input_image_sha256": self._input_image_sha256,"status": status,"error": error,"completed_at": _now_iso(),diff --git a/backend/fs_logging/prompt_reports.py b/backend/fs_logging/prompt_reports.pyindex 37879b93..cbacd587 100644--- a/backend/fs_logging/prompt_reports.py+++ b/backend/fs_logging/prompt_reports.py@@ -19,8 +19,8 @@ from datetime import datetimefrom enum import Enumfrom typing import Any-from agent.providers.pricing import MODEL_PRICING-from agent.providers.token_usage import TokenUsage+from costs.pricing import MODEL_PRICING+from costs.token_usage import TokenUsagefrom agent.state import ensure_strfrom config import PROMPT_REPORTS_ENABLEDfrom llm import Llmdiff --git a/backend/main.py b/backend/main.pyindex 3a4b8541..aa83aa91 100644--- a/backend/main.py+++ b/backend/main.py@@ -16,6 +16,7 @@ from routes import (design_systems,prompt_reports,agent_runs,+ eval_sets,)from uploaded_assets import configure_uploaded_asset_routes@@ -55,3 +56,4 @@ app.include_router(export.router)app.include_router(design_systems.router)app.include_router(prompt_reports.router)app.include_router(agent_runs.router)+app.include_router(eval_sets.router)diff --git a/backend/routes/agent_runs.py b/backend/routes/agent_runs.pyindex a1d1dee2..33628c51 100644--- a/backend/routes/agent_runs.py+++ b/backend/routes/agent_runs.py@@ -54,6 +54,10 @@ _RUN_COLUMNS = ["total_cost_usd","has_unpriced_calls","size_bytes",+ "eval_session",+ "eval_set",+ "input_file",+ "input_image_sha256",]@@ -85,6 +89,10 @@ class AgentRunSummary(BaseModel):total_cost_usd: Optional[float]has_unpriced_calls: boolsize_bytes: Optional[int]+ eval_session: Optional[str]+ eval_set: Optional[str]+ input_file: Optional[str]+ input_image_sha256: Optional[str]class AgentRunListResponse(BaseModel):diff --git a/backend/routes/eval_sets.py b/backend/routes/eval_sets.pynew file mode 100644index 00000000..39df8eb2--- /dev/null+++ b/backend/routes/eval_sets.py@@ -0,0 +1,304 @@+"""API for eval sets, eval sessions, and the per-session run matrix."""++from datetime import datetime, timedelta+from typing import Any, Dict, List, Optional++from fastapi import APIRouter, HTTPException+from fastapi.responses import FileResponse+from pydantic import BaseModel++from evals import sessions as eval_sessions+from evals import sets as eval_sets+from fs_logging.agent_runs import open_index_db++router = APIRouter()++_STALE_RUNNING_AFTER = timedelta(hours=2)+++class EvalSetSummaryModel(BaseModel):+ name: str+ display_name: str+ created_at: Optional[str]+ notes: str+ image_count: int+++class EvalSetImageModel(BaseModel):+ filename: str+ sha256: str+ size_bytes: int+ tags: List[str]+++class EvalSetDetailModel(EvalSetSummaryModel):+ images: List[EvalSetImageModel]+++class EvalSessionModel(BaseModel):+ session_id: str+ name: str+ eval_set: str+ created_at: str+ is_active: bool+++class EvalSessionListResponse(BaseModel):+ sessions: List[EvalSessionModel]+ active_session_id: Optional[str]+++class CreateEvalSessionRequest(BaseModel):+ eval_set: str+ name: Optional[str] = None+++class MatrixRunModel(BaseModel):+ run_id: str+ status: str+ source: str # "eval" | "ui"+ total_duration_ms: Optional[int]+ total_cost_usd: Optional[float]+ created_at: str+ is_stale: bool+++class MatrixCellModel(BaseModel):+ filename: str+ model: str+ runs: List[MatrixRunModel] # newest first+++class MatrixRowModel(BaseModel):+ filename: str+ sha256: str+ image_url: str+++class SessionMatrixResponse(BaseModel):+ session: EvalSessionModel+ eval_set: Optional[EvalSetSummaryModel]+ set_missing: bool+ rows: List[MatrixRowModel]+ models: List[str]+ cells: List[MatrixCellModel]+ unmatched_run_count: int+++def _set_info_to_model(info: eval_sets.EvalSetInfo) -> EvalSetSummaryModel:+ return EvalSetSummaryModel(+ name=info.name,+ display_name=info.display_name,+ created_at=info.created_at,+ notes=info.notes,+ image_count=info.image_count,+ )+++def _session_to_model(session: eval_sessions.EvalSession) -> EvalSessionModel:+ return EvalSessionModel(+ session_id=session.session_id,+ name=session.name,+ eval_set=session.eval_set,+ created_at=session.created_at,+ is_active=session.is_active,+ )++[email protected]("/eval-sets", response_model=List[EvalSetSummaryModel])+async def list_eval_sets() -> List[EvalSetSummaryModel]:+ return [_set_info_to_model(info) for info in eval_sets.list_sets()]++[email protected]("/eval-sets/{set_name}", response_model=EvalSetDetailModel)+async def get_eval_set(set_name: str) -> EvalSetDetailModel:+ try:+ info = eval_sets.get_set(set_name)+ images = eval_sets.list_set_images(set_name)+ except eval_sets.InvalidSetNameError as e:+ raise HTTPException(status_code=400, detail=str(e))+ except eval_sets.EvalSetNotFoundError:+ raise HTTPException(status_code=404, detail=f"Eval set not found: {set_name}")+ return EvalSetDetailModel(+ **_set_info_to_model(info).model_dump(),+ images=[+ EvalSetImageModel(+ filename=image.filename,+ sha256=image.sha256,+ size_bytes=image.size_bytes,+ tags=image.tags,+ )+ for image in images+ ],+ )++[email protected]("/eval-sets/{set_name}/images/{filename}")+async def get_eval_set_image(set_name: str, filename: str) -> FileResponse:+ try:+ path = eval_sets.resolve_set_image_path(set_name, filename)+ except eval_sets.InvalidSetNameError as e:+ raise HTTPException(status_code=400, detail=str(e))+ except eval_sets.EvalSetNotFoundError:+ raise HTTPException(status_code=404, detail=f"Eval set not found: {set_name}")+ except FileNotFoundError:+ raise HTTPException(status_code=404, detail="Image not found")+ return FileResponse(path)++[email protected]("/eval-sessions", response_model=EvalSessionListResponse)+async def list_eval_sessions() -> EvalSessionListResponse:+ sessions = eval_sessions.list_sessions()+ active = next((s for s in sessions if s.is_active), None)+ return EvalSessionListResponse(+ sessions=[_session_to_model(s) for s in sessions],+ active_session_id=active.session_id if active else None,+ )++[email protected]("/eval-sessions/active", response_model=Optional[EvalSessionModel])+async def get_active_eval_session() -> Optional[EvalSessionModel]:+ active = eval_sessions.get_active_session()+ return _session_to_model(active) if active else None++[email protected]("/eval-sessions", response_model=EvalSessionModel)+async def create_eval_session(request: CreateEvalSessionRequest) -> EvalSessionModel:+ try:+ eval_sets.get_set(request.eval_set)+ except eval_sets.InvalidSetNameError as e:+ raise HTTPException(status_code=400, detail=str(e))+ except eval_sets.EvalSetNotFoundError:+ raise HTTPException(+ status_code=404, detail=f"Eval set not found: {request.eval_set}"+ )+ session = eval_sessions.create_session(request.eval_set, request.name)+ return _session_to_model(session)+++ "/eval-sessions/{session_id}/activate", response_model=EvalSessionModel+)+async def activate_eval_session(session_id: str) -> EvalSessionModel:+ if eval_sessions.get_session(session_id) is None:+ raise HTTPException(status_code=404, detail="Session not found")+ session = eval_sessions.activate_session(session_id)+ assert session is not None+ return _session_to_model(session)+++def _is_stale_running(status: str, created_at: str) -> bool:+ if status != "running":+ return False+ try:+ started = datetime.fromisoformat(created_at)+ except ValueError:+ return True+ return datetime.now() - started > _STALE_RUNNING_AFTER+++ "/eval-sessions/{session_id}/matrix", response_model=SessionMatrixResponse+)+async def get_session_matrix(session_id: str) -> SessionMatrixResponse:+ session = eval_sessions.get_session(session_id)+ if session is None:+ raise HTTPException(status_code=404, detail="Session not found")++ set_missing = False+ set_info: Optional[eval_sets.EvalSetInfo] = None+ images: List[eval_sets.EvalSetImage] = []+ try:+ set_info = eval_sets.get_set(session.eval_set)+ images = eval_sets.list_set_images(session.eval_set)+ except (eval_sets.EvalSetNotFoundError, eval_sets.InvalidSetNameError):+ set_missing = True++ rows = [+ MatrixRowModel(+ filename=image.filename,+ sha256=image.sha256,+ image_url=f"/eval-sets/{session.eval_set}/images/{image.filename}",+ )+ for image in images+ ]+ set_filenames = {image.filename for image in images}+ filenames_by_sha: Dict[str, List[str]] = {}+ for image in images:+ filenames_by_sha.setdefault(image.sha256, []).append(image.filename)++ run_query_columns = (+ "run_id, model, status, total_duration_ms, total_cost_usd, "+ "created_at, input_file, input_image_sha256"+ )+ conn = open_index_db()+ try:+ eval_rows = conn.execute(+ f"SELECT {run_query_columns} FROM runs "+ "WHERE eval_session = ? ORDER BY created_at DESC, run_id DESC",+ (session_id,),+ ).fetchall()+ ui_rows: List[tuple[Any, ...]] = []+ if filenames_by_sha:+ placeholders = ", ".join("?" for _ in filenames_by_sha)+ ui_rows = conn.execute(+ f"SELECT {run_query_columns} FROM runs "+ "WHERE eval_session IS NULL AND entry_point != 'eval' "+ f"AND created_at >= ? AND input_image_sha256 IN ({placeholders}) "+ "ORDER BY created_at DESC, run_id DESC",+ (session.created_at, *filenames_by_sha.keys()),+ ).fetchall()+ finally:+ conn.close()++ # (filename, model) -> runs, insertion-ordered so cells stay newest-first.+ cell_runs: Dict[tuple[str, str], List[MatrixRunModel]] = {}+ model_first_seen: Dict[str, str] = {}+ unmatched_run_count = 0++ def add_run(filename: str, row: tuple[Any, ...], source: str) -> None:+ run_id, model, status, duration, cost, created_at = row[:6]+ cell_runs.setdefault((filename, model), []).append(+ MatrixRunModel(+ run_id=run_id,+ status=status,+ source=source,+ total_duration_ms=duration,+ total_cost_usd=cost,+ created_at=created_at,+ is_stale=_is_stale_running(status, created_at),+ )+ )+ # Rows are scanned newest-first; keep overwriting so the oldest+ # occurrence wins and columns order by first use in the session.+ model_first_seen[model] = created_at++ for row in eval_rows:+ input_file = row[6]+ # With a missing set the cells still carry the session's history;+ # otherwise runs whose file left the set are counted, not shown.+ if input_file and (set_missing or input_file in set_filenames):+ add_run(input_file, row, "eval")+ else:+ unmatched_run_count += 1++ for row in ui_rows:+ sha = row[7]+ for filename in filenames_by_sha.get(sha, []):+ add_run(filename, row, "ui")++ models = sorted(model_first_seen, key=lambda m: model_first_seen[m])+ cells = [+ MatrixCellModel(filename=filename, model=model, runs=runs)+ for (filename, model), runs in cell_runs.items()+ ]++ return SessionMatrixResponse(+ session=_session_to_model(session),+ eval_set=_set_info_to_model(set_info) if set_info else None,+ set_missing=set_missing,+ rows=rows,+ models=models,+ cells=cells,+ unmatched_run_count=unmatched_run_count,+ )diff --git a/backend/routes/evals.py b/backend/routes/evals.pyindex 8fc78646..be9cef1a 100644--- a/backend/routes/evals.py+++ b/backend/routes/evals.py@@ -6,8 +6,10 @@ from fastapi.responses import StreamingResponsefrom pydantic import BaseModelfrom evals.utils import image_to_data_urlfrom evals.config import EVALS_DIR-from typing import Set+from typing import Optional, Setfrom evals.runner import run_image_evals, count_pending_eval_tasks+from evals import sessions as eval_sessions+from evals import sets as eval_setsfrom typing import List, Dictfrom llm import Llmfrom prompts.prompt_types import Stack@@ -107,6 +109,47 @@ class RunEvalsRequest(BaseModel):stack: Stackfiles: List[str] = [] # Optional list of specific file paths to run evals ondiff_mode: bool = False+ # When set, inputs come from {EVALS_DIR}/sets/{set_name}/inputs and runs+ # attach to the active eval session (auto-created when none exists).+ set_name: Optional[str] = None+++def _resolve_set_run(+ request: RunEvalsRequest,+) -> tuple[Optional[str], Optional[eval_sessions.EvalSession], Dict[str, set[str]]]:+ """Validate the requested set and resolve the session + per-model skips."""+ if not request.set_name:+ return None, None, {}+ try:+ set_info = eval_sets.get_set(request.set_name)+ except eval_sets.InvalidSetNameError as e:+ raise HTTPException(status_code=400, detail=str(e))+ except eval_sets.EvalSetNotFoundError:+ raise HTTPException(+ status_code=404, detail=f"Eval set not found: {request.set_name}"+ )+ if set_info.image_count == 0:+ raise HTTPException(+ status_code=400, detail=f"Eval set {request.set_name!r} has no images"+ )+ try:+ session = eval_sessions.resolve_session_for_run(request.set_name)+ except eval_sessions.SessionSetMismatchError as e:+ raise HTTPException(+ status_code=400,+ detail=(+ f"Active session {e.active_session.name!r} is pinned to set "+ f"{e.active_session.eval_set!r}. Start a new session for set "+ f"{request.set_name!r} first (POST /eval-sessions)."+ ),+ )+ skip_per_model: Dict[str, set[str]] = {}+ if request.diff_mode:+ for model in request.models:+ skip_per_model[model] = eval_sessions.completed_eval_inputs(+ session.session_id, model, str(request.stack)+ )+ return request.set_name, session, skip_per_modelclass OpenAIInputCompareRequest(BaseModel):@@ -182,6 +225,7 @@ async def compare_openai_inputs_for_evals(@router.post("/run_evals", response_model=List[str])async def run_evals(request: RunEvalsRequest) -> List[str]:"""Run evaluations on selected images in the inputs directory for multiple models"""+ eval_set, session, skip_per_model = _resolve_set_run(request)all_output_files: List[str] = []for model in request.models:@@ -190,16 +234,22 @@ async def run_evals(request: RunEvalsRequest) -> List[str]:stack=request.stack,input_files=request.files,diff_mode=request.diff_mode,+ eval_set=eval_set,+ eval_session_id=session.session_id if session else None,+ skip_input_files=skip_per_model.get(model),)all_output_files.extend(output_files)return all_output_files-def _count_eval_files(selected_files: List[str]) -> int:+def _count_eval_files(selected_files: List[str], set_name: Optional[str] = None) -> int:if selected_files:return len([f for f in selected_files if f.endswith(".png")])+ if set_name:+ return len(eval_sets.list_set_images(set_name))+input_dir = os.path.join(EVALS_DIR, "inputs")return len([f for f in os.listdir(input_dir) if f.endswith(".png")])@@ -210,6 +260,8 @@ async def run_evals_stream(request: RunEvalsRequest):if not request.models:raise HTTPException(status_code=400, detail="At least one model is required")+ eval_set, session, skip_per_model = _resolve_set_run(request)+per_model_task_counts: Dict[str, int] = {}per_model_skipped_existing: Dict[str, int] = {}if request.diff_mode:@@ -220,11 +272,13 @@ async def run_evals_stream(request: RunEvalsRequest):input_files=request.files,n=N,diff_mode=True,+ eval_set=eval_set,+ skip_input_files=skip_per_model.get(model),)per_model_task_counts[model] = pending_tasksper_model_skipped_existing[model] = skipped_taskselse:- per_model_task_count = _count_eval_files(request.files)+ per_model_task_count = _count_eval_files(request.files, eval_set)for model in request.models:per_model_task_counts[model] = per_model_task_countper_model_skipped_existing[model] = 0@@ -252,6 +306,9 @@ async def run_evals_stream(request: RunEvalsRequest):"completed_tasks": 0,"diff_mode": request.diff_mode,"total_skipped_existing": total_skipped_existing,+ "eval_set": eval_set,+ "eval_session_id": session.session_id if session else None,+ "eval_session_name": session.name if session else None,})@@ -288,6 +345,9 @@ async def run_evals_stream(request: RunEvalsRequest):input_files=request.files,diff_mode=request.diff_mode,progress_callback=on_progress,+ eval_set=eval_set,+ eval_session_id=session.session_id if session else None,+ skip_input_files=skip_per_model.get(model),)all_output_files.extend(output_files)completed_offset += model_task_countdiff --git a/backend/tests/test_agent_runs.py b/backend/tests/test_agent_runs.pyindex 13284ba2..6ca0d11c 100644--- a/backend/tests/test_agent_runs.py+++ b/backend/tests/test_agent_runs.py@@ -6,8 +6,8 @@ from typing import Any, Optionalimport pytestfrom agent.providers.base import StreamEvent-from agent.providers.pricing import MODEL_PRICING-from agent.providers.token_usage import TokenUsage+from costs.pricing import MODEL_PRICING+from costs.token_usage import TokenUsagefrom agent.tools.types import ToolCall, ToolExecutionResultfrom fs_logging.agent_runs import (RUN_ID_PATTERN,diff --git a/backend/tests/test_eval_sessions.py b/backend/tests/test_eval_sessions.pynew file mode 100644index 00000000..1faad2b0--- /dev/null+++ b/backend/tests/test_eval_sessions.py@@ -0,0 +1,467 @@+import base64+import sqlite3+from datetime import datetime, timedelta+from pathlib import Path+from typing import Any, Optional++import pytest+from fastapi import HTTPException++import evals.config+from evals.sessions import (+ SessionSetMismatchError,+ activate_session,+ create_session,+ get_active_session,+ list_sessions,+ resolve_session_for_run,+ completed_eval_inputs,+)+from fs_logging.agent_runs import (+ AgentRunRecorder,+ get_agent_runs_db_path,+ get_agent_runs_directory,+ open_index_db,+)+from llm import Llm+++def logs_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ monkeypatch.setenv("LOGS_PATH", str(tmp_path))+ return tmp_path+++def evals_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ monkeypatch.setattr(evals.config, "EVALS_DIR", str(tmp_path / "evals_data"))+ return tmp_path / "evals_data"+++def _make_set(evals_dir: Path, name: str, images: dict[str, bytes]) -> None:+ inputs = evals_dir / "sets" / name / "inputs"+ inputs.mkdir(parents=True)+ for filename, content in images.items():+ (inputs / filename).write_bytes(content)+++# The runs DDL exactly as first shipped, before the eval columns existed.+_LEGACY_RUNS_DDL = """+CREATE TABLE runs (+ run_id TEXT PRIMARY KEY,+ generation_id TEXT NOT NULL,+ variant_index INTEGER NOT NULL,+ entry_point TEXT NOT NULL,+ provider TEXT,+ model TEXT,+ api_model_name TEXT,+ stack TEXT,+ input_mode TEXT,+ generation_type TEXT,+ status TEXT NOT NULL DEFAULT 'running',+ error TEXT,+ created_at TEXT NOT NULL,+ completed_at TEXT,+ total_duration_ms INTEGER,+ llm_time_ms INTEGER,+ tool_time_ms INTEGER,+ num_llm_calls INTEGER NOT NULL DEFAULT 0,+ num_tool_calls INTEGER NOT NULL DEFAULT 0,+ input_tokens INTEGER,+ output_tokens INTEGER,+ cache_read_tokens INTEGER,+ cache_write_tokens INTEGER,+ total_tokens INTEGER,+ total_cost_usd REAL,+ has_unpriced_calls INTEGER NOT NULL DEFAULT 0,+ run_dir TEXT NOT NULL,+ size_bytes INTEGER+)+"""+++def test_migration_upgrades_legacy_db(logs_path: Path) -> None:+ db_dir = Path(get_agent_runs_directory())+ db_dir.mkdir(parents=True)+ conn = sqlite3.connect(get_agent_runs_db_path())+ conn.execute(_LEGACY_RUNS_DDL)+ conn.execute(+ "INSERT INTO runs (run_id, generation_id, variant_index, entry_point, "+ "status, created_at, run_dir) VALUES ('run_legacy', 'gen_x', 0, "+ "'websocket', 'completed', '2026-07-01T10:00:00', '/tmp/x')"+ )+ conn.commit()+ conn.close()++ migrated = open_index_db()+ try:+ columns = {row[1] for row in migrated.execute("PRAGMA table_info(runs)")}+ assert {+ "eval_session",+ "eval_set",+ "input_file",+ "input_image_sha256",+ } <= columns+ row = migrated.execute(+ "SELECT status, eval_session, eval_set, input_file FROM runs "+ "WHERE run_id = 'run_legacy'"+ ).fetchone()+ assert row == ("completed", None, None, None)+ tables = {+ r[0]+ for r in migrated.execute(+ "SELECT name FROM sqlite_master WHERE type = 'table'"+ )+ }+ assert "eval_sessions" in tables+ finally:+ migrated.close()++ # Idempotent on reopen.+ open_index_db().close()+++def _active_count() -> int:+ conn = sqlite3.connect(get_agent_runs_db_path())+ try:+ return conn.execute(+ "SELECT COUNT(*) FROM eval_sessions WHERE is_active = 1"+ ).fetchone()[0]+ finally:+ conn.close()+++def test_create_and_activate_single_active_invariant(logs_path: Path) -> None:+ first = create_session("set-a", "first")+ assert first.is_active+ second = create_session("set-b")+ assert second.name.endswith("session") # auto-named+ assert _active_count() == 1+ active = get_active_session()+ assert active is not None and active.session_id == second.session_id++ activate_session(first.session_id)+ assert _active_count() == 1+ active = get_active_session()+ assert active is not None and active.session_id == first.session_id++ sessions = list_sessions()+ assert {s.session_id for s in sessions} == {+ first.session_id,+ second.session_id,+ }+++def test_resolve_session_for_run(logs_path: Path) -> None:+ # No session: auto-creates and activates.+ session = resolve_session_for_run("set-a")+ assert session.is_active and session.eval_set == "set-a"+ # Same set: reuses.+ again = resolve_session_for_run("set-a")+ assert again.session_id == session.session_id+ # Different set: refuses.+ with pytest.raises(SessionSetMismatchError):+ resolve_session_for_run("set-b")+++def _record_run(+ *,+ model: Llm = Llm.GPT_5_5_HIGH,+ entry_point: str = "eval",+ eval_session: Optional[str] = None,+ eval_set: Optional[str] = None,+ input_file: Optional[str] = None,+ image_bytes: Optional[bytes] = None,+ stack: str = "html_tailwind",+) -> AgentRunRecorder:+ recorder = AgentRunRecorder(+ generation_id="gen_t",+ variant_index=0,+ entry_point=entry_point,+ stack=stack,+ enabled=True,+ eval_session=eval_session,+ eval_set=eval_set,+ input_file=input_file,+ )+ prompt: list[Any] = []+ if image_bytes is not None:+ data_url = "data:image/png;base64," + base64.b64encode(+ image_bytes+ ).decode("ascii")+ prompt = [+ {+ "role": "user",+ "content": [+ {"type": "image_url", "image_url": {"url": data_url}}+ ],+ }+ ]+ recorder.record_run_start(model, prompt)+ return recorder+++def _set_run_status(+ run_id: str, status: str, created_at: Optional[str] = None+) -> None:+ conn = sqlite3.connect(get_agent_runs_db_path())+ try:+ conn.execute(+ "UPDATE runs SET status = ? WHERE run_id = ?", (status, run_id)+ )+ if created_at is not None:+ conn.execute(+ "UPDATE runs SET created_at = ? WHERE run_id = ?",+ (created_at, run_id),+ )+ conn.commit()+ finally:+ conn.close()+++def test_recorder_eval_columns_and_websocket_sha(logs_path: Path) -> None:+ eval_recorder = _record_run(+ eval_session="sess_x",+ eval_set="set-a",+ input_file="shop.png",+ image_bytes=b"shop-image",+ )+ ui_recorder = _record_run(entry_point="websocket", image_bytes=b"ui-image")+ garbage_recorder = AgentRunRecorder(+ generation_id="g", variant_index=0, entry_point="websocket", enabled=True+ )+ garbage_recorder.record_run_start(Llm.GPT_5_5_HIGH, {"not": "a list"})++ conn = sqlite3.connect(get_agent_runs_db_path())+ try:+ rows = {+ row[0]: row+ for row in conn.execute(+ "SELECT run_id, eval_session, eval_set, input_file, "+ "input_image_sha256 FROM runs"+ )+ }+ finally:+ conn.close()++ import hashlib++ eval_row = rows[eval_recorder.run_id]+ assert eval_row[1:4] == ("sess_x", "set-a", "shop.png")+ assert eval_row[4] == hashlib.sha256(b"shop-image").hexdigest()++ ui_row = rows[ui_recorder.run_id]+ assert ui_row[1:4] == (None, None, None)+ assert ui_row[4] == hashlib.sha256(b"ui-image").hexdigest()++ assert rows[garbage_recorder.run_id][4] is None+++def test_completed_eval_inputs(logs_path: Path) -> None:+ session = create_session("set-a")+ done = _record_run(+ eval_session=session.session_id, eval_set="set-a", input_file="a.png"+ )+ _set_run_status(done.run_id, "completed")+ failed = _record_run(+ eval_session=session.session_id, eval_set="set-a", input_file="b.png"+ )+ _set_run_status(failed.run_id, "failed")+ _record_run(+ eval_session=session.session_id,+ eval_set="set-a",+ input_file="c.png",+ model=Llm.CLAUDE_OPUS_4_8_HIGH,+ )++ completed = completed_eval_inputs(+ session.session_id, Llm.GPT_5_5_HIGH.value, "html_tailwind"+ )+ assert completed == {"a.png"}+++async def test_session_matrix(logs_path: Path, evals_dir: Path) -> None:+ from routes.eval_sets import get_session_matrix++ _make_set(+ evals_dir, "set-a", {"one.png": b"image-one", "two.png": b"image-two"}+ )+ session = create_session("set-a")++ completed = _record_run(+ eval_session=session.session_id, eval_set="set-a", input_file="one.png"+ )+ _set_run_status(completed.run_id, "completed")+ failed = _record_run(+ eval_session=session.session_id,+ eval_set="set-a",+ input_file="one.png",+ model=Llm.CLAUDE_OPUS_4_8_HIGH,+ )+ _set_run_status(failed.run_id, "failed")+ running = _record_run(+ eval_session=session.session_id, eval_set="set-a", input_file="two.png"+ )+ stale_time = (datetime.now() - timedelta(hours=3)).isoformat(+ timespec="seconds"+ )+ stale = _record_run(+ eval_session=session.session_id,+ eval_set="set-a",+ input_file="two.png",+ model=Llm.CLAUDE_OPUS_4_8_HIGH,+ )+ _set_run_status(stale.run_id, "running", created_at=stale_time)+ orphan = _record_run(+ eval_session=session.session_id, eval_set="set-a", input_file="gone.png"+ )+ _set_run_status(orphan.run_id, "completed")++ # UI run after session start with a matching image hash: included.+ ui_match = _record_run(entry_point="websocket", image_bytes=b"image-one")+ _set_run_status(ui_match.run_id, "completed")+ # UI run BEFORE session start: excluded.+ ui_before = _record_run(entry_point="websocket", image_bytes=b"image-one")+ _set_run_status(+ ui_before.run_id, "completed", created_at="2020-01-01T00:00:00"+ )++ matrix = await get_session_matrix(session.session_id)+ assert not matrix.set_missing+ assert [row.filename for row in matrix.rows] == ["one.png", "two.png"]+ assert set(matrix.models) == {+ Llm.GPT_5_5_HIGH.value,+ Llm.CLAUDE_OPUS_4_8_HIGH.value,+ }+ assert matrix.unmatched_run_count == 1++ cells = {(c.filename, c.model): c for c in matrix.cells}+ one_gpt = cells[("one.png", Llm.GPT_5_5_HIGH.value)]+ assert {r.run_id for r in one_gpt.runs} == {+ completed.run_id,+ ui_match.run_id,+ }+ sources = {r.run_id: r.source for r in one_gpt.runs}+ assert sources[ui_match.run_id] == "ui"+ assert sources[completed.run_id] == "eval"+ assert ui_before.run_id not in sources++ one_claude = cells[("one.png", Llm.CLAUDE_OPUS_4_8_HIGH.value)]+ assert one_claude.runs[0].status == "failed"++ two_gpt = cells[("two.png", Llm.GPT_5_5_HIGH.value)]+ assert two_gpt.runs[0].status == "running"+ assert two_gpt.runs[0].is_stale is False+ two_claude = cells[("two.png", Llm.CLAUDE_OPUS_4_8_HIGH.value)]+ assert two_claude.runs[0].is_stale is True+++async def test_session_matrix_missing_set(+ logs_path: Path, evals_dir: Path+) -> None:+ from routes.eval_sets import get_session_matrix++ session = create_session("never-existed")+ run = _record_run(+ eval_session=session.session_id,+ eval_set="never-existed",+ input_file="x.png",+ )+ _set_run_status(run.run_id, "completed")++ matrix = await get_session_matrix(session.session_id)+ assert matrix.set_missing is True+ assert matrix.rows == []+ # History still browsable through cells.+ assert any(c.filename == "x.png" for c in matrix.cells)+++async def test_run_evals_threads_set_and_session(+ logs_path: Path, evals_dir: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ import routes.evals as evals_routes+ from routes.evals import RunEvalsRequest, run_evals++ _make_set(evals_dir, "set-a", {"a.png": b"a", "b.png": b"b"})+ _make_set(evals_dir, "set-b", {"x.png": b"x"})++ captured: list[dict[str, Any]] = []++ async def fake_run_image_evals(**kwargs: Any) -> list[str]:+ captured.append(kwargs)+ return []++ monkeypatch.setattr(evals_routes, "run_image_evals", fake_run_image_evals)++ # Seed a completed combo so diff mode skips it.+ session = create_session("set-a")+ done = _record_run(+ eval_session=session.session_id, eval_set="set-a", input_file="a.png"+ )+ _set_run_status(done.run_id, "completed")++ await run_evals(+ RunEvalsRequest(+ models=[Llm.GPT_5_5_HIGH.value],+ stack="html_tailwind",+ set_name="set-a",+ diff_mode=True,+ )+ )+ assert len(captured) == 1+ kwargs = captured[0]+ assert kwargs["eval_set"] == "set-a"+ assert kwargs["eval_session_id"] == session.session_id+ assert kwargs["skip_input_files"] == {"a.png"}++ # Mismatched active session -> 400 with guidance.+ with pytest.raises(HTTPException) as excinfo:+ await run_evals(+ RunEvalsRequest(+ models=[Llm.GPT_5_5_HIGH.value],+ stack="html_tailwind",+ set_name="set-b",+ )+ )+ assert excinfo.value.status_code == 400+ assert "Start a new session" in str(excinfo.value.detail)++ # Unknown set -> 404.+ create_session("set-a") # keep active pinned to an existing set+ with pytest.raises(HTTPException) as excinfo:+ await run_evals(+ RunEvalsRequest(+ models=[Llm.GPT_5_5_HIGH.value],+ stack="html_tailwind",+ set_name="missing-set",+ )+ )+ assert excinfo.value.status_code == 404+++def test_count_pending_eval_tasks_with_set(+ logs_path: Path, evals_dir: Path+) -> None:+ from evals.runner import count_pending_eval_tasks++ _make_set(evals_dir, "set-a", {"a.png": b"a", "b.png": b"b", "c.png": b"c"})++ pending, skipped = count_pending_eval_tasks(+ stack="html_tailwind",+ model=Llm.GPT_5_5_HIGH.value,+ diff_mode=True,+ eval_set="set-a",+ skip_input_files={"a.png"},+ )+ assert (pending, skipped) == (2, 1)++ pending, skipped = count_pending_eval_tasks(+ stack="html_tailwind",+ model=Llm.GPT_5_5_HIGH.value,+ diff_mode=False,+ eval_set="set-a",+ )+ assert (pending, skipped) == (3, 0)diff --git a/backend/tests/test_eval_sets.py b/backend/tests/test_eval_sets.pynew file mode 100644index 00000000..17952abd--- /dev/null+++ b/backend/tests/test_eval_sets.py@@ -0,0 +1,123 @@+import hashlib+import json+import os+import time+from pathlib import Path++import pytest+from fastapi import HTTPException++import evals.config+from evals.sets import (+ EvalSetNotFoundError,+ InvalidSetNameError,+ get_set,+ get_set_inputs_dir,+ list_set_images,+ list_sets,+ resolve_set_image_path,+)+from routes.eval_sets import get_eval_set_image+++def evals_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ monkeypatch.setattr(evals.config, "EVALS_DIR", str(tmp_path))+ return tmp_path+++def _make_set(evals_dir: Path, name: str, images: dict[str, bytes]) -> Path:+ inputs = evals_dir / "sets" / name / "inputs"+ inputs.mkdir(parents=True)+ for filename, content in images.items():+ (inputs / filename).write_bytes(content)+ return inputs+++def test_list_set_images_bootstraps_manifest_with_hashes(evals_dir: Path) -> None:+ _make_set(evals_dir, "jun-21-evals", {"a.png": b"aaa", "b.png": b"bbb"})++ images = list_set_images("jun-21-evals")+ assert [i.filename for i in images] == ["a.png", "b.png"]+ assert images[0].sha256 == hashlib.sha256(b"aaa").hexdigest()++ manifest_path = evals_dir / "sets" / "jun-21-evals" / "manifest.json"+ manifest = json.loads(manifest_path.read_text())+ assert manifest["display_name"] == "jun-21-evals"+ assert manifest["images"]["b.png"]["sha256"] == (+ hashlib.sha256(b"bbb").hexdigest()+ )+++def test_hash_cache_reused_and_invalidated(evals_dir: Path) -> None:+ inputs = _make_set(evals_dir, "s1", {"a.png": b"original"})+ list_set_images("s1")++ # Tamper with the cached sha; with unchanged size+mtime it must be+ # served from cache (i.e. NOT recomputed).+ manifest_path = evals_dir / "sets" / "s1" / "manifest.json"+ manifest = json.loads(manifest_path.read_text())+ manifest["images"]["a.png"]["sha256"] = "cached-sentinel"+ manifest_path.write_text(json.dumps(manifest))+ assert list_set_images("s1")[0].sha256 == "cached-sentinel"++ # Rewriting the file (new mtime) invalidates the cache entry.+ image_path = inputs / "a.png"+ image_path.write_bytes(b"rewritten")+ os.utime(image_path, (time.time() + 5, time.time() + 5))+ images = list_set_images("s1")+ assert images[0].sha256 == hashlib.sha256(b"rewritten").hexdigest()+ manifest = json.loads(manifest_path.read_text())+ assert manifest["images"]["a.png"]["sha256"] == images[0].sha256+++def test_deleted_files_pruned_from_manifest(evals_dir: Path) -> None:+ inputs = _make_set(evals_dir, "s1", {"a.png": b"a", "b.png": b"b"})+ list_set_images("s1")+ (inputs / "b.png").unlink()++ images = list_set_images("s1")+ assert [i.filename for i in images] == ["a.png"]+ manifest = json.loads(+ (evals_dir / "sets" / "s1" / "manifest.json").read_text()+ )+ assert "b.png" not in manifest["images"]+++def test_list_sets_and_get_set(evals_dir: Path) -> None:+ _make_set(evals_dir, "alpha", {"a.png": b"a"})+ _make_set(evals_dir, "beta", {"b.png": b"b", "c.png": b"c"})++ sets = list_sets()+ assert [(s.name, s.image_count) for s in sets] == [("alpha", 1), ("beta", 2)]+ assert get_set("beta").image_count == 2+ with pytest.raises(EvalSetNotFoundError):+ get_set("missing")+++def test_set_name_validation(evals_dir: Path) -> None:+ for bad_name in ("../etc", "/abs", "", ".hidden"):+ with pytest.raises(InvalidSetNameError):+ get_set_inputs_dir(bad_name)+++async def test_image_path_resolution_and_route_guards(evals_dir: Path) -> None:+ _make_set(evals_dir, "s1", {"a.png": b"png-bytes"})++ assert resolve_set_image_path("s1", "a.png").endswith("a.png")+ # Traversal components are stripped to a basename.+ assert resolve_set_image_path("s1", "../a.png").endswith("a.png")+ with pytest.raises(InvalidSetNameError):+ resolve_set_image_path("s1", "notes.txt")+ with pytest.raises(FileNotFoundError):+ resolve_set_image_path("s1", "missing.png")++ response = await get_eval_set_image("s1", "a.png")+ assert response.path == str(evals_dir / "sets" / "s1" / "inputs" / "a.png")+ with pytest.raises(HTTPException) as excinfo:+ await get_eval_set_image("s1", "missing.png")+ assert excinfo.value.status_code == 404+ with pytest.raises(HTTPException) as excinfo:+ await get_eval_set_image("nope", "a.png")+ assert excinfo.value.status_code == 404diff --git a/backend/tests/test_prompt_reports.py b/backend/tests/test_prompt_reports.pyindex fe23efb5..1108fcc8 100644--- a/backend/tests/test_prompt_reports.py+++ b/backend/tests/test_prompt_reports.py@@ -7,7 +7,7 @@ from pathlib import Pathimport pytestfrom fastapi import HTTPException-from agent.providers.token_usage import TokenUsage+from costs.token_usage import TokenUsagefrom fs_logging.prompt_reports import (PROMPT_REPORT_FILENAME_PATTERN,PromptReportLogger,diff --git a/backend/tests/test_token_usage.py b/backend/tests/test_token_usage.pyindex c25bd181..136a0808 100644--- a/backend/tests/test_token_usage.py+++ b/backend/tests/test_token_usage.py@@ -2,8 +2,8 @@from types import SimpleNamespace-from agent.providers.pricing import MODEL_PRICING, ModelPricing-from agent.providers.token_usage import TokenUsage+from costs.pricing import MODEL_PRICING, ModelPricing+from costs.token_usage import TokenUsagefrom agent.providers.anthropic import _extract_anthropic_usagefrom agent.providers.gemini import _extract_usage as _extract_gemini_usagefrom agent.providers.openai import _extract_openai_usagediff --git a/frontend/src/components/evals/AgentRunsPage.tsx b/frontend/src/components/evals/AgentRunsPage.tsxindex d035cfcd..c306f026 100644--- a/frontend/src/components/evals/AgentRunsPage.tsx+++ b/frontend/src/components/evals/AgentRunsPage.tsx@@ -1,6 +1,7 @@import { useCallback, useEffect, useMemo, useState } from "react";import toast from "react-hot-toast";import { BsBoxArrowUpRight } from "react-icons/bs";+import { useSearchParams } from "react-router-dom";import { HTTP_BACKEND_URL } from "../../config";import EvalNavigation from "./EvalNavigation";@@ -47,6 +48,10 @@ interface AgentRunSummary {total_cost_usd: number | null;has_unpriced_calls: boolean;size_bytes: number | null;+ eval_session: string | null;+ eval_set: string | null;+ input_file: string | null;+ input_image_sha256: string | null;}interface AgentRunListResponse {@@ -148,6 +153,7 @@ function SummaryStats({ run }: { run: AgentRunSummary }) {}function AgentRunsPage() {+ const [searchParams, setSearchParams] = useSearchParams();const [runs, setRuns] = useState<AgentRunSummary[]>([]);const [totalSizeBytes, setTotalSizeBytes] = useState(0);const [runsDirectory, setRunsDirectory] = useState("");@@ -192,6 +198,7 @@ function AgentRunsPage() {setActiveTab("timeline");setIncludeDeltas(withDeltas);setExpandSignal({ mode: "best", version: 0 });+ setSearchParams({ run: runId }, { replace: true });setIsLoadingDetail(true);try {const response = await fetch(@@ -212,16 +219,22 @@ function AgentRunsPage() {setIsLoadingDetail(false);}},- []+ [setSearchParams]);useEffect(() => {fetchRuns().then((loadedRuns) => {- if (loadedRuns.length > 0) {+ // Deep link (?run=...) wins over the newest-run default so matrix+ // cells and shared URLs land on the intended run.+ const requestedRun = searchParams.get("run");+ if (requestedRun) {+ openRun(requestedRun);+ } else if (loadedRuns.length > 0) {const groups = groupByGeneration(loadedRuns);openRun(groups[0].runs[0].run_id);}});+ // eslint-disable-next-line react-hooks/exhaustive-deps}, [fetchRuns, openRun]);const handlePrune = async () => {diff --git a/frontend/src/components/evals/EvalNavigation.tsx b/frontend/src/components/evals/EvalNavigation.tsxindex 635f2b5e..3f7c00f0 100644--- a/frontend/src/components/evals/EvalNavigation.tsx+++ b/frontend/src/components/evals/EvalNavigation.tsx@@ -47,6 +47,13 @@ function EvalNavigation() {>Agent Runs</Link>++ <Link+ to="/evals/sessions"+ className="hover:text-blue-300 transition-colors"+ >+ Sessions+ </Link></div><Linkdiff --git a/frontend/src/components/evals/EvalSessionsPage.tsx b/frontend/src/components/evals/EvalSessionsPage.tsxnew file mode 100644index 00000000..9fc78283--- /dev/null+++ b/frontend/src/components/evals/EvalSessionsPage.tsx@@ -0,0 +1,513 @@+import { useCallback, useEffect, useMemo, useState } from "react";+import toast from "react-hot-toast";+import { useNavigate } from "react-router-dom";+import { BsBoxArrowUpRight } from "react-icons/bs";++import { HTTP_BACKEND_URL } from "../../config";+import EvalNavigation from "./EvalNavigation";+import { formatCost, formatMs, formatTimestamp } from "./report-format";++interface EvalSetSummary {+ name: string;+ display_name: string;+ created_at: string | null;+ notes: string;+ image_count: number;+}++interface EvalSession {+ session_id: string;+ name: string;+ eval_set: string;+ created_at: string;+ is_active: boolean;+}++interface MatrixRun {+ run_id: string;+ status: string;+ source: "eval" | "ui";+ total_duration_ms: number | null;+ total_cost_usd: number | null;+ created_at: string;+ is_stale: boolean;+}++interface MatrixCell {+ filename: string;+ model: string;+ runs: MatrixRun[];+}++interface MatrixRow {+ filename: string;+ sha256: string;+ image_url: string;+}++interface SessionMatrix {+ session: EvalSession;+ eval_set: EvalSetSummary | null;+ set_missing: boolean;+ rows: MatrixRow[];+ models: string[];+ cells: MatrixCell[];+ unmatched_run_count: number;+}++function statusDotClass(status: string, isStale: boolean): string {+ if (status === "completed") return "bg-emerald-400";+ if (status === "failed") return "bg-red-400";+ return isStale ? "bg-zinc-500" : "bg-amber-400 animate-pulse";+}++function CellFace({+ runs,+ onOpenRun,+}: {+ runs: MatrixRun[];+ onOpenRun: (runId: string) => void;+}) {+ const [expanded, setExpanded] = useState(false);+ const newest = runs[0];+ if (!newest) return null;++ return (+ <div className="flex flex-col gap-1">+ <div+ onClick={() => onOpenRun(newest.run_id)}+ className={`group flex cursor-pointer items-center gap-1.5 rounded-md border px-1.5 py-1 transition-colors hover:bg-zinc-800 ${+ newest.status === "failed"+ ? "border-red-900 bg-red-950/30"+ : "border-zinc-700 bg-zinc-950/60"+ } ${newest.is_stale ? "opacity-50" : ""}`}+ title={`${newest.status} · ${formatTimestamp(newest.created_at)}`}+ >+ <span+ className={`h-2 w-2 shrink-0 rounded-full ${statusDotClass(+ newest.status,+ newest.is_stale+ )}`}+ />+ <span className="font-mono text-[11px] text-zinc-300">+ {formatMs(newest.total_duration_ms)}+ </span>+ <span className="font-mono text-[11px] text-emerald-400">+ {formatCost(newest.total_cost_usd)}+ </span>+ {newest.source === "ui" && (+ <span className="rounded bg-amber-900/60 px-1 text-[10px] font-medium text-amber-200">+ UI+ </span>+ )}+ {runs.length > 1 && (+ <button+ onClick={(e) => {+ e.stopPropagation();+ setExpanded(!expanded);+ }}+ className="rounded bg-zinc-800 px-1 text-[10px] text-zinc-400 hover:text-zinc-100"+ title="Show all attempts"+ >+ ×{runs.length}+ </button>+ )}+ <a+ href={`${HTTP_BACKEND_URL}/agent-runs/${encodeURIComponent(+ newest.run_id+ )}/output`}+ target="_blank"+ rel="noreferrer"+ onClick={(e) => e.stopPropagation()}+ title="Open captured HTML"+ className="ml-auto shrink-0 rounded p-0.5 text-zinc-600 opacity-0 transition-opacity hover:text-zinc-200 group-hover:opacity-100"+ >+ <BsBoxArrowUpRight className="text-[10px]" />+ </a>+ </div>+ {expanded &&+ runs.slice(1).map((run) => (+ <div+ key={run.run_id}+ onClick={() => onOpenRun(run.run_id)}+ className="flex cursor-pointer items-center gap-1.5 rounded-md border border-zinc-800 px-1.5 py-0.5 opacity-70 hover:bg-zinc-800"+ >+ <span+ className={`h-1.5 w-1.5 shrink-0 rounded-full ${statusDotClass(+ run.status,+ run.is_stale+ )}`}+ />+ <span className="font-mono text-[10px] text-zinc-400">+ {formatMs(run.total_duration_ms)} ·{" "}+ {formatCost(run.total_cost_usd)}+ {run.source === "ui" ? " · UI" : ""}+ </span>+ </div>+ ))}+ </div>+ );+}++function EvalSessionsPage() {+ const navigate = useNavigate();+ const [sessions, setSessions] = useState<EvalSession[]>([]);+ const [sets, setSets] = useState<EvalSetSummary[]>([]);+ const [selectedSessionId, setSelectedSessionId] = useState<string | null>(+ null+ );+ const [matrix, setMatrix] = useState<SessionMatrix | null>(null);+ const [isLoadingMatrix, setIsLoadingMatrix] = useState(false);+ const [newSessionSet, setNewSessionSet] = useState("");+ const [newSessionName, setNewSessionName] = useState("");+ const [isCreating, setIsCreating] = useState(false);++ const cellsByKey = useMemo(() => {+ const map = new Map<string, MatrixCell>();+ for (const cell of matrix?.cells ?? []) {+ map.set(`${cell.filename}|${cell.model}`, cell);+ }+ return map;+ }, [matrix]);++ // Images with outputs float to the top: most models covered first, then+ // most total runs, then filename for a stable order.+ const sortedRows = useMemo(() => {+ if (!matrix) return [];+ const coverage = new Map<string, { models: number; runs: number }>();+ for (const cell of matrix.cells) {+ const entry = coverage.get(cell.filename) ?? { models: 0, runs: 0 };+ if (cell.runs.length > 0) entry.models += 1;+ entry.runs += cell.runs.length;+ coverage.set(cell.filename, entry);+ }+ return [...matrix.rows].sort((a, b) => {+ const ca = coverage.get(a.filename) ?? { models: 0, runs: 0 };+ const cb = coverage.get(b.filename) ?? { models: 0, runs: 0 };+ if (cb.models !== ca.models) return cb.models - ca.models;+ if (cb.runs !== ca.runs) return cb.runs - ca.runs;+ return a.filename.localeCompare(b.filename);+ });+ }, [matrix]);++ const fetchSessions = useCallback(async (): Promise<EvalSession[]> => {+ try {+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`);+ const data = await response.json();+ setSessions(data.sessions);+ return data.sessions;+ } catch (error) {+ console.error("Error fetching sessions", error);+ toast.error("Failed to load sessions.");+ return [];+ }+ }, []);++ const fetchSets = useCallback(async () => {+ try {+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sets`);+ const data: EvalSetSummary[] = await response.json();+ setSets(data);+ if (data.length > 0) {+ setNewSessionSet((current) => current || data[0].name);+ }+ } catch (error) {+ console.error("Error fetching eval sets", error);+ }+ }, []);++ const loadMatrix = useCallback(async (sessionId: string) => {+ setSelectedSessionId(sessionId);+ setIsLoadingMatrix(true);+ try {+ const response = await fetch(+ `${HTTP_BACKEND_URL}/eval-sessions/${encodeURIComponent(+ sessionId+ )}/matrix`+ );+ if (!response.ok) throw new Error("Request failed");+ setMatrix(await response.json());+ } catch (error) {+ console.error("Error loading matrix", error);+ setMatrix(null);+ toast.error("Failed to load session matrix.");+ } finally {+ setIsLoadingMatrix(false);+ }+ }, []);++ useEffect(() => {+ fetchSets();+ fetchSessions().then((loaded) => {+ const active = loaded.find((s) => s.is_active) ?? loaded[0];+ if (active) loadMatrix(active.session_id);+ });+ }, [fetchSets, fetchSessions, loadMatrix]);++ // Live refresh while any cell is actively running.+ useEffect(() => {+ if (!matrix || !selectedSessionId) return;+ const hasLiveRuns = matrix.cells.some((cell) =>+ cell.runs.some((run) => run.status === "running" && !run.is_stale)+ );+ if (!hasLiveRuns) return;+ const interval = window.setInterval(+ () => loadMatrix(selectedSessionId),+ 10_000+ );+ return () => window.clearInterval(interval);+ }, [matrix, selectedSessionId, loadMatrix]);++ const handleActivate = async (sessionId: string) => {+ try {+ const response = await fetch(+ `${HTTP_BACKEND_URL}/eval-sessions/${encodeURIComponent(+ sessionId+ )}/activate`,+ { method: "POST" }+ );+ if (!response.ok) throw new Error("Request failed");+ toast.success("Session activated.");+ fetchSessions();+ } catch (error) {+ console.error("Error activating session", error);+ toast.error("Failed to activate session.");+ }+ };++ const handleCreate = async () => {+ if (!newSessionSet) return;+ setIsCreating(true);+ try {+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ eval_set: newSessionSet,+ name: newSessionName.trim() || null,+ }),+ });+ if (!response.ok) {+ const data = await response.json();+ throw new Error(data.detail || "Request failed");+ }+ const created: EvalSession = await response.json();+ setNewSessionName("");+ toast.success(`Session "${created.name}" started.`);+ await fetchSessions();+ loadMatrix(created.session_id);+ } catch (error) {+ console.error("Error creating session", error);+ toast.error(String(error));+ } finally {+ setIsCreating(false);+ }+ };++ const openRun = (runId: string) => {+ navigate(`/evals/agent-runs?run=${encodeURIComponent(runId)}`);+ };++ return (+ <div className="flex h-screen flex-col overflow-hidden bg-zinc-950 text-white">+ <div className="shrink-0">+ <EvalNavigation />+ </div>+ <div className="mx-auto flex min-h-0 w-full max-w-[1600px] flex-1 flex-col gap-2 px-4 py-2">+ <div className="flex shrink-0 flex-wrap items-center justify-between gap-2 rounded-xl border border-zinc-800 bg-zinc-900/80 px-4 py-2">+ <h1 className="text-base font-semibold tracking-tight">+ Eval Sessions+ </h1>+ <div className="flex items-center gap-2 text-xs">+ <select+ value={newSessionSet}+ onChange={(e) => setNewSessionSet(e.target.value)}+ className="rounded-md border border-zinc-700 bg-zinc-950 px-2 py-1 text-zinc-200"+ >+ {sets.length === 0 && <option value="">No sets found</option>}+ {sets.map((set) => (+ <option key={set.name} value={set.name}>+ {set.display_name} ({set.image_count})+ </option>+ ))}+ </select>+ <input+ value={newSessionName}+ onChange={(e) => setNewSessionName(e.target.value)}+ placeholder="Session name (optional)"+ className="w-44 rounded-md border border-zinc-700 bg-zinc-950 px-2 py-1 text-zinc-200 placeholder:text-zinc-600"+ />+ <button+ onClick={handleCreate}+ disabled={isCreating || !newSessionSet}+ className="rounded-md border border-emerald-800 bg-emerald-950/60 px-2.5 py-1 text-emerald-200 transition-colors hover:bg-emerald-900/60 disabled:opacity-50"+ >+ {isCreating ? "Starting…" : "New session"}+ </button>+ </div>+ </div>++ <div className="grid min-h-0 flex-1 gap-3 lg:grid-cols-[300px_minmax(0,1fr)]">+ <section className="min-h-0 overflow-y-auto rounded-xl border border-zinc-800 bg-zinc-900/60 p-2">+ <div className="px-2 py-1 text-xs uppercase tracking-wide text-zinc-500">+ {sessions.length} session{sessions.length === 1 ? "" : "s"}+ </div>+ {sessions.length === 0 && (+ <p className="px-2 py-4 text-sm text-zinc-400">+ No sessions yet. Pick a set above and start one, or just run+ evals with a set — a session is created automatically.+ </p>+ )}+ {sessions.map((session) => (+ <div+ key={session.session_id}+ onClick={() => loadMatrix(session.session_id)}+ className={`mb-1.5 cursor-pointer rounded-lg border px-2.5 py-2 transition-colors ${+ session.session_id === selectedSessionId+ ? "border-blue-600 bg-blue-950/50"+ : "border-zinc-800 bg-zinc-950/50 hover:bg-zinc-800/60"+ }`}+ >+ <div className="flex items-center gap-2">+ {session.is_active && (+ <span+ className="h-2 w-2 shrink-0 rounded-full bg-emerald-400"+ title="Active session"+ />+ )}+ <span className="min-w-0 flex-1 truncate text-xs text-zinc-100">+ {session.name}+ </span>+ {!session.is_active && (+ <button+ onClick={(e) => {+ e.stopPropagation();+ handleActivate(session.session_id);+ }}+ className="rounded border border-zinc-700 px-1.5 py-0.5 text-[10px] text-zinc-400 hover:bg-zinc-700 hover:text-zinc-100"+ >+ Activate+ </button>+ )}+ </div>+ <div className="mt-1 flex items-center justify-between gap-2 text-[11px] text-zinc-500">+ <span className="rounded-full border border-zinc-700 bg-zinc-800 px-1.5 py-0.5 text-zinc-300">+ {session.eval_set}+ </span>+ <span>{formatTimestamp(session.created_at)}</span>+ </div>+ </div>+ ))}+ </section>++ <section className="min-h-0 overflow-auto rounded-xl border border-zinc-800 bg-zinc-900/60 p-3">+ {!selectedSessionId && (+ <p className="py-8 text-center text-sm text-zinc-400">+ Select a session to see its matrix.+ </p>+ )}+ {isLoadingMatrix && (+ <p className="py-8 text-center text-sm text-zinc-400">+ Loading matrix…+ </p>+ )}+ {!isLoadingMatrix && matrix && (+ <div className="flex flex-col gap-2">+ {matrix.set_missing && (+ <div className="rounded-lg border border-amber-900 bg-amber-950/40 px-3 py-2 text-xs text-amber-200">+ The set "{matrix.session.eval_set}" no longer exists on+ disk — showing recorded history only.+ </div>+ )}+ {matrix.unmatched_run_count > 0 && (+ <div className="rounded-lg border border-zinc-700 bg-zinc-950/60 px-3 py-2 text-xs text-zinc-400">+ {matrix.unmatched_run_count} run+ {matrix.unmatched_run_count === 1 ? "" : "s"} reference+ images no longer in the set.+ </div>+ )}+ {matrix.rows.length === 0 && !matrix.set_missing && (+ <p className="py-8 text-center text-sm text-zinc-400">+ This set has no images. Drop PNGs into the set's inputs+ folder.+ </p>+ )}+ {matrix.models.length === 0 && matrix.rows.length > 0 && (+ <p className="py-2 text-center text-xs text-zinc-500">+ No runs in this session yet — run evals with this set and+ cells will appear here.+ </p>+ )}+ {matrix.rows.length > 0 && (+ <div className="overflow-x-auto">+ <table className="w-full border-separate border-spacing-1">+ <thead>+ <tr>+ <th className="sticky left-0 z-10 bg-zinc-900/95 px-2 py-1 text-left text-[11px] font-medium uppercase tracking-wide text-zinc-500">+ Image+ </th>+ {matrix.models.map((model) => (+ <th+ key={model}+ className="min-w-[140px] px-2 py-1 text-left font-mono text-[11px] font-normal text-zinc-400"+ title={model}+ >+ {model}+ </th>+ ))}+ </tr>+ </thead>+ <tbody>+ {sortedRows.map((row) => (+ <tr key={row.filename}>+ <td className="sticky left-0 z-10 bg-zinc-900/95 px-2 py-1">+ <div className="flex items-center gap-2">+ <img+ src={`${HTTP_BACKEND_URL}${row.image_url}`}+ alt={row.filename}+ className="h-12 w-16 rounded border border-zinc-800 bg-zinc-800 object-cover"+ loading="lazy"+ />+ <span className="max-w-[160px] truncate font-mono text-[11px] text-zinc-300">+ {row.filename}+ </span>+ </div>+ </td>+ {matrix.models.map((model) => {+ const cell = cellsByKey.get(+ `${row.filename}|${model}`+ );+ return (+ <td+ key={model}+ className="min-w-[140px] px-1 py-1 align-top"+ >+ {cell ? (+ <CellFace+ runs={cell.runs}+ onOpenRun={openRun}+ />+ ) : (+ <div className="rounded-md border border-dashed border-zinc-800 px-1.5 py-1.5 text-center text-[10px] text-zinc-700">+ —+ </div>+ )}+ </td>+ );+ })}+ </tr>+ ))}+ </tbody>+ </table>+ </div>+ )}+ </div>+ )}+ </section>+ </div>+ </div>+ </div>+ );+}++export default EvalSessionsPage;diff --git a/frontend/src/components/evals/RunEvalsPage.tsx b/frontend/src/components/evals/RunEvalsPage.tsxindex 41403d36..2923cc62 100644--- a/frontend/src/components/evals/RunEvalsPage.tsx+++ b/frontend/src/components/evals/RunEvalsPage.tsx@@ -37,6 +37,22 @@ interface FailedTask {error: string;}+interface EvalSetSummary {+ name: string;+ display_name: string;+ image_count: number;+}++interface EvalSetImage {+ filename: string;+}++interface ActiveEvalSession {+ session_id: string;+ name: string;+ eval_set: string;+}+function RunEvalsPage() {const faviconFlashIntervalRef = useRef<number | null>(null);const [isRunning, setIsRunning] = useState(false);@@ -46,6 +62,14 @@ function RunEvalsPage() {const [selectedStack, setSelectedStack] = useState<string>("html_tailwind");const [diffMode, setDiffMode] = useState(true);const [selectedFiles, setSelectedFiles] = useState<string[]>([]);+ const [inputSource, setInputSource] = useState<"legacy" | "set">("legacy");+ const [evalSets, setEvalSets] = useState<EvalSetSummary[]>([]);+ const [selectedSet, setSelectedSet] = useState("");+ const [setImages, setSetImages] = useState<EvalSetImage[]>([]);+ const [selectedSetFiles, setSelectedSetFiles] = useState<string[]>([]);+ const [activeSession, setActiveSession] = useState<ActiveEvalSession | null>(+ null+ );const [showPaths, setShowPaths] = useState<boolean>(false);const [completedTasks, setCompletedTasks] = useState(0);const [totalTasks, setTotalTasks] = useState(0);@@ -66,6 +90,57 @@ function RunEvalsPage() {fetchModels();}, []);+ useEffect(() => {+ const fetchEvalSets = async () => {+ try {+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sets`);+ const data: EvalSetSummary[] = await response.json();+ setEvalSets(data);+ if (data.length > 0) {+ setSelectedSet((current) => current || data[0].name);+ }+ } catch (error) {+ console.error("Error fetching eval sets", error);+ }+ };+ fetchEvalSets();+ }, []);++ const refreshActiveSession = useCallback(async () => {+ try {+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions/active`);+ setActiveSession(await response.json());+ } catch (error) {+ console.error("Error fetching active session", error);+ }+ }, []);++ useEffect(() => {+ if (inputSource === "set") refreshActiveSession();+ }, [inputSource, refreshActiveSession]);++ useEffect(() => {+ if (inputSource !== "set" || !selectedSet) return;+ const fetchSetImages = async () => {+ try {+ const response = await fetch(+ `${HTTP_BACKEND_URL}/eval-sets/${encodeURIComponent(selectedSet)}`+ );+ if (!response.ok) throw new Error("Request failed");+ const data = await response.json();+ setSetImages(data.images);+ setSelectedSetFiles(+ data.images.map((image: EvalSetImage) => image.filename)+ );+ } catch (error) {+ console.error("Error fetching set images", error);+ setSetImages([]);+ setSelectedSetFiles([]);+ }+ };+ fetchSetImages();+ }, [inputSource, selectedSet]);+useEffect(() => {return () => {document.title = "Screenshot to Code";@@ -134,7 +209,15 @@ function RunEvalsPage() {setSkippedExistingTasks(0);stopFaviconFlash();- const runFiles = filesToRun ?? selectedFiles;+ const isSetRun = inputSource === "set" && selectedSet;+ const runFiles =+ filesToRun ??+ (isSetRun+ ? // Whole set = empty list so the backend resolves every image.+ selectedSetFiles.length === setImages.length+ ? []+ : selectedSetFiles+ : selectedFiles);const response = await fetch(`${HTTP_BACKEND_URL}/run_evals_stream`, {method: "POST",@@ -146,6 +229,7 @@ function RunEvalsPage() {stack: selectedStack,files: runFiles,diff_mode: diffMode,+ set_name: isSetRun ? selectedSet : null,}),});@@ -273,7 +357,30 @@ function RunEvalsPage() {return `${selectedModels.slice(0, 2).join(", ")} +${selectedModels.length - 2} more`;};- const canRunEvals = selectedModels.length > 0 && selectedFiles.length > 0;+ const canRunEvals =+ selectedModels.length > 0 &&+ (inputSource === "set"+ ? Boolean(selectedSet) && selectedSetFiles.length > 0+ : selectedFiles.length > 0);+ const sessionSetMismatch =+ inputSource === "set" &&+ activeSession !== null &&+ Boolean(selectedSet) &&+ activeSession.eval_set !== selectedSet;++ const startSessionForSelectedSet = async () => {+ try {+ const response = await fetch(`${HTTP_BACKEND_URL}/eval-sessions`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ eval_set: selectedSet }),+ });+ if (!response.ok) throw new Error("Request failed");+ await refreshActiveSession();+ } catch (error) {+ console.error("Error starting session", error);+ }+ };const failedFilesForRerun = Array.from(new Set(failedTaskDetails.map((task) => task.inputFile).filter(Boolean)));@@ -510,11 +617,113 @@ function RunEvalsPage() {{/* Input Files Section */}<div className="bg-white rounded-lg border border-gray-200 shadow-sm lg:col-span-1 md:col-span-2">- <div className="border-b border-gray-200 px-4 py-3 bg-gray-50 rounded-t-lg">+ <div className="border-b border-gray-200 px-4 py-3 bg-gray-50 rounded-t-lg flex items-center justify-between"><h2 className="font-medium">Select Input Files</h2>+ <div className="flex rounded-md border border-gray-300 overflow-hidden text-xs">+ {(+ [+ ["legacy", "Legacy inputs"],+ ["set", "Eval set"],+ ] as const+ ).map(([source, label]) => (+ <button+ key={source}+ onClick={() => setInputSource(source)}+ className={`px-2.5 py-1 transition-colors ${+ inputSource === source+ ? "bg-blue-600 text-white"+ : "bg-white text-gray-600 hover:bg-gray-100"+ }`}+ >+ {label}+ </button>+ ))}+ </div></div><div className="p-3">- <InputFileSelector onFilesSelected={handleFilesSelected} />+ {inputSource === "legacy" ? (+ <InputFileSelector onFilesSelected={handleFilesSelected} />+ ) : (+ <div className="flex flex-col gap-3">+ <select+ value={selectedSet}+ onChange={(e) => setSelectedSet(e.target.value)}+ className="w-full py-2 px-3 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500"+ >+ {evalSets.length === 0 && (+ <option value="">No eval sets found</option>+ )}+ {evalSets.map((set) => (+ <option key={set.name} value={set.name}>+ {set.display_name} ({set.image_count} images)+ </option>+ ))}+ </select>++ {sessionSetMismatch && activeSession && (+ <div className="rounded-md border border-amber-300 bg-amber-50 px-3 py-2 text-xs text-amber-800">+ Active session "{activeSession.name}" is pinned to set "+ {activeSession.eval_set}".{" "}+ <button+ onClick={() => void startSessionForSelectedSet()}+ className="font-medium underline hover:text-amber-900"+ >+ Start a new session for "{selectedSet}"+ </button>{" "}+ before running.+ </div>+ )}+ {!sessionSetMismatch && inputSource === "set" && (+ <div className="rounded-md border border-blue-200 bg-blue-50 px-3 py-2 text-xs text-blue-800">+ {activeSession+ ? `Runs will attach to session "${activeSession.name}" (set "${activeSession.eval_set}").`+ : "No active session — one will be created automatically."}+ </div>+ )}++ <div className="border rounded-md max-h-[220px] overflow-y-auto">+ <div className="grid grid-cols-1 divide-y divide-gray-100">+ {setImages.map((image) => {+ const checked = selectedSetFiles.includes(+ image.filename+ );+ return (+ <div+ key={image.filename}+ className={`flex items-center justify-between px-3 py-1.5 cursor-pointer hover:bg-gray-50 ${+ checked ? "bg-blue-50" : ""+ }`}+ onClick={() =>+ setSelectedSetFiles((previous) =>+ checked+ ? previous.filter(+ (f) => f !== image.filename+ )+ : [...previous, image.filename]+ )+ }+ >+ <span+ className="text-sm truncate"+ title={image.filename}+ >+ {image.filename}+ </span>+ {checked ? (+ <BsCheckLg className="h-3.5 w-3.5 text-blue-500 flex-shrink-0" />+ ) : (+ <div className="h-3.5 w-3.5 border rounded-sm flex-shrink-0" />+ )}+ </div>+ );+ })}+ </div>+ </div>+ <p className="text-xs text-gray-500">+ Selected: {selectedSetFiles.length} / {setImages.length}+ </p>+ </div>+ )}</div></div></div>diff --git a/frontend/src/main.tsx b/frontend/src/main.tsxindex d2f2f065..38900450 100644--- a/frontend/src/main.tsx+++ b/frontend/src/main.tsx@@ -10,6 +10,7 @@ import AllEvalsPage from "./components/evals/AllEvalsPage.tsx";import OpenAIInputComparePage from "./components/evals/OpenAIInputComparePage.tsx";import PromptReportsPage from "./components/evals/PromptReportsPage.tsx";import AgentRunsPage from "./components/evals/AgentRunsPage.tsx";+import EvalSessionsPage from "./components/evals/EvalSessionsPage.tsx";ReactDOM.createRoot(document.getElementById("root")!).render(<React.StrictMode>@@ -25,6 +26,7 @@ ReactDOM.createRoot(document.getElementById("root")!).render(/><Route path="/evals/prompt-reports" element={<PromptReportsPage />} /><Route path="/evals/agent-runs" element={<AgentRunsPage />} />+ <Route path="/evals/sessions" element={<EvalSessionsPage />} /></Routes></Router><Toaster toastOptions={{ className: "dark:bg-zinc-950 dark:text-white" }} />