Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
"""API for eval sets, eval sessions, and the per-session run matrix."""
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse
from pydantic import BaseModel
from evals import sessions as eval_sessions
from evals import sets as eval_sets
from fs_logging.agent_runs import open_index_db
from routes.model_choice_sets import (
ALL_KEYS_MODELS_DEFAULT,
ALL_KEYS_MODELS_TEXT_CREATE,
)
router = APIRouter()
_STALE_RUNNING_AFTER = timedelta(hours=2)
class EvalSetSummaryModel(BaseModel):
name: str
display_name: str
created_at: Optional[str]
notes: str
image_count: int
kind: str = "image"
class EvalSetImageModel(BaseModel):
filename: str
sha256: str
size_bytes: int
tags: List[str]
class EvalSetBriefModel(BaseModel):
id: str
title: str
brief: str
tests: str
class EvalSetDetailModel(EvalSetSummaryModel):
images: List[EvalSetImageModel]
briefs: List[EvalSetBriefModel] = []
class EvalSessionModel(BaseModel):
session_id: str
name: str
eval_set: str
created_at: str
is_active: bool
class EvalSessionListResponse(BaseModel):
sessions: List[EvalSessionModel]
active_session_id: Optional[str]
class CreateEvalSessionRequest(BaseModel):
eval_set: str
name: Optional[str] = None
class MatrixRunModel(BaseModel):
run_id: str
status: str
source: str # "eval" | "ui"
total_duration_ms: Optional[int]
total_cost_usd: Optional[float]
created_at: str
is_stale: bool
class MatrixCellModel(BaseModel):
filename: str
model: str
runs: List[MatrixRunModel] # newest first
class MatrixRowModel(BaseModel):
filename: str # image filename, or brief id for text sets
sha256: Optional[str] = None
image_url: Optional[str] = None
title: Optional[str] = None
brief: Optional[str] = None
class SessionMatrixResponse(BaseModel):
session: EvalSessionModel
eval_set: Optional[EvalSetSummaryModel]
set_missing: bool
rows: List[MatrixRowModel]
models: List[str]
model_notes: Dict[str, str]
cells: List[MatrixCellModel]
unmatched_run_count: int
# Current production create defaults for the set's input kind.
default_models: List[str] = []
class ModelOrderRequest(BaseModel):
models: List[str]
class ModelNotesRequest(BaseModel):
model: str
notes: str
def _set_info_to_model(info: eval_sets.EvalSetInfo) -> EvalSetSummaryModel:
return EvalSetSummaryModel(
name=info.name,
display_name=info.display_name,
created_at=info.created_at,
notes=info.notes,
image_count=info.image_count,
kind=info.kind,
)
def _default_models_for_set(set_info: Optional[eval_sets.EvalSetInfo]) -> List[str]:
if set_info is None:
return []
if set_info.kind == "text":
return [model.value for model in ALL_KEYS_MODELS_TEXT_CREATE]
return [model.value for model in ALL_KEYS_MODELS_DEFAULT]
def _session_to_model(session: eval_sessions.EvalSession) -> EvalSessionModel:
return EvalSessionModel(
session_id=session.session_id,
name=session.name,
eval_set=session.eval_set,
created_at=session.created_at,
is_active=session.is_active,
)
@router.get("/eval-sets", response_model=List[EvalSetSummaryModel])
async def list_eval_sets() -> List[EvalSetSummaryModel]:
return [_set_info_to_model(info) for info in eval_sets.list_sets()]
@router.get("/eval-sets/{set_name}", response_model=EvalSetDetailModel)
async def get_eval_set(set_name: str) -> EvalSetDetailModel:
try:
info = eval_sets.get_set(set_name)
if info.kind == "text":
briefs = eval_sets.list_set_briefs(set_name)
return EvalSetDetailModel(
**_set_info_to_model(info).model_dump(),
images=[],
briefs=[
EvalSetBriefModel(
id=b.id, title=b.title, brief=b.brief, tests=b.tests
)
for b in briefs
],
)
images = eval_sets.list_set_images(set_name)
except eval_sets.InvalidSetNameError as e:
raise HTTPException(status_code=400, detail=str(e))
except eval_sets.EvalSetNotFoundError:
raise HTTPException(status_code=404, detail=f"Eval set not found: {set_name}")
return EvalSetDetailModel(
**_set_info_to_model(info).model_dump(),
images=[
EvalSetImageModel(
filename=image.filename,
sha256=image.sha256,
size_bytes=image.size_bytes,
tags=image.tags,
)
for image in images
],
)
@router.get("/eval-sets/{set_name}/images/{filename}")
async def get_eval_set_image(set_name: str, filename: str) -> FileResponse:
try:
path = eval_sets.resolve_set_image_path(set_name, filename)
except eval_sets.InvalidSetNameError as e:
raise HTTPException(status_code=400, detail=str(e))
except eval_sets.EvalSetNotFoundError:
raise HTTPException(status_code=404, detail=f"Eval set not found: {set_name}")
except FileNotFoundError:
raise HTTPException(status_code=404, detail="Image not found")
return FileResponse(path)
@router.get("/eval-sessions", response_model=EvalSessionListResponse)
async def list_eval_sessions() -> EvalSessionListResponse:
sessions = eval_sessions.list_sessions()
active = next((s for s in sessions if s.is_active), None)
return EvalSessionListResponse(
sessions=[_session_to_model(s) for s in sessions],
active_session_id=active.session_id if active else None,
)
@router.get("/eval-sessions/active", response_model=Optional[EvalSessionModel])
async def get_active_eval_session() -> Optional[EvalSessionModel]:
active = eval_sessions.get_active_session()
return _session_to_model(active) if active else None
@router.post("/eval-sessions", response_model=EvalSessionModel)
async def create_eval_session(request: CreateEvalSessionRequest) -> EvalSessionModel:
try:
eval_sets.get_set(request.eval_set)
except eval_sets.InvalidSetNameError as e:
raise HTTPException(status_code=400, detail=str(e))
except eval_sets.EvalSetNotFoundError:
raise HTTPException(
status_code=404, detail=f"Eval set not found: {request.eval_set}"
)
session = eval_sessions.create_session(request.eval_set, request.name)
return _session_to_model(session)
@router.post(
"/eval-sessions/{session_id}/activate", response_model=EvalSessionModel
)
async def activate_eval_session(session_id: str) -> EvalSessionModel:
if eval_sessions.get_session(session_id) is None:
raise HTTPException(status_code=404, detail="Session not found")
session = eval_sessions.activate_session(session_id)
assert session is not None
return _session_to_model(session)
def _is_stale_running(status: str, created_at: str) -> bool:
if status != "running":
return False
try:
started = datetime.fromisoformat(created_at)
except ValueError:
return True
return datetime.now() - started > _STALE_RUNNING_AFTER
@router.get(
"/eval-sessions/{session_id}/matrix", response_model=SessionMatrixResponse
)
async def get_session_matrix(session_id: str) -> SessionMatrixResponse:
session = eval_sessions.get_session(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
set_missing = False
set_info: Optional[eval_sets.EvalSetInfo] = None
images: List[eval_sets.EvalSetImage] = []
briefs: List[eval_sets.EvalSetBrief] = []
try:
set_info = eval_sets.get_set(session.eval_set)
if set_info.kind == "text":
briefs = eval_sets.list_set_briefs(session.eval_set)
else:
images = eval_sets.list_set_images(session.eval_set)
except (eval_sets.EvalSetNotFoundError, eval_sets.InvalidSetNameError):
set_missing = True
if briefs:
rows = [
MatrixRowModel(filename=b.id, title=b.title, brief=b.brief)
for b in briefs
]
set_filenames = {b.id for b in briefs}
else:
rows = [
MatrixRowModel(
filename=image.filename,
sha256=image.sha256,
image_url=f"/eval-sets/{session.eval_set}/images/{image.filename}",
)
for image in images
]
set_filenames = {image.filename for image in images}
filenames_by_sha: Dict[str, List[str]] = {}
for image in images:
filenames_by_sha.setdefault(image.sha256, []).append(image.filename)
run_query_columns = (
"run_id, model, status, total_duration_ms, total_cost_usd, "
"created_at, input_file, input_image_sha256"
)
conn = open_index_db()
try:
eval_rows = conn.execute(
f"SELECT {run_query_columns} FROM runs "
"WHERE eval_session = ? ORDER BY created_at DESC, run_id DESC",
(session_id,),
).fetchall()
ui_rows: List[tuple[Any, ...]] = []
if filenames_by_sha:
placeholders = ", ".join("?" for _ in filenames_by_sha)
ui_rows = conn.execute(
f"SELECT {run_query_columns} FROM runs "
"WHERE eval_session IS NULL AND entry_point != 'eval' "
f"AND created_at >= ? AND input_image_sha256 IN ({placeholders}) "
"ORDER BY created_at DESC, run_id DESC",
(session.created_at, *filenames_by_sha.keys()),
).fetchall()
finally:
conn.close()
# (filename, model) -> runs, insertion-ordered so cells stay newest-first.
cell_runs: Dict[tuple[str, str], List[MatrixRunModel]] = {}
model_first_seen: Dict[str, str] = {}
unmatched_run_count = 0
def add_run(filename: str, row: tuple[Any, ...], source: str) -> None:
run_id, model, status, duration, cost, created_at = row[:6]
cell_runs.setdefault((filename, model), []).append(
MatrixRunModel(
run_id=run_id,
status=status,
source=source,
total_duration_ms=duration,
total_cost_usd=cost,
created_at=created_at,
is_stale=_is_stale_running(status, created_at),
)
)
# Rows are scanned newest-first; keep overwriting so the oldest
# occurrence wins and columns order by first use in the session.
model_first_seen[model] = created_at
for row in eval_rows:
input_file = row[6]
# With a missing set the cells still carry the session's history;
# otherwise runs whose file left the set are counted, not shown.
if input_file and (set_missing or input_file in set_filenames):
add_run(input_file, row, "eval")
else:
unmatched_run_count += 1
for row in ui_rows:
sha = row[7]
for filename in filenames_by_sha.get(sha, []):
add_run(filename, row, "ui")
# Saved per-session column order wins; models without a saved position
# append afterwards in first-use order.
meta = eval_sessions.get_model_meta(session_id)
positions = {
model: m.position for model, m in meta.items() if m.position is not None
}
models = sorted(
model_first_seen,
key=lambda m: (
0 if m in positions else 1,
positions.get(m, 0),
model_first_seen[m],
),
)
model_notes = {
model: meta[model].notes
for model in models
if model in meta and meta[model].notes
}
cells = [
MatrixCellModel(filename=filename, model=model, runs=runs)
for (filename, model), runs in cell_runs.items()
]
return SessionMatrixResponse(
session=_session_to_model(session),
eval_set=_set_info_to_model(set_info) if set_info else None,
set_missing=set_missing,
rows=rows,
models=models,
model_notes=model_notes,
cells=cells,
unmatched_run_count=unmatched_run_count,
default_models=_default_models_for_set(set_info),
)
@router.put("/eval-sessions/{session_id}/model-order")
async def set_session_model_order(
session_id: str, request: ModelOrderRequest
) -> Dict[str, str]:
if eval_sessions.get_session(session_id) is None:
raise HTTPException(status_code=404, detail="Session not found")
eval_sessions.set_model_order(session_id, request.models)
return {"status": "ok"}
@router.put("/eval-sessions/{session_id}/model-notes")
async def set_session_model_notes(
session_id: str, request: ModelNotesRequest
) -> Dict[str, str]:
if eval_sessions.get_session(session_id) is None:
raise HTTPException(status_code=404, detail="Session not found")
eval_sessions.set_model_notes(session_id, request.model, request.notes)
return {"status": "ok"}