Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add per-session model column ordering and notes to the eval matrix

Abi Raja7c308b25461e
diff --git a/backend/evals/sessions.py b/backend/evals/sessions.py
index 2610fb70..a8db1ce5 100644
--- a/backend/evals/sessions.py
+++ b/backend/evals/sessions.py
@@ -163,6 +163,58 @@ def resolve_session_for_run(set_name: str) -> EvalSession:
return active
+@dataclass
+class SessionModelMeta:
+ position: Optional[int]
+ notes: str
+
+
+def get_model_meta(session_id: str) -> dict[str, SessionModelMeta]:
+ conn = open_index_db()
+ try:
+ rows = conn.execute(
+ "SELECT model, position, notes FROM eval_session_models "
+ "WHERE session_id = ?",
+ (session_id,),
+ ).fetchall()
+ finally:
+ conn.close()
+ return {
+ row[0]: SessionModelMeta(position=row[1], notes=row[2] or "")
+ for row in rows
+ }
+
+
+def set_model_order(session_id: str, models: list[str]) -> None:
+ """Persist the column order; models not listed keep their notes."""
+ conn = open_index_db()
+ try:
+ for position, model in enumerate(models):
+ conn.execute(
+ "INSERT INTO eval_session_models (session_id, model, position) "
+ "VALUES (?, ?, ?) "
+ "ON CONFLICT(session_id, model) DO UPDATE SET position = ?",
+ (session_id, model, position, position),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+
+def set_model_notes(session_id: str, model: str, notes: str) -> None:
+ conn = open_index_db()
+ try:
+ conn.execute(
+ "INSERT INTO eval_session_models (session_id, model, notes) "
+ "VALUES (?, ?, ?) "
+ "ON CONFLICT(session_id, model) DO UPDATE SET notes = ?",
+ (session_id, model, notes, notes),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+
def completed_eval_inputs(session_id: str, model: str, stack: str) -> set[str]:
"""Input filenames already completed in this session for (model, stack)."""
conn = open_index_db()
diff --git a/backend/fs_logging/agent_runs.py b/backend/fs_logging/agent_runs.py
index fbdd3910..fa2c6d78 100644
--- a/backend/fs_logging/agent_runs.py
+++ b/backend/fs_logging/agent_runs.py
@@ -120,6 +120,13 @@ CREATE TABLE IF NOT EXISTS eval_sessions (
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_eval_sessions_active
ON eval_sessions(is_active) WHERE is_active = 1;
+CREATE TABLE IF NOT EXISTS eval_session_models (
+ session_id TEXT NOT NULL,
+ model TEXT NOT NULL,
+ position INTEGER,
+ notes TEXT NOT NULL DEFAULT '',
+ PRIMARY KEY (session_id, model)
+);
CREATE TABLE IF NOT EXISTS llm_calls (
run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE,
step INTEGER NOT NULL,
diff --git a/backend/routes/eval_sets.py b/backend/routes/eval_sets.py
index 39df8eb2..713e1864 100644
--- a/backend/routes/eval_sets.py
+++ b/backend/routes/eval_sets.py
@@ -81,10 +81,20 @@ class SessionMatrixResponse(BaseModel):
set_missing: bool
rows: List[MatrixRowModel]
models: List[str]
+ model_notes: Dict[str, str]
cells: List[MatrixCellModel]
unmatched_run_count: int
+class ModelOrderRequest(BaseModel):
+ models: List[str]
+
+
+class ModelNotesRequest(BaseModel):
+ model: str
+ notes: str
+
+
def _set_info_to_model(info: eval_sets.EvalSetInfo) -> EvalSetSummaryModel:
return EvalSetSummaryModel(
name=info.name,
@@ -287,7 +297,25 @@ async def get_session_matrix(session_id: str) -> SessionMatrixResponse:
for filename in filenames_by_sha.get(sha, []):
add_run(filename, row, "ui")
- models = sorted(model_first_seen, key=lambda m: model_first_seen[m])
+ # Saved per-session column order wins; models without a saved position
+ # append afterwards in first-use order.
+ meta = eval_sessions.get_model_meta(session_id)
+ positions = {
+ model: m.position for model, m in meta.items() if m.position is not None
+ }
+ models = sorted(
+ model_first_seen,
+ key=lambda m: (
+ 0 if m in positions else 1,
+ positions.get(m, 0),
+ model_first_seen[m],
+ ),
+ )
+ model_notes = {
+ model: meta[model].notes
+ for model in models
+ if model in meta and meta[model].notes
+ }
cells = [
MatrixCellModel(filename=filename, model=model, runs=runs)
for (filename, model), runs in cell_runs.items()
@@ -299,6 +327,27 @@ async def get_session_matrix(session_id: str) -> SessionMatrixResponse:
set_missing=set_missing,
rows=rows,
models=models,
+ model_notes=model_notes,
cells=cells,
unmatched_run_count=unmatched_run_count,
)
+
+
[email protected]("/eval-sessions/{session_id}/model-order")
+async def set_session_model_order(
+ session_id: str, request: ModelOrderRequest
+) -> Dict[str, str]:
+ if eval_sessions.get_session(session_id) is None:
+ raise HTTPException(status_code=404, detail="Session not found")
+ eval_sessions.set_model_order(session_id, request.models)
+ return {"status": "ok"}
+
+
[email protected]("/eval-sessions/{session_id}/model-notes")
+async def set_session_model_notes(
+ session_id: str, request: ModelNotesRequest
+) -> Dict[str, str]:
+ if eval_sessions.get_session(session_id) is None:
+ raise HTTPException(status_code=404, detail="Session not found")
+ eval_sessions.set_model_notes(session_id, request.model, request.notes)
+ return {"status": "ok"}
diff --git a/frontend/src/components/evals/EvalSessionsPage.tsx b/frontend/src/components/evals/EvalSessionsPage.tsx
index 142a7b64..df299fe9 100644
--- a/frontend/src/components/evals/EvalSessionsPage.tsx
+++ b/frontend/src/components/evals/EvalSessionsPage.tsx
@@ -1,7 +1,7 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import toast from "react-hot-toast";
import { useNavigate } from "react-router-dom";
-import { BsBoxArrowUpRight } from "react-icons/bs";
+import { BsBoxArrowUpRight, BsGripVertical } from "react-icons/bs";
import { HTTP_BACKEND_URL } from "../../config";
import EvalNavigation from "./EvalNavigation";
@@ -51,6 +51,7 @@ interface SessionMatrix {
set_missing: boolean;
rows: MatrixRow[];
models: string[];
+ model_notes: Record<string, string>;
cells: MatrixCell[];
unmatched_run_count: number;
}
@@ -161,6 +162,12 @@ function EvalSessionsPage() {
const [newSessionSet, setNewSessionSet] = useState("");
const [newSessionName, setNewSessionName] = useState("");
const [isCreating, setIsCreating] = useState(false);
+ const [editingNotesModel, setEditingNotesModel] = useState<string | null>(
+ null
+ );
+ const [notesDraft, setNotesDraft] = useState("");
+ const [dragModel, setDragModel] = useState<string | null>(null);
+ const [dragOverModel, setDragOverModel] = useState<string | null>(null);
const cellsByKey = useMemo(() => {
const map = new Map<string, MatrixCell>();
@@ -308,6 +315,66 @@ function EvalSessionsPage() {
navigate(`/evals/agent-runs?run=${encodeURIComponent(runId)}`);
};
+ const saveOrder = async (reordered: string[]) => {
+ if (!matrix || !selectedSessionId) return;
+ setMatrix({ ...matrix, models: reordered });
+ try {
+ await fetch(
+ `${HTTP_BACKEND_URL}/eval-sessions/${encodeURIComponent(
+ selectedSessionId
+ )}/model-order`,
+ {
+ method: "PUT",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ models: reordered }),
+ }
+ );
+ } catch (error) {
+ console.error("Error saving model order", error);
+ toast.error("Failed to save column order.");
+ }
+ };
+
+ const handleColumnDrop = (target: string) => {
+ if (!matrix || !dragModel || dragModel === target) return;
+ const dragIndex = matrix.models.indexOf(dragModel);
+ const targetIndex = matrix.models.indexOf(target);
+ if (dragIndex < 0 || targetIndex < 0) return;
+ const reordered = matrix.models.filter((m) => m !== dragModel);
+ // Dragging rightwards drops after the target, leftwards before it.
+ const insertAt =
+ dragIndex < targetIndex
+ ? reordered.indexOf(target) + 1
+ : reordered.indexOf(target);
+ reordered.splice(insertAt, 0, dragModel);
+ void saveOrder(reordered);
+ };
+
+ const saveNotes = async (model: string) => {
+ if (!matrix || !selectedSessionId) return;
+ const notes = notesDraft.trim();
+ setEditingNotesModel(null);
+ setMatrix({
+ ...matrix,
+ model_notes: { ...matrix.model_notes, [model]: notes },
+ });
+ try {
+ await fetch(
+ `${HTTP_BACKEND_URL}/eval-sessions/${encodeURIComponent(
+ selectedSessionId
+ )}/model-notes`,
+ {
+ method: "PUT",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ model, notes }),
+ }
+ );
+ } catch (error) {
+ console.error("Error saving model notes", error);
+ toast.error("Failed to save notes.");
+ }
+ };
+
return (
<div className="flex h-screen flex-col overflow-hidden bg-zinc-950 text-white">
<div className="shrink-0">
@@ -449,10 +516,82 @@ function EvalSessionsPage() {
{matrix.models.map((model) => (
<th
key={model}
- className="min-w-[140px] px-2 py-1 text-left font-mono text-[11px] font-normal text-zinc-400"
+ onDragOver={(e) => {
+ e.preventDefault();
+ if (dragModel && dragModel !== model) {
+ setDragOverModel(model);
+ }
+ }}
+ onDragLeave={() =>
+ setDragOverModel((current) =>
+ current === model ? null : current
+ )
+ }
+ onDrop={(e) => {
+ e.preventDefault();
+ handleColumnDrop(model);
+ setDragModel(null);
+ setDragOverModel(null);
+ }}
+ className={`group/col min-w-[140px] px-2 py-1 text-left align-top font-mono text-[11px] font-normal text-zinc-400 transition-colors ${
+ dragOverModel === model
+ ? "rounded-md bg-blue-950/60 ring-1 ring-blue-600"
+ : ""
+ } ${dragModel === model ? "opacity-40" : ""}`}
title={model}
>
- {model}
+ <div
+ draggable
+ onDragStart={(e) => {
+ e.dataTransfer.effectAllowed = "move";
+ e.dataTransfer.setData("text/plain", model);
+ setDragModel(model);
+ }}
+ onDragEnd={() => {
+ setDragModel(null);
+ setDragOverModel(null);
+ }}
+ className="flex cursor-grab items-start gap-1 active:cursor-grabbing"
+ >
+ <BsGripVertical className="mt-0.5 shrink-0 text-zinc-600 opacity-0 transition-opacity group-hover/col:opacity-100" />
+ <span className="min-w-0 flex-1">{model}</span>
+ </div>
+ {editingNotesModel === model ? (
+ <input
+ autoFocus
+ value={notesDraft}
+ onChange={(e) =>
+ setNotesDraft(e.target.value)
+ }
+ onBlur={() => void saveNotes(model)}
+ onKeyDown={(e) => {
+ if (e.key === "Enter") {
+ void saveNotes(model);
+ } else if (e.key === "Escape") {
+ setEditingNotesModel(null);
+ }
+ }}
+ placeholder="Notes for this model…"
+ className="mt-1 w-full rounded border border-zinc-600 bg-zinc-950 px-1.5 py-0.5 font-sans text-[11px] text-zinc-200 placeholder:text-zinc-600"
+ />
+ ) : (
+ <button
+ onClick={() => {
+ setEditingNotesModel(model);
+ setNotesDraft(
+ matrix.model_notes[model] ?? ""
+ );
+ }}
+ title="Click to edit notes"
+ className={`mt-1 block w-full truncate rounded px-1 py-0.5 text-left font-sans text-[11px] transition-colors hover:bg-zinc-800 ${
+ matrix.model_notes[model]
+ ? "text-amber-200/90"
+ : "text-zinc-600 opacity-0 group-hover/col:opacity-100"
+ }`}
+ >
+ {matrix.model_notes[model] || "+ add note"}
+ </button>
+ )}
</th>
))}
</tr>