Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add eval rerun and diff mode

Abi Raja705d309f8d6f
diff --git a/backend/evals/runner.py b/backend/evals/runner.py
index 4a0c96b6..7e4c672a 100644
--- a/backend/evals/runner.py
+++ b/backend/evals/runner.py
@@ -12,6 +12,49 @@ from .config import EVALS_DIR
MAX_EVAL_RETRIES = 2
+
+def _resolve_eval_filenames(input_files: Optional[List[str]]) -> List[str]:
+ input_dir = EVALS_DIR + "/inputs"
+ if input_files and len(input_files) > 0:
+ return [os.path.basename(f) for f in input_files if f.endswith(".png")]
+ return [f for f in os.listdir(input_dir) if f.endswith(".png")]
+
+
+def _output_html_filename(original_filename: str, attempt_idx: int) -> str:
+ return f"{os.path.splitext(original_filename)[0]}_{attempt_idx}.html"
+
+
+def get_eval_output_subfolder(stack: Stack, model: str) -> str:
+ today = datetime.now().strftime("%b_%d_%Y")
+ output_dir = EVALS_DIR + "/outputs"
+ return os.path.join(output_dir, f"{today}_{model}_{stack}")
+
+
+def count_pending_eval_tasks(
+ stack: Stack,
+ model: str,
+ input_files: Optional[List[str]] = None,
+ n: int = 1,
+ diff_mode: bool = False,
+) -> Tuple[int, int]:
+ evals = _resolve_eval_filenames(input_files)
+ if not diff_mode:
+ return len(evals) * n, 0
+
+ output_subfolder = get_eval_output_subfolder(stack=stack, model=model)
+ pending_tasks = 0
+ skipped_existing_tasks = 0
+ for original_filename in evals:
+ for n_idx in range(n):
+ output_filename = _output_html_filename(original_filename, n_idx)
+ output_path = os.path.join(output_subfolder, output_filename)
+ if os.path.exists(output_path):
+ skipped_existing_tasks += 1
+ else:
+ pending_tasks += 1
+ return pending_tasks, skipped_existing_tasks
+
+
async def generate_code_and_time(
image_url: str,
stack: Stack,
@@ -69,18 +112,11 @@ async def run_image_evals(
model: Optional[str] = None,
n: int = 1,
input_files: Optional[List[str]] = None,
+ diff_mode: bool = False,
progress_callback: Optional[Callable[[dict[str, Any]], Any | Awaitable[Any]]] = None,
) -> List[str]:
INPUT_DIR = EVALS_DIR + "/inputs"
- OUTPUT_DIR = EVALS_DIR + "/outputs"
-
- # Get all evaluation image files
- if input_files and len(input_files) > 0:
- # Use the explicitly provided file list
- evals = [os.path.basename(f) for f in input_files if f.endswith(".png")]
- else:
- # Use all PNG files from the input directory
- evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]
+ evals = _resolve_eval_filenames(input_files)
if not stack:
raise ValueError("No stack was provided")
@@ -97,9 +133,9 @@ async def run_image_evals(
else:
print(f"Running on all {len(evals)} files in {INPUT_DIR}")
- today = datetime.now().strftime("%b_%d_%Y")
- output_subfolder = os.path.join(
- OUTPUT_DIR, f"{today}_{selected_model.value}_{stack}"
+ output_subfolder = get_eval_output_subfolder(
+ stack=stack,
+ model=selected_model.value,
)
os.makedirs(output_subfolder, exist_ok=True)
@@ -110,6 +146,7 @@ async def run_image_evals(
Tuple[str, int, Optional[str], Optional[float], Optional[Exception], int],
]
] = []
+ skipped_existing_tasks = 0
for original_filename in evals:
# Handle both full paths and relative filenames
if os.path.isabs(original_filename):
@@ -117,9 +154,17 @@ async def run_image_evals(
original_filename = os.path.basename(original_filename)
else:
filepath = os.path.join(INPUT_DIR, original_filename)
-
- data_url = await image_to_data_url(filepath)
+
+ data_url: Optional[str] = None
for n_idx in range(n):
+ output_filename = _output_html_filename(original_filename, n_idx)
+ output_path = os.path.join(output_subfolder, output_filename)
+ if diff_mode and os.path.exists(output_path):
+ skipped_existing_tasks += 1
+ continue
+
+ if data_url is None:
+ data_url = await image_to_data_url(filepath)
current_model_for_task = (
selected_model if n_idx == 0 else Llm.GPT_4_1_2025_04_14
)
@@ -132,6 +177,12 @@ async def run_image_evals(
)
task_coroutines.append(coro)
+ if diff_mode and skipped_existing_tasks > 0:
+ print(
+ f"Diff mode: skipping {skipped_existing_tasks} existing outputs for "
+ f"{selected_model.value}"
+ )
+
print(f"Processing {len(task_coroutines)} tasks...")
total_tasks = len(task_coroutines)
completed_tasks = 0
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index f6a044bb..fc5cd5ed 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -7,7 +7,7 @@ from pydantic import BaseModel
from evals.utils import image_to_data_url
from evals.config import EVALS_DIR
from typing import Set
-from evals.runner import run_image_evals
+from evals.runner import run_image_evals, count_pending_eval_tasks
from typing import List, Dict
from llm import Llm
from prompts.prompt_types import Stack
@@ -180,6 +180,7 @@ class RunEvalsRequest(BaseModel):
models: List[str]
stack: Stack
files: List[str] = [] # Optional list of specific file paths to run evals on
+ diff_mode: bool = False
@router.post("/run_evals", response_model=List[str])
@@ -189,7 +190,10 @@ async def run_evals(request: RunEvalsRequest) -> List[str]:
for model in request.models:
output_files = await run_image_evals(
- model=model, stack=request.stack, input_files=request.files
+ model=model,
+ stack=request.stack,
+ input_files=request.files,
+ diff_mode=request.diff_mode,
)
all_output_files.extend(output_files)
@@ -210,8 +214,27 @@ async def run_evals_stream(request: RunEvalsRequest):
if not request.models:
raise HTTPException(status_code=400, detail="At least one model is required")
- per_model_task_count = _count_eval_files(request.files)
- total_tasks = per_model_task_count * len(request.models)
+ per_model_task_counts: Dict[str, int] = {}
+ per_model_skipped_existing: Dict[str, int] = {}
+ if request.diff_mode:
+ for model in request.models:
+ pending_tasks, skipped_tasks = count_pending_eval_tasks(
+ stack=request.stack,
+ model=model,
+ input_files=request.files,
+ n=N,
+ diff_mode=True,
+ )
+ per_model_task_counts[model] = pending_tasks
+ per_model_skipped_existing[model] = skipped_tasks
+ else:
+ per_model_task_count = _count_eval_files(request.files)
+ for model in request.models:
+ per_model_task_counts[model] = per_model_task_count
+ per_model_skipped_existing[model] = 0
+
+ total_tasks = sum(per_model_task_counts.values())
+ total_skipped_existing = sum(per_model_skipped_existing.values())
async def event_generator():
queue: asyncio.Queue[dict] = asyncio.Queue()
@@ -228,19 +251,25 @@ async def run_evals_stream(request: RunEvalsRequest):
{
"type": "start",
"total_models": len(request.models),
- "tasks_per_model": per_model_task_count,
+ "tasks_per_model": per_model_task_counts,
"total_tasks": total_tasks,
"completed_tasks": 0,
+ "diff_mode": request.diff_mode,
+ "total_skipped_existing": total_skipped_existing,
}
)
for model_index, model in enumerate(request.models, start=1):
+ model_task_count = per_model_task_counts.get(model, 0)
+ model_skipped_existing = per_model_skipped_existing.get(model, 0)
await emit(
{
"type": "model_start",
"model": model,
"model_index": model_index,
"total_models": len(request.models),
+ "model_tasks": model_task_count,
+ "model_skipped_existing": model_skipped_existing,
}
)
@@ -261,10 +290,11 @@ async def run_evals_stream(request: RunEvalsRequest):
model=model,
stack=request.stack,
input_files=request.files,
+ diff_mode=request.diff_mode,
progress_callback=on_progress,
)
all_output_files.extend(output_files)
- completed_offset += per_model_task_count
+ completed_offset += model_task_count
await emit(
{
diff --git a/frontend/src/components/evals/RunEvalsPage.tsx b/frontend/src/components/evals/RunEvalsPage.tsx
index 6710e95f..6faab295 100644
--- a/frontend/src/components/evals/RunEvalsPage.tsx
+++ b/frontend/src/components/evals/RunEvalsPage.tsx
@@ -25,6 +25,10 @@ interface EvalProgressEvent {
success?: boolean;
error?: string;
output_files?: string[];
+ diff_mode?: boolean;
+ total_skipped_existing?: number;
+ model_tasks?: number;
+ model_skipped_existing?: number;
}
interface FailedTask {
@@ -39,6 +43,7 @@ function RunEvalsPage() {
const [stacks, setStacks] = useState<string[]>([]);
const [selectedModels, setSelectedModels] = useState<string[]>([]);
const [selectedStack, setSelectedStack] = useState<string>("html_tailwind");
+ const [diffMode, setDiffMode] = useState(false);
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
const [showPaths, setShowPaths] = useState<boolean>(false);
const [completedTasks, setCompletedTasks] = useState(0);
@@ -48,6 +53,7 @@ function RunEvalsPage() {
const [lastProcessedFile, setLastProcessedFile] = useState<string>("");
const [failedTasks, setFailedTasks] = useState(0);
const [failedTaskDetails, setFailedTaskDetails] = useState<FailedTask[]>([]);
+ const [skippedExistingTasks, setSkippedExistingTasks] = useState(0);
useEffect(() => {
const fetchModels = async () => {
@@ -85,6 +91,7 @@ function RunEvalsPage() {
setLastProcessedFile("");
setFailedTasks(0);
setFailedTaskDetails([]);
+ setSkippedExistingTasks(0);
const runFiles = filesToRun ?? selectedFiles;
@@ -97,6 +104,7 @@ function RunEvalsPage() {
models: selectedModels,
stack: selectedStack,
files: runFiles,
+ diff_mode: diffMode,
}),
});
@@ -132,12 +140,21 @@ function RunEvalsPage() {
if (event.type === "start") {
const eventTotalTasks = event.total_tasks ?? 0;
setTotalTasks(eventTotalTasks);
- setStatusMessage("Starting evaluation run...");
+ setSkippedExistingTasks(event.total_skipped_existing ?? 0);
+ setStatusMessage(
+ event.diff_mode
+ ? `Starting diff run (${event.total_skipped_existing ?? 0} existing outputs skipped)...`
+ : "Starting evaluation run..."
+ );
updateRunningTitle(0, eventTotalTasks);
} else if (event.type === "model_start") {
if (event.model) setCurrentModel(event.model);
setStatusMessage(
- `Running model ${event.model_index ?? 1}/${event.total_models ?? selectedModels.length}: ${event.model ?? "Unknown"}`
+ `Running model ${event.model_index ?? 1}/${event.total_models ?? selectedModels.length}: ${event.model ?? "Unknown"}${
+ diffMode && (event.model_skipped_existing ?? 0) > 0
+ ? ` (${event.model_skipped_existing} skipped)`
+ : ""
+ }`
);
} else if (event.type === "task_complete") {
const globalCompleted = event.global_completed_tasks ?? 0;
@@ -242,6 +259,12 @@ function RunEvalsPage() {
{isRunning ? "Running..." : "Run Evals"}
</Button>
</div>
+
+ {diffMode && (
+ <div className="rounded-md border border-emerald-200 bg-emerald-50 px-3 py-2 text-sm text-emerald-800">
+ Diff mode enabled: only input files missing outputs in today's model folders will run. Existing outputs are skipped and never overwritten.
+ </div>
+ )}
<div className="grid grid-cols-1 md:grid-cols-3 gap-3 border-t border-gray-100 pt-3">
<div className="flex flex-col">
@@ -307,6 +330,12 @@ function RunEvalsPage() {
<span className="font-mono">{lastProcessedFile || "-"}</span>
</span>
</div>
+ {diffMode && (
+ <div className="mt-2 text-xs text-emerald-700">
+ Existing outputs skipped:{" "}
+ <span className="font-medium">{skippedExistingTasks}</span>
+ </div>
+ )}
{failedTaskDetails.length > 0 && (
<div className="mt-3 border-t border-gray-100 pt-3">
<div className="flex items-center justify-between mb-2">
@@ -424,6 +453,15 @@ function RunEvalsPage() {
</option>
))}
</select>
+ <label className="mt-3 flex items-center gap-2 text-sm text-gray-700 cursor-pointer">
+ <input
+ type="checkbox"
+ checked={diffMode}
+ onChange={(e) => setDiffMode(e.target.checked)}
+ className="h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500"
+ />
+ <span>Diff mode (only run missing outputs, no overwrite)</span>
+ </label>
</div>
</div>