Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Add eval rerun and diff mode
Abi Raja705d309f8d6f
diff --git a/backend/evals/runner.py b/backend/evals/runner.pyindex 4a0c96b6..7e4c672a 100644--- a/backend/evals/runner.py+++ b/backend/evals/runner.py@@ -12,6 +12,49 @@ from .config import EVALS_DIRMAX_EVAL_RETRIES = 2++def _resolve_eval_filenames(input_files: Optional[List[str]]) -> List[str]:+ input_dir = EVALS_DIR + "/inputs"+ if input_files and len(input_files) > 0:+ return [os.path.basename(f) for f in input_files if f.endswith(".png")]+ return [f for f in os.listdir(input_dir) if f.endswith(".png")]+++def _output_html_filename(original_filename: str, attempt_idx: int) -> str:+ return f"{os.path.splitext(original_filename)[0]}_{attempt_idx}.html"+++def get_eval_output_subfolder(stack: Stack, model: str) -> str:+ today = datetime.now().strftime("%b_%d_%Y")+ output_dir = EVALS_DIR + "/outputs"+ return os.path.join(output_dir, f"{today}_{model}_{stack}")+++def count_pending_eval_tasks(+ stack: Stack,+ model: str,+ input_files: Optional[List[str]] = None,+ n: int = 1,+ diff_mode: bool = False,+) -> Tuple[int, int]:+ evals = _resolve_eval_filenames(input_files)+ if not diff_mode:+ return len(evals) * n, 0++ output_subfolder = get_eval_output_subfolder(stack=stack, model=model)+ pending_tasks = 0+ skipped_existing_tasks = 0+ for original_filename in evals:+ for n_idx in range(n):+ output_filename = _output_html_filename(original_filename, n_idx)+ output_path = os.path.join(output_subfolder, output_filename)+ if os.path.exists(output_path):+ skipped_existing_tasks += 1+ else:+ pending_tasks += 1+ return pending_tasks, skipped_existing_tasks++async def generate_code_and_time(image_url: str,stack: Stack,@@ -69,18 +112,11 @@ async def run_image_evals(model: Optional[str] = None,n: int = 1,input_files: Optional[List[str]] = None,+ diff_mode: bool = False,progress_callback: Optional[Callable[[dict[str, Any]], Any | Awaitable[Any]]] = None,) -> List[str]:INPUT_DIR = EVALS_DIR + "/inputs"- OUTPUT_DIR = EVALS_DIR + "/outputs"-- # Get all evaluation image files- if input_files and len(input_files) > 0:- # Use the explicitly provided file list- evals = [os.path.basename(f) for f in input_files if f.endswith(".png")]- else:- # Use all PNG files from the input directory- evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]+ evals = _resolve_eval_filenames(input_files)if not stack:raise ValueError("No stack was provided")@@ -97,9 +133,9 @@ async def run_image_evals(else:print(f"Running on all {len(evals)} files in {INPUT_DIR}")- today = datetime.now().strftime("%b_%d_%Y")- output_subfolder = os.path.join(- OUTPUT_DIR, f"{today}_{selected_model.value}_{stack}"+ output_subfolder = get_eval_output_subfolder(+ stack=stack,+ model=selected_model.value,)os.makedirs(output_subfolder, exist_ok=True)@@ -110,6 +146,7 @@ async def run_image_evals(Tuple[str, int, Optional[str], Optional[float], Optional[Exception], int],]] = []+ skipped_existing_tasks = 0for original_filename in evals:# Handle both full paths and relative filenamesif os.path.isabs(original_filename):@@ -117,9 +154,17 @@ async def run_image_evals(original_filename = os.path.basename(original_filename)else:filepath = os.path.join(INPUT_DIR, original_filename)-- data_url = await image_to_data_url(filepath)++ data_url: Optional[str] = Nonefor n_idx in range(n):+ output_filename = _output_html_filename(original_filename, n_idx)+ output_path = os.path.join(output_subfolder, output_filename)+ if diff_mode and os.path.exists(output_path):+ skipped_existing_tasks += 1+ continue++ if data_url is None:+ data_url = await image_to_data_url(filepath)current_model_for_task = (selected_model if n_idx == 0 else Llm.GPT_4_1_2025_04_14)@@ -132,6 +177,12 @@ async def run_image_evals()task_coroutines.append(coro)+ if diff_mode and skipped_existing_tasks > 0:+ print(+ f"Diff mode: skipping {skipped_existing_tasks} existing outputs for "+ f"{selected_model.value}"+ )+print(f"Processing {len(task_coroutines)} tasks...")total_tasks = len(task_coroutines)completed_tasks = 0diff --git a/backend/routes/evals.py b/backend/routes/evals.pyindex f6a044bb..fc5cd5ed 100644--- a/backend/routes/evals.py+++ b/backend/routes/evals.py@@ -7,7 +7,7 @@ from pydantic import BaseModelfrom evals.utils import image_to_data_urlfrom evals.config import EVALS_DIRfrom typing import Set-from evals.runner import run_image_evals+from evals.runner import run_image_evals, count_pending_eval_tasksfrom typing import List, Dictfrom llm import Llmfrom prompts.prompt_types import Stack@@ -180,6 +180,7 @@ class RunEvalsRequest(BaseModel):models: List[str]stack: Stackfiles: List[str] = [] # Optional list of specific file paths to run evals on+ diff_mode: bool = False@router.post("/run_evals", response_model=List[str])@@ -189,7 +190,10 @@ async def run_evals(request: RunEvalsRequest) -> List[str]:for model in request.models:output_files = await run_image_evals(- model=model, stack=request.stack, input_files=request.files+ model=model,+ stack=request.stack,+ input_files=request.files,+ diff_mode=request.diff_mode,)all_output_files.extend(output_files)@@ -210,8 +214,27 @@ async def run_evals_stream(request: RunEvalsRequest):if not request.models:raise HTTPException(status_code=400, detail="At least one model is required")- per_model_task_count = _count_eval_files(request.files)- total_tasks = per_model_task_count * len(request.models)+ per_model_task_counts: Dict[str, int] = {}+ per_model_skipped_existing: Dict[str, int] = {}+ if request.diff_mode:+ for model in request.models:+ pending_tasks, skipped_tasks = count_pending_eval_tasks(+ stack=request.stack,+ model=model,+ input_files=request.files,+ n=N,+ diff_mode=True,+ )+ per_model_task_counts[model] = pending_tasks+ per_model_skipped_existing[model] = skipped_tasks+ else:+ per_model_task_count = _count_eval_files(request.files)+ for model in request.models:+ per_model_task_counts[model] = per_model_task_count+ per_model_skipped_existing[model] = 0++ total_tasks = sum(per_model_task_counts.values())+ total_skipped_existing = sum(per_model_skipped_existing.values())async def event_generator():queue: asyncio.Queue[dict] = asyncio.Queue()@@ -228,19 +251,25 @@ async def run_evals_stream(request: RunEvalsRequest):{"type": "start","total_models": len(request.models),- "tasks_per_model": per_model_task_count,+ "tasks_per_model": per_model_task_counts,"total_tasks": total_tasks,"completed_tasks": 0,+ "diff_mode": request.diff_mode,+ "total_skipped_existing": total_skipped_existing,})for model_index, model in enumerate(request.models, start=1):+ model_task_count = per_model_task_counts.get(model, 0)+ model_skipped_existing = per_model_skipped_existing.get(model, 0)await emit({"type": "model_start","model": model,"model_index": model_index,"total_models": len(request.models),+ "model_tasks": model_task_count,+ "model_skipped_existing": model_skipped_existing,})@@ -261,10 +290,11 @@ async def run_evals_stream(request: RunEvalsRequest):model=model,stack=request.stack,input_files=request.files,+ diff_mode=request.diff_mode,progress_callback=on_progress,)all_output_files.extend(output_files)- completed_offset += per_model_task_count+ completed_offset += model_task_countawait emit({diff --git a/frontend/src/components/evals/RunEvalsPage.tsx b/frontend/src/components/evals/RunEvalsPage.tsxindex 6710e95f..6faab295 100644--- a/frontend/src/components/evals/RunEvalsPage.tsx+++ b/frontend/src/components/evals/RunEvalsPage.tsx@@ -25,6 +25,10 @@ interface EvalProgressEvent {success?: boolean;error?: string;output_files?: string[];+ diff_mode?: boolean;+ total_skipped_existing?: number;+ model_tasks?: number;+ model_skipped_existing?: number;}interface FailedTask {@@ -39,6 +43,7 @@ function RunEvalsPage() {const [stacks, setStacks] = useState<string[]>([]);const [selectedModels, setSelectedModels] = useState<string[]>([]);const [selectedStack, setSelectedStack] = useState<string>("html_tailwind");+ const [diffMode, setDiffMode] = useState(false);const [selectedFiles, setSelectedFiles] = useState<string[]>([]);const [showPaths, setShowPaths] = useState<boolean>(false);const [completedTasks, setCompletedTasks] = useState(0);@@ -48,6 +53,7 @@ function RunEvalsPage() {const [lastProcessedFile, setLastProcessedFile] = useState<string>("");const [failedTasks, setFailedTasks] = useState(0);const [failedTaskDetails, setFailedTaskDetails] = useState<FailedTask[]>([]);+ const [skippedExistingTasks, setSkippedExistingTasks] = useState(0);useEffect(() => {const fetchModels = async () => {@@ -85,6 +91,7 @@ function RunEvalsPage() {setLastProcessedFile("");setFailedTasks(0);setFailedTaskDetails([]);+ setSkippedExistingTasks(0);const runFiles = filesToRun ?? selectedFiles;@@ -97,6 +104,7 @@ function RunEvalsPage() {models: selectedModels,stack: selectedStack,files: runFiles,+ diff_mode: diffMode,}),});@@ -132,12 +140,21 @@ function RunEvalsPage() {if (event.type === "start") {const eventTotalTasks = event.total_tasks ?? 0;setTotalTasks(eventTotalTasks);- setStatusMessage("Starting evaluation run...");+ setSkippedExistingTasks(event.total_skipped_existing ?? 0);+ setStatusMessage(+ event.diff_mode+ ? `Starting diff run (${event.total_skipped_existing ?? 0} existing outputs skipped)...`+ : "Starting evaluation run..."+ );updateRunningTitle(0, eventTotalTasks);} else if (event.type === "model_start") {if (event.model) setCurrentModel(event.model);setStatusMessage(- `Running model ${event.model_index ?? 1}/${event.total_models ?? selectedModels.length}: ${event.model ?? "Unknown"}`+ `Running model ${event.model_index ?? 1}/${event.total_models ?? selectedModels.length}: ${event.model ?? "Unknown"}${+ diffMode && (event.model_skipped_existing ?? 0) > 0+ ? ` (${event.model_skipped_existing} skipped)`+ : ""+ }`);} else if (event.type === "task_complete") {const globalCompleted = event.global_completed_tasks ?? 0;@@ -242,6 +259,12 @@ function RunEvalsPage() {{isRunning ? "Running..." : "Run Evals"}</Button></div>++ {diffMode && (+ <div className="rounded-md border border-emerald-200 bg-emerald-50 px-3 py-2 text-sm text-emerald-800">+ Diff mode enabled: only input files missing outputs in today's model folders will run. Existing outputs are skipped and never overwritten.+ </div>+ )}<div className="grid grid-cols-1 md:grid-cols-3 gap-3 border-t border-gray-100 pt-3"><div className="flex flex-col">@@ -307,6 +330,12 @@ function RunEvalsPage() {<span className="font-mono">{lastProcessedFile || "-"}</span></span></div>+ {diffMode && (+ <div className="mt-2 text-xs text-emerald-700">+ Existing outputs skipped:{" "}+ <span className="font-medium">{skippedExistingTasks}</span>+ </div>+ )}{failedTaskDetails.length > 0 && (<div className="mt-3 border-t border-gray-100 pt-3"><div className="flex items-center justify-between mb-2">@@ -424,6 +453,15 @@ function RunEvalsPage() {</option>))}</select>+ <label className="mt-3 flex items-center gap-2 text-sm text-gray-700 cursor-pointer">+ <input+ type="checkbox"+ checked={diffMode}+ onChange={(e) => setDiffMode(e.target.checked)}+ className="h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500"+ />+ <span>Diff mode (only run missing outputs, no overwrite)</span>+ </label></div></div>