Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
import os
import asyncio
import json
from fastapi import APIRouter, Query, Request, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from evals.utils import image_to_data_url
from evals.config import EVALS_DIR
from typing import Optional, Set
from evals.runner import run_image_evals, count_pending_eval_tasks
from evals import sessions as eval_sessions
from evals import sets as eval_sets
from typing import List, Dict
from llm import Llm
from prompts.prompt_types import Stack
from pathlib import Path
from fs_logging.openai_input_compare import (
compare_openai_inputs,
format_openai_input_comparison,
)
router = APIRouter()
# Update this if the number of outputs generated per input changes
N = 1
class Eval(BaseModel):
input: str
outputs: list[str]
class InputFile(BaseModel):
name: str
path: str
@router.get("/eval_input_files", response_model=List[InputFile])
async def get_eval_input_files():
"""Get a list of all input files available for evaluations"""
input_dir = os.path.join(EVALS_DIR, "inputs")
try:
files: list[InputFile] = []
for filename in os.listdir(input_dir):
if filename.endswith(".png"):
file_path = os.path.join(input_dir, filename)
files.append(InputFile(name=filename, path=file_path))
return sorted(files, key=lambda x: x.name)
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Error reading input files: {str(e)}"
)
@router.get("/evals", response_model=list[Eval])
async def get_evals(folder: str):
if not folder:
raise HTTPException(status_code=400, detail="Folder path is required")
folder_path = Path(folder)
if not folder_path.exists():
raise HTTPException(status_code=404, detail=f"Folder not found: {folder}")
try:
evals: list[Eval] = []
# Get all HTML files from folder
files = {
f: os.path.join(folder, f)
for f in os.listdir(folder)
if f.endswith(".html")
}
# Extract base names
base_names: Set[str] = set()
for filename in files.keys():
base_name = (
filename.rsplit("_", 1)[0]
if "_" in filename
else filename.replace(".html", "")
)
base_names.add(base_name)
for base_name in base_names:
input_path = os.path.join(EVALS_DIR, "inputs", f"{base_name}.png")
if not os.path.exists(input_path):
continue
# Find matching output file
output_file = None
for filename, filepath in files.items():
if filename.startswith(base_name):
output_file = filepath
break
if output_file:
input_data = await image_to_data_url(input_path)
with open(output_file, "r", encoding="utf-8") as f:
output_html = f.read()
evals.append(Eval(input=input_data, outputs=[output_html]))
return evals
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error processing evals: {str(e)}")
class RunEvalsRequest(BaseModel):
models: List[str]
stack: Stack
files: List[str] = [] # Optional list of specific file paths to run evals on
diff_mode: bool = False
# When set, inputs come from {EVALS_DIR}/sets/{set_name}/inputs and runs
# attach to the active eval session (auto-created when none exists).
set_name: Optional[str] = None
def _resolve_set_run(
request: RunEvalsRequest,
) -> tuple[Optional[str], Optional[eval_sessions.EvalSession], Dict[str, set[str]]]:
"""Validate the requested set and resolve the session + per-model skips."""
if not request.set_name:
return None, None, {}
try:
set_info = eval_sets.get_set(request.set_name)
except eval_sets.InvalidSetNameError as e:
raise HTTPException(status_code=400, detail=str(e))
except eval_sets.EvalSetNotFoundError:
raise HTTPException(
status_code=404, detail=f"Eval set not found: {request.set_name}"
)
if set_info.image_count == 0:
raise HTTPException(
status_code=400, detail=f"Eval set {request.set_name!r} has no images"
)
try:
session = eval_sessions.resolve_session_for_run(request.set_name)
except eval_sessions.SessionSetMismatchError as e:
raise HTTPException(
status_code=400,
detail=(
f"Active session {e.active_session.name!r} is pinned to set "
f"{e.active_session.eval_set!r}. Start a new session for set "
f"{request.set_name!r} first (POST /eval-sessions)."
),
)
skip_per_model: Dict[str, set[str]] = {}
if request.diff_mode:
for model in request.models:
skip_per_model[model] = eval_sessions.completed_eval_inputs(
session.session_id, model, str(request.stack)
)
return request.set_name, session, skip_per_model
class OpenAIInputCompareRequest(BaseModel):
left_json: str
right_json: str
class OpenAIInputCompareDifferenceResponse(BaseModel):
item_index: int
path: str
left_summary: str
right_summary: str
left_value: object | None
right_value: object | None
class OpenAIInputCompareResponse(BaseModel):
common_prefix_items: int
left_item_count: int
right_item_count: int
difference: OpenAIInputCompareDifferenceResponse | None
formatted: str
def _load_openai_input_compare_payload(raw_json: str, side: str) -> object:
try:
payload = json.loads(raw_json)
except json.JSONDecodeError as error:
raise HTTPException(
status_code=400,
detail=(
f"Invalid {side} JSON: {error.msg} "
f"(line {error.lineno}, column {error.colno})"
),
)
try:
compare_openai_inputs(payload, payload)
except ValueError as error:
raise HTTPException(status_code=400, detail=f"Invalid {side} payload: {error}")
return payload
@router.post("/openai-input-compare", response_model=OpenAIInputCompareResponse)
async def compare_openai_inputs_for_evals(
request: OpenAIInputCompareRequest,
) -> OpenAIInputCompareResponse:
left_payload = _load_openai_input_compare_payload(request.left_json, "left")
right_payload = _load_openai_input_compare_payload(request.right_json, "right")
comparison = compare_openai_inputs(left_payload, right_payload)
difference = None
if comparison.difference is not None:
difference = OpenAIInputCompareDifferenceResponse(
item_index=comparison.difference.item_index,
path=comparison.difference.path,
left_summary=comparison.difference.left_summary,
right_summary=comparison.difference.right_summary,
left_value=comparison.difference.left_value,
right_value=comparison.difference.right_value,
)
return OpenAIInputCompareResponse(
common_prefix_items=comparison.common_prefix_items,
left_item_count=comparison.left_item_count,
right_item_count=comparison.right_item_count,
difference=difference,
formatted=format_openai_input_comparison(comparison),
)
@router.post("/run_evals", response_model=List[str])
async def run_evals(request: RunEvalsRequest) -> List[str]:
"""Run evaluations on selected images in the inputs directory for multiple models"""
eval_set, session, skip_per_model = _resolve_set_run(request)
all_output_files: List[str] = []
for model in request.models:
output_files = await run_image_evals(
model=model,
stack=request.stack,
input_files=request.files,
diff_mode=request.diff_mode,
eval_set=eval_set,
eval_session_id=session.session_id if session else None,
skip_input_files=skip_per_model.get(model),
)
all_output_files.extend(output_files)
return all_output_files
def _count_eval_files(selected_files: List[str], set_name: Optional[str] = None) -> int:
if set_name and eval_sets.get_set_kind(set_name) == "text":
brief_ids = {b.id for b in eval_sets.list_set_briefs(set_name)}
if selected_files:
return len([f for f in selected_files if f in brief_ids])
return len(brief_ids)
if selected_files:
return len([f for f in selected_files if f.endswith(".png")])
if set_name:
return len(eval_sets.list_set_images(set_name))
input_dir = os.path.join(EVALS_DIR, "inputs")
return len([f for f in os.listdir(input_dir) if f.endswith(".png")])
@router.post("/run_evals_stream")
async def run_evals_stream(request: RunEvalsRequest):
"""Run evaluations and stream progress events as newline-delimited JSON."""
if not request.models:
raise HTTPException(status_code=400, detail="At least one model is required")
eval_set, session, skip_per_model = _resolve_set_run(request)
per_model_task_counts: Dict[str, int] = {}
per_model_skipped_existing: Dict[str, int] = {}
if request.diff_mode:
for model in request.models:
pending_tasks, skipped_tasks = count_pending_eval_tasks(
stack=request.stack,
model=model,
input_files=request.files,
n=N,
diff_mode=True,
eval_set=eval_set,
skip_input_files=skip_per_model.get(model),
)
per_model_task_counts[model] = pending_tasks
per_model_skipped_existing[model] = skipped_tasks
else:
per_model_task_count = _count_eval_files(request.files, eval_set)
for model in request.models:
per_model_task_counts[model] = per_model_task_count
per_model_skipped_existing[model] = 0
total_tasks = sum(per_model_task_counts.values())
total_skipped_existing = sum(per_model_skipped_existing.values())
async def event_generator():
queue: asyncio.Queue[dict] = asyncio.Queue()
async def emit(event: dict) -> None:
await queue.put(event)
async def run_all_models() -> None:
all_output_files: List[str] = []
completed_offset = 0
try:
await emit(
{
"type": "start",
"total_models": len(request.models),
"tasks_per_model": per_model_task_counts,
"total_tasks": total_tasks,
"completed_tasks": 0,
"diff_mode": request.diff_mode,
"total_skipped_existing": total_skipped_existing,
"eval_set": eval_set,
"eval_session_id": session.session_id if session else None,
"eval_session_name": session.name if session else None,
}
)
for model_index, model in enumerate(request.models, start=1):
model_task_count = per_model_task_counts.get(model, 0)
model_skipped_existing = per_model_skipped_existing.get(model, 0)
await emit(
{
"type": "model_start",
"model": model,
"model_index": model_index,
"total_models": len(request.models),
"model_tasks": model_task_count,
"model_skipped_existing": model_skipped_existing,
}
)
async def on_progress(event: dict) -> None:
await emit(
{
**event,
"model": model,
"model_index": model_index,
"total_models": len(request.models),
"global_completed_tasks": completed_offset
+ event.get("completed_tasks", 0),
"global_total_tasks": total_tasks,
}
)
output_files = await run_image_evals(
model=model,
stack=request.stack,
input_files=request.files,
diff_mode=request.diff_mode,
progress_callback=on_progress,
eval_set=eval_set,
eval_session_id=session.session_id if session else None,
skip_input_files=skip_per_model.get(model),
)
all_output_files.extend(output_files)
completed_offset += model_task_count
await emit(
{
"type": "complete",
"completed_tasks": total_tasks,
"total_tasks": total_tasks,
"output_files": all_output_files,
}
)
except Exception as e:
await emit({"type": "error", "message": str(e)})
finally:
await emit({"type": "done"})
producer = asyncio.create_task(run_all_models())
while True:
event = await queue.get()
if event.get("type") == "done":
break
yield json.dumps(event) + "\n"
await producer
return StreamingResponse(event_generator(), media_type="application/x-ndjson")
@router.get("/models", response_model=Dict[str, List[str]])
async def get_models():
current_models = [model.value for model in Llm]
# Import Stack type from prompts.prompt_types and get all literal values
available_stacks = list(Stack.__args__)
return {"models": current_models, "stacks": available_stacks}
class BestOfNEvalsResponse(BaseModel):
evals: list[Eval]
folder_names: list[str]
@router.get("/best-of-n-evals", response_model=BestOfNEvalsResponse)
async def get_best_of_n_evals(request: Request):
# Get all query parameters
query_params = dict(request.query_params)
# Extract all folder paths (folder1, folder2, folder3, etc.)
folders: list[str] = []
i = 1
while f"folder{i}" in query_params:
folders.append(query_params[f"folder{i}"])
i += 1
if not folders:
return {"error": "No folders provided"}
# Validate folders exist
for folder in folders:
if not os.path.exists(folder):
return {"error": f"Folder does not exist: {folder}"}
evals: list[Eval] = []
folder_names = [os.path.basename(folder) for folder in folders]
# Get HTML files from all folders
files_by_folder = []
for folder in folders:
files = {
f: os.path.join(folder, f)
for f in os.listdir(folder)
if f.endswith(".html")
}
files_by_folder.append(files)
# Find common base names across all folders
common_names: Set[str] = set()
base_names_first_folder = {
f.rsplit("_", 1)[0] if "_" in f else f.replace(".html", "")
for f in files_by_folder[0].keys()
}
for base_name in base_names_first_folder:
found_in_all = True
for folder_files in files_by_folder[1:]:
if not any(f.startswith(base_name) for f in folder_files.keys()):
found_in_all = False
break
if found_in_all:
common_names.add(base_name)
# For each matching set, create an eval
for base_name in common_names:
# Find the corresponding input image
input_image = None
input_path = os.path.join(EVALS_DIR, "inputs", f"{base_name}.png")
if os.path.exists(input_path):
input_image = await image_to_data_url(input_path)
else:
input_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
# Get HTML contents from all folders
outputs: list[str] = []
for folder_files in files_by_folder:
output_content: str | None = None
for filename in folder_files.keys():
if filename.startswith(base_name):
with open(folder_files[filename], "r") as f:
output_content = f.read()
break
if output_content:
outputs.append(output_content)
else:
outputs.append("<html><body>Output not found</body></html>")
if len(outputs) == len(folders): # Only add if we have outputs from all folders
evals.append(Eval(input=input_image, outputs=outputs))
return BestOfNEvalsResponse(evals=evals, folder_names=folder_names)
class OutputFolder(BaseModel):
name: str
path: str
modified_time: float
@router.get("/output_folders", response_model=List[OutputFolder])
async def get_output_folders():
"""Get a list of all output folders available for evaluations, sorted by recently modified"""
output_dir = os.path.join(EVALS_DIR, "results")
try:
folders: list[OutputFolder] = []
for folder_name in os.listdir(output_dir):
folder_path = os.path.join(output_dir, folder_name)
if os.path.isdir(folder_path) and not folder_name.startswith("."):
# Get modification time
modified_time = os.path.getmtime(folder_path)
folders.append(
OutputFolder(
name=folder_name, path=folder_path, modified_time=modified_time
)
)
# Sort by modified time, most recent first
return sorted(folders, key=lambda x: x.modified_time, reverse=True)
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Error reading output folders: {str(e)}"
)