Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

build of best n evals viewer

Abi Rajaa484544897ba
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index da902fce..3a9073ed 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -1,5 +1,5 @@
import os
-from fastapi import APIRouter, Query
+from fastapi import APIRouter, Query, Request
from pydantic import BaseModel
from evals.utils import image_to_data_url
from evals.config import EVALS_DIR
@@ -170,3 +170,90 @@ async def get_models():
available_stacks = list(Stack.__args__)
return {"models": current_models, "stacks": available_stacks}
+
+
+class BestOfNEvalsResponse(BaseModel):
+ evals: list[Eval]
+ folder_names: list[str]
+
+
[email protected]("/best-of-n-evals", response_model=BestOfNEvalsResponse)
+async def get_best_of_n_evals(request: Request):
+ # Get all query parameters
+ query_params = dict(request.query_params)
+
+ # Extract all folder paths (folder1, folder2, folder3, etc.)
+ folders = []
+ i = 1
+ while f"folder{i}" in query_params:
+ folders.append(query_params[f"folder{i}"])
+ i += 1
+
+ if not folders:
+ return {"error": "No folders provided"}
+
+ # Validate folders exist
+ for folder in folders:
+ if not os.path.exists(folder):
+ return {"error": f"Folder does not exist: {folder}"}
+
+ evals: list[Eval] = []
+ folder_names = [os.path.basename(folder) for folder in folders]
+
+ # Get HTML files from all folders
+ files_by_folder = []
+ for folder in folders:
+ files = {
+ f: os.path.join(folder, f)
+ for f in os.listdir(folder)
+ if f.endswith(".html")
+ }
+ files_by_folder.append(files)
+
+ # Find common base names across all folders
+ common_names: Set[str] = set()
+ base_names_first_folder = {
+ f.rsplit("_", 1)[0] if "_" in f else f.replace(".html", "")
+ for f in files_by_folder[0].keys()
+ }
+
+ for base_name in base_names_first_folder:
+ found_in_all = True
+ for folder_files in files_by_folder[1:]:
+ if not any(f.startswith(base_name) for f in folder_files.keys()):
+ found_in_all = False
+ break
+ if found_in_all:
+ common_names.add(base_name)
+
+ # For each matching set, create an eval
+ for base_name in common_names:
+ # Find the corresponding input image
+ input_image = None
+ input_path = os.path.join(EVALS_DIR, "inputs", f"{base_name}.png")
+ if os.path.exists(input_path):
+ input_image = await image_to_data_url(input_path)
+ else:
+ input_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
+
+ # Get HTML contents from all folders
+ outputs = []
+ for folder_files in files_by_folder:
+ output_content = None
+ for filename in folder_files.keys():
+ if filename.startswith(base_name):
+ with open(folder_files[filename], "r") as f:
+ output_content = f.read()
+ break
+ if output_content:
+ outputs.append(output_content)
+ else:
+ outputs.append("<html><body>Output not found</body></html>")
+
+ if len(outputs) == len(folders): # Only add if we have outputs from all folders
+ evals.append(Eval(input=input_image, outputs=outputs))
+
+ return BestOfNEvalsResponse(
+ evals=evals,
+ folder_names=folder_names
+ )
diff --git a/frontend/src/components/evals/BestOfNEvalsPage.tsx b/frontend/src/components/evals/BestOfNEvalsPage.tsx
index ed841812..4211c54f 100644
--- a/frontend/src/components/evals/BestOfNEvalsPage.tsx
+++ b/frontend/src/components/evals/BestOfNEvalsPage.tsx
@@ -7,74 +7,76 @@ interface Eval {
outputs: string[];
}
-type Outcome = "left" | "right" | "tie" | null;
+// Update type to support any folder as winner
+type Outcome = number | "tie" | null;
-interface PairwiseEvalsResponse {
+interface BestOfNEvalsResponse {
evals: Eval[];
- folder1_name: string;
- folder2_name: string;
+ folder_names: string[];
}
-function PairwiseEvalsPage() {
+function BestOfNEvalsPage() {
const [evals, setEvals] = React.useState<Eval[]>([]);
const [outcomes, setOutcomes] = React.useState<Outcome[]>([]);
- const [folderNames, setFolderNames] = useState<{
- left: string;
- right: string;
- }>({
- left: "",
- right: "",
- });
- const [folder1Path, setFolder1Path] = useState("");
- const [folder2Path, setFolder2Path] = useState("");
+ const [folderNames, setFolderNames] = useState<string[]>([]);
+ // Track multiple folder paths
+ const [folderPaths, setFolderPaths] = useState<string[]>(['']);
const [isLoading, setIsLoading] = useState(false);
const [selectedHtml, setSelectedHtml] = useState<string>("");
- // Calculate statistics
- const totalVotes = outcomes.filter((o) => o !== null).length;
- const leftWins = outcomes.filter((o) => o === "left").length;
- const rightWins = outcomes.filter((o) => o === "right").length;
- const ties = outcomes.filter((o) => o === "tie").length;
- // Calculate percentages
- const leftPercentage = totalVotes
- ? ((leftWins / totalVotes) * 100).toFixed(2)
- : "0.00";
- const rightPercentage = totalVotes
- ? ((rightWins / totalVotes) * 100).toFixed(2)
- : "0.00";
- const tiePercentage = totalVotes
- ? ((ties / totalVotes) * 100).toFixed(2)
- : "0.00";
+ // Add/remove folder input fields
+ const addFolderInput = () => {
+ setFolderPaths([...folderPaths, '']);
+ };
+
+ const removeFolderInput = (index: number) => {
+ setFolderPaths(folderPaths.filter((_, i) => i !== index));
+ };
+
+ const updateFolderPath = (index: number, value: string) => {
+ const newPaths = [...folderPaths];
+ newPaths[index] = value;
+ setFolderPaths(newPaths);
+ };
+
+ // Calculate statistics for N models
+ const calculateStats = () => {
+ const totalVotes = outcomes.filter((o) => o !== null).length;
+ const stats = folderNames.map((name, index) => {
+ const wins = outcomes.filter((o) => o === index).length;
+ const percentage = totalVotes ? ((wins / totalVotes) * 100).toFixed(2) : "0.00";
+ return { name, wins, percentage };
+ });
+ const ties = outcomes.filter((o) => o === "tie").length;
+ const tiePercentage = totalVotes ? ((ties / totalVotes) * 100).toFixed(2) : "0.00";
+
+ return { stats, ties, tiePercentage, totalVotes };
+ };
const loadEvals = async () => {
- if (!folder1Path || !folder2Path) {
- alert("Please enter both folder paths");
+ if (folderPaths.some(path => !path)) {
+ alert("Please enter all folder paths");
return;
}
setIsLoading(true);
try {
- const queryParams = new URLSearchParams({
- folder1: `/Users/abi/Downloads/${folder1Path}`,
- folder2: `/Users/abi/Downloads/${folder2Path}`,
+ const queryParams = new URLSearchParams();
+ folderPaths.forEach((path, index) => {
+ queryParams.append(`folder${index + 1}`, `/Users/abi/Downloads/${path}`);
});
const response = await fetch(
- `${HTTP_BACKEND_URL}/pairwise-evals?${queryParams}`
+ `${HTTP_BACKEND_URL}/best-of-n-evals?${queryParams}`
);
- const data: PairwiseEvalsResponse = await response.json();
+ const data: BestOfNEvalsResponse = await response.json();
setEvals(data.evals);
setOutcomes(new Array(data.evals.length).fill(null));
- setFolderNames({
- left: data.folder1_name,
- right: data.folder2_name,
- });
+ setFolderNames(data.folder_names);
} catch (error) {
console.error("Error loading evals:", error);
- alert(
- "Error loading evals. Please check the folder paths and try again."
- );
+ alert("Error loading evals. Please check the folder paths and try again.");
} finally {
setIsLoading(false);
}
@@ -86,24 +88,39 @@ function PairwiseEvalsPage() {
setOutcomes(newOutcomes);
};
+ const stats = calculateStats();
+
return (
<div className="mx-auto">
<div className="flex flex-col items-center justify-center w-full py-4 bg-zinc-950 text-white">
<div className="flex flex-col gap-4 mb-4 w-full max-w-2xl px-4">
- <input
- type="text"
- value={folder1Path}
- onChange={(e) => setFolder1Path(e.target.value)}
- placeholder="Enter folder name in Downloads"
- className="w-full px-4 py-2 rounded text-black"
- />
- <input
- type="text"
- value={folder2Path}
- onChange={(e) => setFolder2Path(e.target.value)}
- placeholder="Enter folder name in Downloads"
- className="w-full px-4 py-2 rounded text-black"
- />
+ {folderPaths.map((path, index) => (
+ <div key={index} className="flex gap-2">
+ <input
+ type="text"
+ value={path}
+ onChange={(e) => updateFolderPath(index, e.target.value)}
+ placeholder="Enter folder name in Downloads"
+ className="w-full px-4 py-2 rounded text-black"
+ />
+ {index > 0 && (
+ <button
+ onClick={() => removeFolderInput(index)}
+ className="bg-red-500 px-3 py-2 rounded"
+ >
+ ✕
+ </button>
+ )}
+ </div>
+ ))}
+
+ <button
+ onClick={addFolderInput}
+ className="bg-green-500 hover:bg-green-600 text-white px-4 py-2 rounded"
+ >
+ Add Model
+ </button>
+
<button
onClick={loadEvals}
disabled={isLoading}
@@ -116,17 +133,16 @@ function PairwiseEvalsPage() {
{evals.length > 0 && (
<>
<span className="text-2xl font-semibold">
- Total votes: {totalVotes}
+ Total votes: {stats.totalVotes}
</span>
<div className="text-lg mt-2">
+ {stats.stats.map((stat, i) => (
+ <span key={i}>
+ {stat.name}: {stat.wins} ({stat.percentage}%) |{" "}
+ </span>
+ ))}
<span>
- {folderNames.left}: {leftWins} ({leftPercentage}%) |{" "}
- </span>
- <span>
- {folderNames.right}: {rightWins} ({rightPercentage}%) |{" "}
- </span>
- <span>
- Ties: {ties} ({tiePercentage}%)
+ Ties: {stats.ties} ({stats.tiePercentage}%)
</span>
</div>
</>
@@ -147,10 +163,10 @@ function PairwiseEvalsPage() {
</div>
<div className="flex gap-x-4 justify-center">
- {e.outputs.slice(0, 2).map((output, outputIndex) => (
+ {e.outputs.map((output, outputIndex) => (
<div
className={`w-1/2 p-1 border ${
- outcomes[index] === (outputIndex === 0 ? "left" : "right")
+ outcomes[index] === outputIndex
? "border-green-500 border-4"
: ""
}`}
@@ -184,16 +200,19 @@ function PairwiseEvalsPage() {
</div>
<div className="flex justify-center gap-x-4 mt-4">
- <button
- className={`px-4 py-2 rounded ${
- outcomes[index] === "left"
- ? "bg-green-500 text-white"
- : "bg-gray-200 hover:bg-gray-300"
- }`}
- onClick={() => handleVote(index, "left")}
- >
- Left Wins
- </button>
+ {folderNames.map((name, i) => (
+ <button
+ key={i}
+ className={`px-4 py-2 rounded ${
+ outcomes[index] === i
+ ? "bg-green-500 text-white"
+ : "bg-gray-200 hover:bg-gray-300"
+ }`}
+ onClick={() => handleVote(index, i)}
+ >
+ {name} Wins
+ </button>
+ ))}
<button
className={`px-4 py-2 rounded ${
outcomes[index] === "tie"
@@ -204,16 +223,6 @@ function PairwiseEvalsPage() {
>
Tie
</button>
- <button
- className={`px-4 py-2 rounded ${
- outcomes[index] === "right"
- ? "bg-green-500 text-white"
- : "bg-gray-200 hover:bg-gray-300"
- }`}
- onClick={() => handleVote(index, "right")}
- >
- Right Wins
- </button>
</div>
</div>
))}
@@ -222,4 +231,4 @@ function PairwiseEvalsPage() {
);
}
-export default PairwiseEvalsPage;
+export default BestOfNEvalsPage;