Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

create a UI to do pairwise evals easily

Abi Raja0d58fdff018d
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index 22262cdb..ac0ab31a 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -1,8 +1,9 @@
import os
-from fastapi import APIRouter
+from fastapi import APIRouter, Query
from pydantic import BaseModel
from evals.utils import image_to_data_url
from evals.config import EVALS_DIR
+from typing import Set
router = APIRouter()
@@ -53,3 +54,81 @@ async def get_evals():
)
return evals
+
+
+class PairwiseEvalResponse(BaseModel):
+ evals: list[Eval]
+ folder1_name: str
+ folder2_name: str
+
+
[email protected]("/pairwise-evals", response_model=PairwiseEvalResponse)
+async def get_pairwise_evals(
+ folder1: str = Query(
+ "...",
+ description="Absolute path to first folder",
+ ),
+ folder2: str = Query(
+ "..",
+ description="Absolute path to second folder",
+ ),
+):
+ if not os.path.exists(folder1) or not os.path.exists(folder2):
+ return {"error": "One or both folders do not exist"}
+
+ evals: list[Eval] = []
+
+ # Get all HTML files from first folder
+ files1 = {
+ f: os.path.join(folder1, f) for f in os.listdir(folder1) if f.endswith(".html")
+ }
+ files2 = {
+ f: os.path.join(folder2, f) for f in os.listdir(folder2) if f.endswith(".html")
+ }
+
+ # Find common base names (ignoring any suffixes)
+ common_names: Set[str] = set()
+ for f1 in files1.keys():
+ base_name: str = f1.rsplit("_", 1)[0] if "_" in f1 else f1.replace(".html", "")
+ for f2 in files2.keys():
+ if f2.startswith(base_name):
+ common_names.add(base_name)
+
+ # For each matching pair, create an eval
+ for base_name in common_names:
+ # Find the corresponding input image
+ input_image = None
+ input_path = os.path.join(EVALS_DIR, "inputs", f"{base_name}.png")
+ if os.path.exists(input_path):
+ input_image = await image_to_data_url(input_path)
+ else:
+ input_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" # 1x1 transparent PNG
+
+ # Get the HTML contents
+ output1 = None
+ output2 = None
+
+ # Find matching files in folder1
+ for f1 in files1.keys():
+ if f1.startswith(base_name):
+ with open(files1[f1], "r") as f:
+ output1 = f.read()
+ break
+
+ # Find matching files in folder2
+ for f2 in files2.keys():
+ if f2.startswith(base_name):
+ with open(files2[f2], "r") as f:
+ output2 = f.read()
+ break
+
+ if output1 and output2:
+ evals.append(Eval(input=input_image, outputs=[output1, output2]))
+
+ # Extract folder names for the UI
+ folder1_name = os.path.basename(folder1)
+ folder2_name = os.path.basename(folder2)
+
+ return PairwiseEvalResponse(
+ evals=evals, folder1_name=folder1_name, folder2_name=folder2_name
+ )
diff --git a/frontend/src/components/evals/PairwiseEvalsPage.tsx b/frontend/src/components/evals/PairwiseEvalsPage.tsx
new file mode 100644
index 00000000..cf5f6e46
--- /dev/null
+++ b/frontend/src/components/evals/PairwiseEvalsPage.tsx
@@ -0,0 +1,156 @@
+import React, { useEffect, useState } from "react";
+import { HTTP_BACKEND_URL } from "../../config";
+
+interface Eval {
+ input: string;
+ outputs: string[];
+}
+
+type Outcome = "left" | "right" | "tie" | null;
+
+interface PairwiseEvalsResponse {
+ evals: Eval[];
+ folder1_name: string;
+ folder2_name: string;
+}
+
+function PairwiseEvalsPage() {
+ const [evals, setEvals] = React.useState<Eval[]>([]);
+ const [outcomes, setOutcomes] = React.useState<Outcome[]>([]);
+ const [folderNames, setFolderNames] = useState<{
+ left: string;
+ right: string;
+ }>({
+ left: "",
+ right: "",
+ });
+
+ // Calculate statistics
+ const totalVotes = outcomes.filter((o) => o !== null).length;
+ const leftWins = outcomes.filter((o) => o === "left").length;
+ const rightWins = outcomes.filter((o) => o === "right").length;
+ const ties = outcomes.filter((o) => o === "tie").length;
+
+ // Calculate percentages
+ const leftPercentage = totalVotes
+ ? ((leftWins / totalVotes) * 100).toFixed(1)
+ : "0.0";
+ const rightPercentage = totalVotes
+ ? ((rightWins / totalVotes) * 100).toFixed(1)
+ : "0.0";
+ const tiePercentage = totalVotes
+ ? ((ties / totalVotes) * 100).toFixed(1)
+ : "0.0";
+
+ useEffect(() => {
+ if (evals.length > 0) return;
+
+ fetch(`${HTTP_BACKEND_URL}/pairwise-evals`)
+ .then((res) => res.json())
+ .then((data: PairwiseEvalsResponse) => {
+ setEvals(data.evals);
+ setOutcomes(new Array(data.evals.length).fill(null));
+ setFolderNames({
+ left: data.folder1_name,
+ right: data.folder2_name,
+ });
+ });
+ }, [evals]);
+
+ const handleVote = (index: number, outcome: Outcome) => {
+ const newOutcomes = [...outcomes];
+ newOutcomes[index] = outcome;
+ setOutcomes(newOutcomes);
+ };
+
+ return (
+ <div className="mx-auto">
+ <div className="flex flex-col items-center justify-center w-full py-4 bg-zinc-950 text-white">
+ <span className="text-2xl font-semibold">
+ Total votes: {totalVotes}
+ </span>
+ <div className="text-lg mt-2">
+ <span>
+ {folderNames.left}: {leftWins} ({leftPercentage}%) |{" "}
+ </span>
+ <span>
+ {folderNames.right}: {rightWins} ({rightPercentage}%) |{" "}
+ </span>
+ <span>
+ Ties: {ties} ({tiePercentage}%)
+ </span>
+ </div>
+ </div>
+
+ <div className="flex flex-col gap-y-8 mt-4 mx-auto justify-center">
+ {evals.map((e, index) => (
+ <div className="flex flex-col justify-center" key={index}>
+ <h2 className="font-bold text-lg ml-4 mb-2">
+ Comparison {index + 1}
+ </h2>
+
+ <div className="w-full flex justify-center mb-4">
+ <div className="w-1/2 p-1 border">
+ <img src={e.input} alt={`Input for comparison ${index}`} />
+ </div>
+ </div>
+
+ <div className="flex gap-x-4 justify-center">
+ {e.outputs.slice(0, 2).map((output, outputIndex) => (
+ <div
+ className={`w-1/2 p-1 border ${
+ outcomes[index] === (outputIndex === 0 ? "left" : "right")
+ ? "border-green-500 border-4"
+ : ""
+ }`}
+ key={outputIndex}
+ >
+ <iframe
+ srcDoc={output}
+ className="w-[1200px] h-[800px] transform scale-[0.55]"
+ style={{ transformOrigin: "top left" }}
+ ></iframe>
+ </div>
+ ))}
+ </div>
+
+ <div className="flex justify-center gap-x-4 mt-4">
+ <button
+ className={`px-4 py-2 rounded ${
+ outcomes[index] === "left"
+ ? "bg-green-500 text-white"
+ : "bg-gray-200 hover:bg-gray-300"
+ }`}
+ onClick={() => handleVote(index, "left")}
+ >
+ Left Wins
+ </button>
+ <button
+ className={`px-4 py-2 rounded ${
+ outcomes[index] === "tie"
+ ? "bg-green-500 text-white"
+ : "bg-gray-200 hover:bg-gray-300"
+ }`}
+ onClick={() => handleVote(index, "tie")}
+ >
+ Tie
+ </button>
+ <button
+ className={`px-4 py-2 rounded ${
+ outcomes[index] === "right"
+ ? "bg-green-500 text-white"
+ : "bg-gray-200 hover:bg-gray-300"
+ }`}
+ onClick={() => handleVote(index, "right")}
+ >
+ Right Wins
+ </button>
+ </div>
+ </div>
+ ))}
+ </div>
+ </div>
+ );
+}
+
+export default PairwiseEvalsPage;
diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx
index c4224f46..1da1c4ae 100644
--- a/frontend/src/main.tsx
+++ b/frontend/src/main.tsx
@@ -5,6 +5,7 @@ import "./index.css";
import { Toaster } from "react-hot-toast";
import EvalsPage from "./components/evals/EvalsPage.tsx";
import { BrowserRouter as Router, Routes, Route } from "react-router-dom";
+import PairwiseEvalsPage from "./components/evals/PairwiseEvalsPage";
ReactDOM.createRoot(document.getElementById("root")!).render(
<React.StrictMode>
@@ -12,6 +13,7 @@ ReactDOM.createRoot(document.getElementById("root")!).render(
<Routes>
<Route path="/" element={<App />} />
<Route path="/evals" element={<EvalsPage />} />
+ <Route path="/pairwise-evals" element={<PairwiseEvalsPage />} />
</Routes>
</Router>
<Toaster toastOptions={{ className: "dark:bg-zinc-950 dark:text-white" }} />