Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Remove Pairwise and Single Model eval modes

Abi Rajadfec5e1db219
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index 0f5d4a68..8fc78646 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -102,84 +102,6 @@ async def get_evals(folder: str):
raise HTTPException(status_code=500, detail=f"Error processing evals: {str(e)}")
-class PairwiseEvalResponse(BaseModel):
- evals: list[Eval]
- folder1_name: str
- folder2_name: str
-
-
[email protected]("/pairwise-evals", response_model=PairwiseEvalResponse)
-async def get_pairwise_evals(
- folder1: str = Query(
- "...",
- description="Absolute path to first folder",
- ),
- folder2: str = Query(
- "..",
- description="Absolute path to second folder",
- ),
-):
- if not os.path.exists(folder1) or not os.path.exists(folder2):
- return {"error": "One or both folders do not exist"}
-
- evals: list[Eval] = []
-
- # Get all HTML files from first folder
- files1 = {
- f: os.path.join(folder1, f) for f in os.listdir(folder1) if f.endswith(".html")
- }
- files2 = {
- f: os.path.join(folder2, f) for f in os.listdir(folder2) if f.endswith(".html")
- }
-
- # Find common base names (ignoring any suffixes)
- common_names: Set[str] = set()
- for f1 in files1.keys():
- base_name: str = f1.rsplit("_", 1)[0] if "_" in f1 else f1.replace(".html", "")
- for f2 in files2.keys():
- if f2.startswith(base_name):
- common_names.add(base_name)
-
- # For each matching pair, create an eval
- for base_name in common_names:
- # Find the corresponding input image
- input_image = None
- input_path = os.path.join(EVALS_DIR, "inputs", f"{base_name}.png")
- if os.path.exists(input_path):
- input_image = await image_to_data_url(input_path)
- else:
- input_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" # 1x1 transparent PNG
-
- # Get the HTML contents
- output1 = None
- output2 = None
-
- # Find matching files in folder1
- for f1 in files1.keys():
- if f1.startswith(base_name):
- with open(files1[f1], "r") as f:
- output1 = f.read()
- break
-
- # Find matching files in folder2
- for f2 in files2.keys():
- if f2.startswith(base_name):
- with open(files2[f2], "r") as f:
- output2 = f.read()
- break
-
- if output1 and output2:
- evals.append(Eval(input=input_image, outputs=[output1, output2]))
-
- # Extract folder names for the UI
- folder1_name = os.path.basename(folder1)
- folder2_name = os.path.basename(folder2)
-
- return PairwiseEvalResponse(
- evals=evals, folder1_name=folder1_name, folder2_name=folder2_name
- )
-
-
class RunEvalsRequest(BaseModel):
models: List[str]
stack: Stack
diff --git a/frontend/src/components/evals/AllEvalsPage.tsx b/frontend/src/components/evals/AllEvalsPage.tsx
index c5b83476..2ac8b4b6 100644
--- a/frontend/src/components/evals/AllEvalsPage.tsx
+++ b/frontend/src/components/evals/AllEvalsPage.tsx
@@ -26,18 +26,6 @@ function AllEvalsPage() {
</p>
</Link>
- <Link
- to="/evals/pairwise"
- className="block w-full p-4 bg-white rounded-lg shadow hover:shadow-md transition-shadow border border-gray-200"
- >
- <h2 className="text-xl font-semibold text-gray-800">
- Pairwise Comparison
- </h2>
- <p className="text-gray-600">
- Compare outputs from two different models
- </p>
- </Link>
-
<Link
to="/evals/best-of-n"
className="block w-full p-4 bg-white rounded-lg shadow hover:shadow-md transition-shadow border border-gray-200"
@@ -48,16 +36,6 @@ function AllEvalsPage() {
</p>
</Link>
- <Link
- to="/evals/single"
- className="block w-full p-4 bg-white rounded-lg shadow hover:shadow-md transition-shadow border border-gray-200"
- >
- <h2 className="text-xl font-semibold text-gray-800">
- Single Model Eval
- </h2>
- <p className="text-gray-600">Score outputs from a single model</p>
- </Link>
-
<Link
to="/evals/openai-input-compare"
className="block w-full p-4 bg-white rounded-lg shadow hover:shadow-md transition-shadow border border-gray-200"
diff --git a/frontend/src/components/evals/EvalNavigation.tsx b/frontend/src/components/evals/EvalNavigation.tsx
index 08400ff4..079fefcd 100644
--- a/frontend/src/components/evals/EvalNavigation.tsx
+++ b/frontend/src/components/evals/EvalNavigation.tsx
@@ -20,13 +20,6 @@ function EvalNavigation() {
Run
</Link>
- <Link
- to="/evals/pairwise"
- className="hover:text-blue-300 transition-colors"
- >
- Pairwise
- </Link>
-
<Link
to="/evals/best-of-n"
className="hover:text-blue-300 transition-colors"
@@ -34,13 +27,6 @@ function EvalNavigation() {
Best of N
</Link>
- <Link
- to="/evals/single"
- className="hover:text-blue-300 transition-colors"
- >
- Single
- </Link>
-
<Link
to="/evals/openai-input-compare"
className="hover:text-blue-300 transition-colors"
diff --git a/frontend/src/components/evals/EvalsPage.tsx b/frontend/src/components/evals/EvalsPage.tsx
deleted file mode 100644
index f9c71b97..00000000
--- a/frontend/src/components/evals/EvalsPage.tsx
+++ /dev/null
@@ -1,266 +0,0 @@
-import React, { useState } from "react";
-import { HTTP_BACKEND_URL } from "../../config";
-import RatingPicker from "./RatingPicker";
-import EvalNavigation from "./EvalNavigation";
-
-interface Eval {
- input: string;
- outputs: string[];
-}
-
-interface RatingCriteria {
- stackAdherence: number;
- accuracy: number;
- codeQuality: number;
- mobileResponsiveness: number;
- imageCaptionQuality: number;
-}
-
-interface OutputDisplay {
- showSource: boolean;
-}
-
-function EvalsPage() {
- const [evals, setEvals] = React.useState<Eval[]>([]);
- const [ratings, setRatings] = React.useState<RatingCriteria[]>([]);
- const [folderPath, setFolderPath] = useState("");
- const [isLoading, setIsLoading] = useState(false);
- const [outputDisplays, setOutputDisplays] = useState<OutputDisplay[]>([]);
-
- const calculateScores = () => {
- if (ratings.length === 0) {
- return {
- stackAdherence: { total: 0, max: 0, percentage: "0.00" },
- accuracy: { total: 0, max: 0, percentage: "0.00" },
- codeQuality: { total: 0, max: 0, percentage: "0.00" },
- mobileResponsiveness: { total: 0, max: 0, percentage: "0.00" },
- imageCaptionQuality: { total: 0, max: 0, percentage: "0.00" },
- };
- }
-
- const maxPerCriterion = ratings.length * 5; // max score of 5 * number of evals
-
- const totals = ratings.reduce(
- (acc, rating) => ({
- stackAdherence: acc.stackAdherence + rating.stackAdherence,
- accuracy: acc.accuracy + rating.accuracy,
- codeQuality: acc.codeQuality + rating.codeQuality,
- mobileResponsiveness:
- acc.mobileResponsiveness + rating.mobileResponsiveness,
- imageCaptionQuality:
- acc.imageCaptionQuality + rating.imageCaptionQuality,
- }),
- {
- stackAdherence: 0,
- accuracy: 0,
- codeQuality: 0,
- mobileResponsiveness: 0,
- imageCaptionQuality: 0,
- }
- );
-
- return Object.entries(totals).reduce(
- (acc, [key, total]) => ({
- ...acc,
- [key]: {
- total,
- max: maxPerCriterion,
- percentage: ((total / maxPerCriterion) * 100).toFixed(2),
- },
- }),
- {} as Record<
- keyof RatingCriteria,
- { total: number; max: number; percentage: string }
- >
- );
- };
-
- const loadEvals = async () => {
- if (!folderPath) {
- alert("Please enter a folder path");
- return;
- }
-
- setIsLoading(true);
- try {
- const queryParams = new URLSearchParams({
- folder: `/Users/abi/Downloads/${folderPath}`,
- });
-
- const response = await fetch(`${HTTP_BACKEND_URL}/evals?${queryParams}`);
- const data = await response.json();
-
- console.log(data);
-
- setEvals(data);
- setRatings(
- data.map(() => ({
- stackAdherence: 0,
- accuracy: 0,
- codeQuality: 0,
- mobileResponsiveness: 0,
- imageCaptionQuality: 0,
- }))
- );
- } catch (error) {
- console.error("Error loading evals:", error);
- alert("Error loading evals. Please check the folder path and try again.");
- } finally {
- setIsLoading(false);
- }
- };
-
- const updateRating = (
- index: number,
- criterion: keyof RatingCriteria,
- value: number
- ) => {
- const newRatings = [...ratings];
- newRatings[index] = {
- ...newRatings[index],
- [criterion]: value,
- };
- setRatings(newRatings);
- };
-
- const toggleSourceView = (evalIndex: number) => {
- const newDisplays = [...outputDisplays];
- if (!newDisplays[evalIndex]) {
- newDisplays[evalIndex] = { showSource: false };
- }
- newDisplays[evalIndex] = { showSource: !newDisplays[evalIndex].showSource };
- setOutputDisplays(newDisplays);
- };
-
- return (
- <div className="mx-auto">
- <EvalNavigation />
- <div className="flex flex-col items-center justify-center w-full py-4 bg-zinc-950 text-white">
- <div className="flex flex-col gap-4 mb-4 w-full max-w-2xl px-4">
- <input
- type="text"
- value={folderPath}
- onChange={(e) => setFolderPath(e.target.value)}
- placeholder="Enter folder name in Downloads"
- className="w-full px-4 py-2 rounded text-black"
- />
- <button
- onClick={loadEvals}
- disabled={isLoading}
- className="bg-blue-500 hover:bg-blue-600 text-white px-4 py-2 rounded disabled:bg-blue-300"
- >
- {isLoading ? "Loading..." : "Load Evals"}
- </button>
- </div>
-
- {evals.length > 0 && (
- <div className="flex flex-col items-center gap-2 text-lg">
- <h2 className="text-2xl font-semibold mb-2">Scores by Category</h2>
- {Object.entries(calculateScores()).map(([criterion, score]) => (
- <div key={criterion} className="flex gap-x-4 items-center">
- <span className="min-w-[200px] text-right capitalize">
- {criterion.replace(/([A-Z])/g, " $1").trim()}:
- </span>
- <span>
- {score.total} / {score.max} ({score.percentage}%)
- </span>
- </div>
- ))}
- </div>
- )}
- </div>
-
- <div className="flex flex-col gap-y-8 mt-4 mx-auto justify-center">
- {evals.map((e, index) => (
- <div className="flex flex-col justify-center" key={index}>
- <h2 className="font-bold text-lg ml-4">Evaluation {index + 1}</h2>
- <div className="flex gap-x-2 justify-center ml-4">
- <div className="w-1/2 p-1 border">
- <img src={e.input} alt={`Input for eval ${index}`} />
- </div>
- {e.outputs.map((output, outputIndex) => (
- <div className="w-1/2 p-1 border" key={outputIndex}>
- <div className="mb-2">
- <button
- onClick={() => toggleSourceView(index)}
- className="bg-gray-500 hover:bg-gray-600 text-white px-2 py-1 rounded text-sm"
- >
- {outputDisplays[index]?.showSource ? "Show Preview" : "Show Source"}
- </button>
- </div>
- {outputDisplays[index]?.showSource ? (
- <pre className="whitespace-pre-wrap text-sm p-2 bg-gray-100 max-h-[480px] overflow-auto">
- {output}
- </pre>
- ) : (
- <iframe
- srcDoc={output}
- className="w-[1200px] h-[800px] transform scale-[0.60]"
- style={{ transformOrigin: "top left" }}
- ></iframe>
- )}
- </div>
- ))}
- </div>
- <div className="ml-8 mt-4 space-y-2">
- <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
- <div className="flex items-center gap-x-4">
- <span className="min-w-[160px]">Stack Adherence:</span>
- <RatingPicker
- onSelect={(rating) =>
- updateRating(index, "stackAdherence", rating)
- }
- maxRating={5}
- value={ratings[index].stackAdherence}
- />
- </div>
- <div className="flex items-center gap-x-4">
- <span className="min-w-[160px]">Accuracy:</span>
- <RatingPicker
- onSelect={(rating) =>
- updateRating(index, "accuracy", rating)
- }
- maxRating={5}
- value={ratings[index].accuracy}
- />
- </div>
- <div className="flex items-center gap-x-4">
- <span className="min-w-[160px]">Code Quality:</span>
- <RatingPicker
- onSelect={(rating) =>
- updateRating(index, "codeQuality", rating)
- }
- maxRating={5}
- value={ratings[index].codeQuality}
- />
- </div>
- <div className="flex items-center gap-x-4">
- <span className="min-w-[160px]">Mobile Responsiveness:</span>
- <RatingPicker
- onSelect={(rating) =>
- updateRating(index, "mobileResponsiveness", rating)
- }
- maxRating={5}
- value={ratings[index].mobileResponsiveness}
- />
- </div>
- <div className="flex items-center gap-x-4">
- <span className="min-w-[160px]">Image Caption Quality:</span>
- <RatingPicker
- onSelect={(rating) =>
- updateRating(index, "imageCaptionQuality", rating)
- }
- maxRating={5}
- value={ratings[index].imageCaptionQuality}
- />
- </div>
- </div>
- </div>
- </div>
- ))}
- </div>
- </div>
- );
-}
-
-export default EvalsPage;
diff --git a/frontend/src/components/evals/PairwiseEvalsPage.tsx b/frontend/src/components/evals/PairwiseEvalsPage.tsx
deleted file mode 100644
index 45b63d63..00000000
--- a/frontend/src/components/evals/PairwiseEvalsPage.tsx
+++ /dev/null
@@ -1,227 +0,0 @@
-import React, { useState } from "react";
-import { HTTP_BACKEND_URL } from "../../config";
-import { Dialog, DialogContent, DialogTrigger } from "../ui/dialog";
-import EvalNavigation from "./EvalNavigation";
-
-interface Eval {
- input: string;
- outputs: string[];
-}
-
-type Outcome = "left" | "right" | "tie" | null;
-
-interface PairwiseEvalsResponse {
- evals: Eval[];
- folder1_name: string;
- folder2_name: string;
-}
-
-function PairwiseEvalsPage() {
- const [evals, setEvals] = React.useState<Eval[]>([]);
- const [outcomes, setOutcomes] = React.useState<Outcome[]>([]);
- const [folderNames, setFolderNames] = useState<{
- left: string;
- right: string;
- }>({
- left: "",
- right: "",
- });
- const [folder1Path, setFolder1Path] = useState("");
- const [folder2Path, setFolder2Path] = useState("");
- const [isLoading, setIsLoading] = useState(false);
- const [selectedHtml, setSelectedHtml] = useState<string>("");
-
- // Calculate statistics
- const totalVotes = outcomes.filter((o) => o !== null).length;
- const leftWins = outcomes.filter((o) => o === "left").length;
- const rightWins = outcomes.filter((o) => o === "right").length;
- const ties = outcomes.filter((o) => o === "tie").length;
- // Calculate percentages
- const leftPercentage = totalVotes
- ? ((leftWins / totalVotes) * 100).toFixed(2)
- : "0.00";
- const rightPercentage = totalVotes
- ? ((rightWins / totalVotes) * 100).toFixed(2)
- : "0.00";
- const tiePercentage = totalVotes
- ? ((ties / totalVotes) * 100).toFixed(2)
- : "0.00";
-
- const loadEvals = async () => {
- if (!folder1Path || !folder2Path) {
- alert("Please enter both folder paths");
- return;
- }
-
- setIsLoading(true);
- try {
- const queryParams = new URLSearchParams({
- folder1: `/Users/abi/Downloads/${folder1Path}`,
- folder2: `/Users/abi/Downloads/${folder2Path}`,
- });
-
- const response = await fetch(
- `${HTTP_BACKEND_URL}/pairwise-evals?${queryParams}`
- );
- const data: PairwiseEvalsResponse = await response.json();
-
- setEvals(data.evals);
- setOutcomes(new Array(data.evals.length).fill(null));
- setFolderNames({
- left: data.folder1_name,
- right: data.folder2_name,
- });
- } catch (error) {
- console.error("Error loading evals:", error);
- alert(
- "Error loading evals. Please check the folder paths and try again."
- );
- } finally {
- setIsLoading(false);
- }
- };
-
- const handleVote = (index: number, outcome: Outcome) => {
- const newOutcomes = [...outcomes];
- newOutcomes[index] = outcome;
- setOutcomes(newOutcomes);
- };
-
- return (
- <div className="mx-auto">
- <EvalNavigation />
- <div className="flex flex-col items-center justify-center w-full py-4 bg-zinc-950 text-white">
- <div className="flex flex-col gap-4 mb-4 w-full max-w-2xl px-4">
- <input
- type="text"
- value={folder1Path}
- onChange={(e) => setFolder1Path(e.target.value)}
- placeholder="Enter folder name in Downloads"
- className="w-full px-4 py-2 rounded text-black"
- />
- <input
- type="text"
- value={folder2Path}
- onChange={(e) => setFolder2Path(e.target.value)}
- placeholder="Enter folder name in Downloads"
- className="w-full px-4 py-2 rounded text-black"
- />
- <button
- onClick={loadEvals}
- disabled={isLoading}
- className="bg-blue-500 hover:bg-blue-600 text-white px-4 py-2 rounded disabled:bg-blue-300"
- >
- {isLoading ? "Loading..." : "Start Comparison"}
- </button>
- </div>
-
- {evals.length > 0 && (
- <>
- <span className="text-2xl font-semibold">
- Total votes: {totalVotes}
- </span>
- <div className="text-lg mt-2">
- <span>
- {folderNames.left}: {leftWins} ({leftPercentage}%) |{" "}
- </span>
- <span>
- {folderNames.right}: {rightWins} ({rightPercentage}%) |{" "}
- </span>
- <span>
- Ties: {ties} ({tiePercentage}%)
- </span>
- </div>
- </>
- )}
- </div>
-
- <div className="flex flex-col gap-y-8 mt-4 mx-auto justify-center">
- {evals.map((e, index) => (
- <div className="flex flex-col justify-center" key={index}>
- <h2 className="font-bold text-lg ml-4 mb-2">
- Comparison {index + 1}
- </h2>
-
- <div className="w-full flex justify-center mb-4">
- <div className="w-1/2 p-1 border">
- <img src={e.input} alt={`Input for comparison ${index}`} />
- </div>
- </div>
-
- <div className="flex gap-x-4 justify-center">
- {e.outputs.slice(0, 2).map((output, outputIndex) => (
- <div
- className={`w-1/2 p-1 border ${
- outcomes[index] === (outputIndex === 0 ? "left" : "right")
- ? "border-green-500 border-4"
- : ""
- }`}
- key={outputIndex}
- >
- <div className="relative">
- <iframe
- srcDoc={output}
- className="w-[1200px] h-[800px] transform scale-[0.55]"
- style={{ transformOrigin: "top left" }}
- ></iframe>
- <Dialog>
- <DialogTrigger asChild>
- <button
- className="absolute top-2 right-2 bg-blue-500 text-white px-2 py-1 rounded text-sm"
- onClick={() => setSelectedHtml(output)}
- >
- Full Screen
- </button>
- </DialogTrigger>
- <DialogContent className="w-[95vw] max-w-[95vw] h-[95vh] max-h-[95vh]">
- <iframe
- srcDoc={selectedHtml}
- className="w-[1400px] h-[800px] transform scale-[0.90]"
- ></iframe>
- </DialogContent>
- </Dialog>
- </div>
- </div>
- ))}
- </div>
-
- <div className="flex justify-center gap-x-4 mt-4">
- <button
- className={`px-4 py-2 rounded ${
- outcomes[index] === "left"
- ? "bg-green-500 text-white"
- : "bg-gray-200 hover:bg-gray-300"
- }`}
- onClick={() => handleVote(index, "left")}
- >
- Left Wins
- </button>
- <button
- className={`px-4 py-2 rounded ${
- outcomes[index] === "tie"
- ? "bg-green-500 text-white"
- : "bg-gray-200 hover:bg-gray-300"
- }`}
- onClick={() => handleVote(index, "tie")}
- >
- Tie
- </button>
- <button
- className={`px-4 py-2 rounded ${
- outcomes[index] === "right"
- ? "bg-green-500 text-white"
- : "bg-gray-200 hover:bg-gray-300"
- }`}
- onClick={() => handleVote(index, "right")}
- >
- Right Wins
- </button>
- </div>
- </div>
- ))}
- </div>
- </div>
- );
-}
-
-export default PairwiseEvalsPage;
diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx
index a30a6d38..4709025b 100644
--- a/frontend/src/main.tsx
+++ b/frontend/src/main.tsx
@@ -3,9 +3,7 @@ import ReactDOM from "react-dom/client";
import App from "./App.tsx";
import "./index.css";
import { Toaster } from "react-hot-toast";
-import EvalsPage from "./components/evals/EvalsPage.tsx";
import { BrowserRouter as Router, Routes, Route } from "react-router-dom";
-import PairwiseEvalsPage from "./components/evals/PairwiseEvalsPage";
import RunEvalsPage from "./components/evals/RunEvalsPage.tsx";
import BestOfNEvalsPage from "./components/evals/BestOfNEvalsPage.tsx";
import AllEvalsPage from "./components/evals/AllEvalsPage.tsx";
@@ -18,8 +16,6 @@ ReactDOM.createRoot(document.getElementById("root")!).render(
<Routes>
<Route path="/" element={<App />} />
<Route path="/evals" element={<AllEvalsPage />} />
- <Route path="/evals/single" element={<EvalsPage />} />
- <Route path="/evals/pairwise" element={<PairwiseEvalsPage />} />
<Route path="/evals/best-of-n" element={<BestOfNEvalsPage />} />
<Route path="/evals/run" element={<RunEvalsPage />} />
<Route