Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

run evals from the evals page

Abi Raja3fee079b57d6
diff --git a/backend/evals/runner.py b/backend/evals/runner.py
new file mode 100644
index 00000000..178cc2bd
--- /dev/null
+++ b/backend/evals/runner.py
@@ -0,0 +1,59 @@
+from typing import Any, Coroutine, List
+import asyncio
+import os
+from llm import Llm
+from .core import generate_code_for_image
+from .utils import image_to_data_url
+from .config import EVALS_DIR
+from typing_extensions import Literal
+
+Stack = Literal['html_css', 'html_tailwind', 'react_tailwind', 'bootstrap', 'ionic_tailwind', 'vue_tailwind', 'svg']
+
+async def run_image_evals(
+ stack: Stack = "html_tailwind",
+ model: Llm = Llm.CLAUDE_3_5_SONNET_2024_10_22,
+ n: int = 1
+) -> List[str]:
+ INPUT_DIR = EVALS_DIR + "/inputs"
+ OUTPUT_DIR = EVALS_DIR + "/outputs"
+
+ # Get all the files in the directory (only grab pngs)
+ evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]
+
+ tasks: list[Coroutine[Any, Any, str]] = []
+ for filename in evals:
+ filepath = os.path.join(INPUT_DIR, filename)
+ data_url = await image_to_data_url(filepath)
+ for n_idx in range(n): # Generate N tasks for each input
+ if n_idx == 0:
+ task = generate_code_for_image(
+ image_url=data_url,
+ stack=stack,
+ model=model,
+ )
+ else:
+ task = generate_code_for_image(
+ image_url=data_url, stack=stack, model=Llm.GPT_4O_2024_05_13
+ )
+ tasks.append(task)
+
+ print(f"Generating {len(tasks)} codes")
+
+ results = await asyncio.gather(*tasks)
+
+ os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+ output_files: List[str] = []
+ for i, content in enumerate(results):
+ # Calculate index for filename and output number
+ eval_index = i // n
+ output_number = i % n
+ filename = evals[eval_index]
+ # File name is derived from the original filename in evals with an added output number
+ output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html"
+ output_filepath = os.path.join(OUTPUT_DIR, output_filename)
+ with open(output_filepath, "w") as file:
+ file.write(content)
+ output_files.append(output_filename)
+
+ return output_files
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index ac0ab31a..3360e5c4 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -4,6 +4,8 @@ from pydantic import BaseModel
from evals.utils import image_to_data_url
from evals.config import EVALS_DIR
from typing import Set
+from evals.runner import run_image_evals
+from typing import List
router = APIRouter()
@@ -132,3 +134,10 @@ async def get_pairwise_evals(
return PairwiseEvalResponse(
evals=evals, folder1_name=folder1_name, folder2_name=folder2_name
)
+
+
[email protected]("/run_evals", response_model=List[str])
+async def run_evals():
+ """Run evaluations on all images in the inputs directory"""
+ output_files = await run_image_evals()
+ return output_files
diff --git a/backend/run_evals.py b/backend/run_evals.py
index f98f7f5d..d637bea7 100644
--- a/backend/run_evals.py
+++ b/backend/run_evals.py
@@ -1,107 +1,15 @@
# Load environment variables first
from dotenv import load_dotenv
-from llm import Llm
-
load_dotenv()
-import os
-from typing import Any, Coroutine
import asyncio
-
-from evals.config import EVALS_DIR
-from evals.core import generate_code_for_image
-from evals.utils import image_to_data_url
-
-STACK = "html_tailwind"
-MODEL = Llm.CLAUDE_3_5_SONNET_2024_10_22
-N = 1 # Number of outputs to generate
+from evals.runner import run_image_evals
async def main():
- INPUT_DIR = EVALS_DIR + "/inputs"
- OUTPUT_DIR = EVALS_DIR + "/outputs"
-
- # Get all the files in the directory (only grab pngs)
- evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]
-
- tasks: list[Coroutine[Any, Any, str]] = []
- for filename in evals:
- filepath = os.path.join(INPUT_DIR, filename)
- data_url = await image_to_data_url(filepath)
- for n in range(N): # Generate N tasks for each input
- if n == 0:
- task = generate_code_for_image(
- image_url=data_url,
- stack=STACK,
- model=MODEL,
- )
- else:
- task = generate_code_for_image(
- image_url=data_url, stack=STACK, model=Llm.GPT_4O_2024_05_13
- )
- tasks.append(task)
-
- print(f"Generating {len(tasks)} codes")
-
- results = await asyncio.gather(*tasks)
-
- os.makedirs(OUTPUT_DIR, exist_ok=True)
-
- for i, content in enumerate(results):
- # Calculate index for filename and output number
- eval_index = i // N
- output_number = i % N
- filename = evals[eval_index]
- # File name is derived from the original filename in evals with an added output number
- output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html"
- output_filepath = os.path.join(OUTPUT_DIR, output_filename)
- with open(output_filepath, "w") as file:
- file.write(content)
-
-
-# async def text_main():
-# OUTPUT_DIR = EVALS_DIR + "/outputs"
-
-# GENERAL_TEXT_V1 = [
-# "Login form",
-# "Simple notification",
-# "button",
-# "saas dashboard",
-# "landing page for barber shop",
-# ]
-
-# tasks: list[Coroutine[Any, Any, str]] = []
-# for prompt in GENERAL_TEXT_V1:
-# for n in range(N): # Generate N tasks for each input
-# if n == 0:
-# task = generate_code_for_text(
-# text=prompt,
-# stack=STACK,
-# model=Llm.CLAUDE_3_5_SONNET_2024_06_20,
-# )
-# else:
-# task = generate_code_for_text(
-# text=prompt, stack=STACK, model=Llm.GPT_4O_2024_05_13
-# )
-# tasks.append(task)
-
-# print(f"Generating {len(tasks)} codes")
-
-# results = await asyncio.gather(*tasks)
-
-# os.makedirs(OUTPUT_DIR, exist_ok=True)
-
-# for i, content in enumerate(results):
-# # Calculate index for filename and output number
-# eval_index = i // N
-# output_number = i % N
-# filename = GENERAL_TEXT_V1[eval_index]
-# # File name is derived from the original filename in evals with an added output number
-# output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html"
-# output_filepath = os.path.join(OUTPUT_DIR, output_filename)
-# with open(output_filepath, "w") as file:
-# file.write(content)
+ await run_image_evals()
-asyncio.run(main())
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/frontend/src/components/evals/EvalsPage.tsx b/frontend/src/components/evals/EvalsPage.tsx
index 6d2adbd4..935f774c 100644
--- a/frontend/src/components/evals/EvalsPage.tsx
+++ b/frontend/src/components/evals/EvalsPage.tsx
@@ -1,6 +1,7 @@
-import React, { useEffect } from "react";
+import React, { useEffect, useState } from "react";
import { HTTP_BACKEND_URL } from "../../config";
import RatingPicker from "./RatingPicker";
+import { Button } from "../ui/button";
interface Eval {
input: string;
@@ -10,6 +11,7 @@ interface Eval {
function EvalsPage() {
const [evals, setEvals] = React.useState<Eval[]>([]);
const [ratings, setRatings] = React.useState<number[]>([]);
+ const [isRunning, setIsRunning] = useState(false);
const total = ratings.reduce((a, b) => a + b, 0);
const max = ratings.length * 4;
@@ -26,13 +28,42 @@ function EvalsPage() {
});
}, [evals]);
+ const runEvals = async () => {
+ try {
+ setIsRunning(true);
+ const response = await fetch(`${HTTP_BACKEND_URL}/run_evals`, {
+ method: "POST",
+ });
+
+ if (!response.ok) {
+ throw new Error("Failed to run evals");
+ }
+
+ const outputFiles = await response.json();
+ console.log("Generated files:", outputFiles);
+
+ // Refresh the evals list after running
+ const evalsResponse = await fetch(`${HTTP_BACKEND_URL}/evals`);
+ const newEvals = await evalsResponse.json();
+ setEvals(newEvals);
+ setRatings(new Array(newEvals.length).fill(0));
+ } catch (error) {
+ console.error("Error running evals:", error);
+ // You might want to add a toast notification here
+ } finally {
+ setIsRunning(false);
+ }
+ };
+
return (
<div className="mx-auto">
- {/* Display total */}
- <div className="flex items-center justify-center w-full h-12 bg-zinc-950">
- <span className="text-2xl font-semibold text-white">
+ <div className="flex items-center justify-between w-full h-12 bg-zinc-950">
+ <span className="text-2xl font-semibold text-white ml-4">
Total: {total} out of {max} ({score}%)
</span>
+ <Button onClick={runEvals} disabled={isRunning} className="mr-4">
+ {isRunning ? "Running Evals..." : "Run Evals"}
+ </Button>
</div>
<div className="flex flex-col gap-y-4 mt-4 mx-auto justify-center">