Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Expand image eval runner for flux and z-image turbo

Abi Rajaff2adbd20025
diff --git a/backend/run_image_generation_evals.py b/backend/run_image_generation_evals.py
index 42161a3e..a665fdf1 100644
--- a/backend/run_image_generation_evals.py
+++ b/backend/run_image_generation_evals.py
@@ -1,84 +1,305 @@
+import argparse
import asyncio
+import json
import os
-from typing import List, Optional, Literal
-from dotenv import load_dotenv
+import re
+import time
+from pathlib import Path
+from typing import Any, Literal, TypedDict, cast
+from urllib.parse import urlparse
+
import aiohttp
-from image_generation.generation import process_tasks
-
-EVALS = [
- "Romantic Background",
- "Company logo: A stylized green sprout emerging from a circle",
- "Placeholder image of a PDF cover with abstract design",
- "A complex bubble diagram showing various interconnected features and aspects of FestivalPro, with a large central bubble surrounded by smaller bubbles of different colors representing different categories and functionalities",
- "A vibrant, abstract visualization of the RhythmRise experience ecosystem, featuring interconnected neon elements representing music, technology, and human connection",
- "Banner with text 'LiblibAI学院 课程入口'",
- "Profile picture of Pierre-Louis Labonne",
- "Two hands holding iPhone 14 models with colorful displays",
- "Portrait of a woman with long dark hair smiling at the camera",
- "Threadless logo on a gradient background from light pink to coral",
- "Jordan Schlansky Shows Conan His Favorite Nose Hair Trimmer",
- "Team Coco",
- "Intro to Large Language Models",
- "Andrej Karpathy",
- "He built a $200 million toy company",
- "CNBC International",
- "What will happen in year three of the war?",
- "Channel",
- "This is it",
- "How ASML Dominates Chip Machines",
-]
-
-# Load environment variables
-load_dotenv()
-
-# Get API keys from environment variables
-OPENAI_API_KEY: Optional[str] = os.getenv("OPENAI_API_KEY")
-REPLICATE_API_TOKEN: Optional[str] = os.getenv("REPLICATE_API_TOKEN")
-
-# Directory to save generated images
-OUTPUT_DIR: str = "generated_images"
-
-
-async def generate_and_save_images(
- prompts: List[str],
- model: Literal["dalle3", "flux"],
- api_key: Optional[str],
-) -> None:
- # Ensure the output directory exists
- os.makedirs(OUTPUT_DIR, exist_ok=True)
-
- if api_key is None:
- raise ValueError(f"API key for {model} is not set in the environment variables")
-
- # Generate images
- results: List[Optional[str]] = await process_tasks(
- prompts, api_key, None, model=model
+from dotenv import load_dotenv
+from typing_extensions import NotRequired
+
+from image_generation.replicate import (
+ FLUX_2_KLEIN_MODEL_PATH,
+ Z_IMAGE_TURBO_MODEL_PATH,
+ call_replicate_model,
+)
+
+
+ReplicateEvalModel = Literal["flux_2_klein", "z_image_turbo"]
+
+
+class PromptItem(TypedDict):
+ id: str
+ category: str
+ prompt: str
+
+
+class EvalResult(TypedDict):
+ id: str
+ category: str
+ prompt: str
+ status: str
+ total_seconds: NotRequired[float]
+ source_url: NotRequired[str]
+ file: NotRequired[str]
+ generation_seconds: NotRequired[float]
+ download_seconds: NotRequired[float]
+ bytes: NotRequired[int]
+ error: NotRequired[str]
+
+
+MODEL_PATHS: dict[ReplicateEvalModel, str] = {
+ "flux_2_klein": FLUX_2_KLEIN_MODEL_PATH,
+ "z_image_turbo": Z_IMAGE_TURBO_MODEL_PATH,
+}
+
+DEFAULT_PROMPT_FILE = Path("image_generation/eval_sets/recent_assets_20_prompts.json")
+DEFAULT_OUTPUT_ROOT = Path("image_generation/eval_results")
+
+
+def slugify(text: str) -> str:
+ text = text.lower()
+ text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
+ return text[:60] or "prompt"
+
+
+def load_prompts(prompt_file: Path) -> list[PromptItem]:
+ raw: Any = json.loads(prompt_file.read_text())
+ if not isinstance(raw, list):
+ raise ValueError("Prompt file must contain a JSON array.")
+
+ prompts: list[PromptItem] = []
+ for index, item in enumerate(cast(list[Any], raw), start=1):
+ if isinstance(item, str):
+ prompts.append(
+ {
+ "id": f"{index:03d}",
+ "category": "Prompt",
+ "prompt": item,
+ }
+ )
+ continue
+
+ if not isinstance(item, dict):
+ raise ValueError(f"Prompt item {index} must be a string or object.")
+
+ item_dict = cast(dict[str, Any], item)
+ prompt = item_dict.get("prompt")
+ if not isinstance(prompt, str) or not prompt.strip():
+ raise ValueError(f"Prompt item {index} is missing a prompt string.")
+
+ prompt_id = item_dict.get("id")
+ category = item_dict.get("category")
+ prompts.append(
+ {
+ "id": prompt_id if isinstance(prompt_id, str) else f"{index:03d}",
+ "category": category if isinstance(category, str) else "Prompt",
+ "prompt": prompt,
+ }
+ )
+
+ return prompts
+
+
+def build_replicate_input(
+ model: ReplicateEvalModel, prompt: str
+) -> dict[str, str | int | float | bool]:
+ if model == "flux_2_klein":
+ return {
+ "prompt": prompt,
+ "aspect_ratio": "1:1",
+ "output_format": "png",
+ }
+
+ return {
+ "prompt": prompt,
+ "width": 1024,
+ "height": 1024,
+ "go_fast": False,
+ "output_format": "png",
+ "guidance_scale": 0,
+ "num_inference_steps": 8,
+ }
+
+
+def extract_output_url(result: Any, model: ReplicateEvalModel) -> str:
+ if isinstance(result, str):
+ return result
+
+ if isinstance(result, dict):
+ url = cast(dict[str, Any], result).get("url")
+ if isinstance(url, str) and url:
+ return url
+
+ if isinstance(result, list) and result:
+ first = cast(list[Any], result)[0]
+ if isinstance(first, str) and first:
+ return first
+ if isinstance(first, dict):
+ url = cast(dict[str, Any], first).get("url")
+ if isinstance(url, str) and url:
+ return url
+
+ raise ValueError(f"Unexpected response from {model}: {result}")
+
+
+async def generate_image(
+ model: ReplicateEvalModel, prompt: str, api_key: str
+) -> str:
+ result = await call_replicate_model(
+ MODEL_PATHS[model],
+ build_replicate_input(model, prompt),
+ api_key,
)
+ return extract_output_url(result, model)
+
+
+async def download_image(
+ session: aiohttp.ClientSession, url: str, path: Path
+) -> int:
+ async with session.get(url) as response:
+ response.raise_for_status()
+ data = await response.read()
+ path.write_bytes(data)
+ return len(data)
+
+
+async def run_one(
+ session: aiohttp.ClientSession,
+ output_dir: Path,
+ model: ReplicateEvalModel,
+ item: PromptItem,
+ api_key: str,
+) -> EvalResult:
+ started = time.perf_counter()
+ output: EvalResult = {
+ "id": item["id"],
+ "category": item["category"],
+ "prompt": item["prompt"],
+ "status": "failed",
+ }
+
+ try:
+ image_url = await generate_image(model, item["prompt"], api_key)
+ generated_at = time.perf_counter()
+ suffix = Path(urlparse(image_url).path).suffix or ".png"
+ filename = f"{item['id']}_{slugify(item['category'])}{suffix}"
+ byte_count = await download_image(session, image_url, output_dir / filename)
+ completed = time.perf_counter()
+ generation_seconds = round(generated_at - started, 3)
+ download_seconds = round(completed - generated_at, 3)
+ total_seconds = round(completed - started, 3)
+
+ output.update(
+ {
+ "status": "ok",
+ "source_url": image_url,
+ "file": filename,
+ "generation_seconds": generation_seconds,
+ "download_seconds": download_seconds,
+ "total_seconds": total_seconds,
+ "bytes": byte_count,
+ }
+ )
+ print(
+ f"ok {item['id']} {model} generation={generation_seconds}s "
+ f"total={total_seconds}s {filename}",
+ flush=True,
+ )
+ except Exception as exc:
+ completed = time.perf_counter()
+ total_seconds = round(completed - started, 3)
+ output.update(
+ {
+ "error": str(exc),
+ "total_seconds": total_seconds,
+ }
+ )
+ print(
+ f"failed {item['id']} {model} total={total_seconds}s error={exc}",
+ flush=True,
+ )
+
+ return output
- # Save images to disk
+
+async def run_model(
+ model: ReplicateEvalModel,
+ prompts: list[PromptItem],
+ api_key: str,
+ output_root: Path,
+) -> Path:
+ run_id = time.strftime("%Y%m%d_%H%M%S")
+ output_dir = output_root / f"{run_id}_{model}"
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ started = time.perf_counter()
async with aiohttp.ClientSession() as session:
- for i, image_url in enumerate(results):
- if image_url:
- # Get the image data
- async with session.get(image_url) as response:
- image_data: bytes = await response.read()
-
- # Save the image with a filename based on the input eval
- prefix = "replicate_" if model == "flux" else "dalle3_"
- filename: str = (
- f"{prefix}{prompts[i][:50].replace(' ', '_').replace(':', '')}.png"
- )
- filepath: str = os.path.join(OUTPUT_DIR, filename)
- with open(filepath, "wb") as f:
- f.write(image_data)
- print(f"Saved {model} image: {filepath}")
- else:
- print(f"Failed to generate {model} image for prompt: {prompts[i]}")
+ results = await asyncio.gather(
+ *(run_one(session, output_dir, model, item, api_key) for item in prompts)
+ )
+ completed = time.perf_counter()
+
+ success_count = sum(1 for result in results if result["status"] == "ok")
+ manifest: dict[str, Any] = {
+ "run_id": run_id,
+ "model_label": model,
+ "model_path": MODEL_PATHS[model],
+ "output_dir": str(output_dir),
+ "total_wall_seconds": round(completed - started, 3),
+ "count": len(results),
+ "success_count": success_count,
+ "failure_count": len(results) - success_count,
+ "results": results,
+ }
+
+ (output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))
+ (output_dir / "timings.tsv").write_text(
+ "id\tcategory\tstatus\tgeneration_seconds\tdownload_seconds\t"
+ "total_seconds\tfile\tprompt\n"
+ + "".join(
+ f"{result['id']}\t{result['category']}\t{result['status']}\t"
+ f"{result.get('generation_seconds', '')}\t"
+ f"{result.get('download_seconds', '')}\t"
+ f"{result.get('total_seconds', '')}\t"
+ f"{result.get('file', '')}\t"
+ f"{result['prompt'].replace(chr(9), ' ')}\n"
+ for result in results
+ )
+ )
+
+ print(
+ f"manifest={output_dir / 'manifest.json'}\n"
+ f"total_wall_seconds={manifest['total_wall_seconds']} "
+ f"success={manifest['success_count']} failure={manifest['failure_count']}"
+ )
+ return output_dir
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Run Replicate image generation evals.")
+ parser.add_argument("--prompt-file", type=Path, default=DEFAULT_PROMPT_FILE)
+ parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT_ROOT)
+ parser.add_argument(
+ "--model",
+ choices=["flux_2_klein", "z_image_turbo", "both"],
+ default="z_image_turbo",
+ )
+ return parser.parse_args()
async def main() -> None:
- # await generate_and_save_images(EVALS, "dalle3", OPENAI_API_KEY)
- await generate_and_save_images(EVALS, "flux", REPLICATE_API_TOKEN)
+ load_dotenv(Path(".env"))
+ args = parse_args()
+ api_key = os.environ.get("REPLICATE_API_KEY") or os.environ.get(
+ "REPLICATE_API_TOKEN"
+ )
+ if not api_key:
+ raise SystemExit("Missing REPLICATE_API_KEY or REPLICATE_API_TOKEN")
+
+ prompts = load_prompts(args.prompt_file)
+ models: list[ReplicateEvalModel]
+ if args.model == "both":
+ models = ["flux_2_klein", "z_image_turbo"]
+ else:
+ models = [args.model]
+
+ for model in models:
+ await run_model(model, prompts, api_key, args.output_root)
if __name__ == "__main__":