Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Update runner.py

Abi Raja57331fd4de9f
diff --git a/backend/evals/runner.py b/backend/evals/runner.py
index d321e1db..8c9d685d 100644
--- a/backend/evals/runner.py
+++ b/backend/evals/runner.py
@@ -1,4 +1,4 @@
-from typing import Any, Coroutine, List, Optional
+from typing import Any, Coroutine, List, Optional, Tuple
import asyncio
import os
from datetime import datetime
@@ -11,15 +11,32 @@ from .config import EVALS_DIR
async def generate_code_and_time(
- image_url: str, stack: Stack, model: Llm
-) -> tuple[str, float]:
- """Generates code for an image and measures the time taken."""
+ image_url: str,
+ stack: Stack,
+ model: Llm,
+ original_input_filename: str,
+ attempt_idx: int,
+) -> Tuple[str, int, Optional[str], Optional[float], Optional[Exception]]:
+ """
+ Generates code for an image, measures the time taken, and returns identifiers
+ along with success/failure status.
+ Returns a tuple: (original_input_filename, attempt_idx, content, duration, error_object)
+ content and duration are None if an error occurs during generation.
+ """
start_time = time.perf_counter()
- content = await generate_code_for_image(
- image_url=image_url, stack=stack, model=model
- )
- end_time = time.perf_counter()
- return content, end_time - start_time
+ try:
+ content = await generate_code_for_image(
+ image_url=image_url, stack=stack, model=model
+ )
+ end_time = time.perf_counter()
+ duration = end_time - start_time
+ return original_input_filename, attempt_idx, content, duration, None
+ except Exception as e:
+ # Log the error specific to this task attempt
+ print(
+ f"Error during code generation for {original_input_filename} (attempt {attempt_idx}): {e}"
+ )
+ return original_input_filename, attempt_idx, None, None, e
async def run_image_evals(
@@ -28,74 +45,123 @@ async def run_image_evals(
INPUT_DIR = EVALS_DIR + "/inputs"
OUTPUT_DIR = EVALS_DIR + "/outputs"
- # Get all the files in the directory (only grab pngs)
evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")]
if not stack:
raise ValueError("No stack was provided")
-
- print("User selected stack:", stack)
- print("User selected model:", model)
-
- # If model is provided as string, convert it to Llm enum
if not model:
raise ValueError("No model was provided")
+ print("User selected stack:", stack)
+ print("User selected model:", model)
selected_model = Llm(model)
print(f"Running evals for {selected_model} model")
- # Create output subfolder with date, model and stack
today = datetime.now().strftime("%b_%d_%Y")
output_subfolder = os.path.join(
OUTPUT_DIR, f"{today}_{selected_model.value}_{stack}"
)
os.makedirs(output_subfolder, exist_ok=True)
- tasks: list[Coroutine[Any, Any, tuple[str, float]]] = []
- for filename in evals:
- filepath = os.path.join(INPUT_DIR, filename)
+ task_coroutines: List[
+ Coroutine[
+ Any,
+ Any,
+ Tuple[str, int, Optional[str], Optional[float], Optional[Exception]],
+ ]
+ ] = []
+ for original_filename in evals: # Renamed for clarity
+ filepath = os.path.join(INPUT_DIR, original_filename)
data_url = await image_to_data_url(filepath)
- for n_idx in range(n): # Generate N tasks for each input
- current_model = selected_model if n_idx == 0 else Llm.GPT_4O_2024_05_13
- task = generate_code_and_time(
+ for n_idx in range(n):
+ current_model_for_task = (
+ selected_model if n_idx == 0 else Llm.GPT_4O_2024_05_13
+ )
+ coro = generate_code_and_time(
image_url=data_url,
stack=stack,
- model=current_model,
+ model=current_model_for_task,
+ original_input_filename=original_filename,
+ attempt_idx=n_idx,
)
- tasks.append(task)
-
- print(f"Generating {len(tasks)} codes")
+ task_coroutines.append(coro)
- # Results will now be a list of tuples (content, duration)
- results_with_times = await asyncio.gather(*tasks)
+ print(f"Processing {len(task_coroutines)} tasks...")
output_files: List[str] = []
timing_data: List[str] = []
+ failed_tasks_log: List[str] = []
- for i, (content, duration) in enumerate(results_with_times):
- # Calculate index for filename and output number
- eval_index = i // n
- output_number = i % n
- filename = evals[eval_index]
- # File name is derived from the original filename in evals with an added output number
- output_filename = f"{os.path.splitext(filename)[0]}_{output_number}.html"
- output_filepath = os.path.join(output_subfolder, output_filename)
+ for future in asyncio.as_completed(task_coroutines):
try:
- with open(output_filepath, "w") as file:
- file.write(content)
- timing_data.append(f"{output_filename}: {duration:.2f} seconds")
+ # Unpack results from generate_code_and_time
+ task_orig_fn, task_attempt_idx, generated_content, time_taken, error_obj = (
+ await future
+ )
+
+ output_html_filename_base = os.path.splitext(task_orig_fn)[0]
+ final_output_html_filename = (
+ f"{output_html_filename_base}_{task_attempt_idx}.html"
+ )
+ output_html_filepath = os.path.join(
+ output_subfolder, final_output_html_filename
+ )
+
+ if error_obj is not None:
+ # Generation itself failed
+ failed_tasks_log.append(
+ f"Input: {task_orig_fn}, Attempt: {task_attempt_idx}, OutputFile: {final_output_html_filename}, Error: Generation failed - {str(error_obj)}"
+ )
+ elif generated_content is not None and time_taken is not None:
+ # Generation succeeded, try to write file
+ try:
+ with open(output_html_filepath, "w") as file:
+ file.write(generated_content)
+ timing_data.append(
+ f"{final_output_html_filename}: {time_taken:.2f} seconds"
+ )
+ output_files.append(final_output_html_filename)
+ print(
+ f"Successfully processed and wrote {final_output_html_filename}"
+ )
+ except Exception as e_write:
+ # Failure during writing the successful result
+ failed_tasks_log.append(
+ f"Input: {task_orig_fn}, Attempt: {task_attempt_idx}, OutputFile: {final_output_html_filename}, Error: Writing to file failed - {str(e_write)}"
+ )
+ else:
+ # Should not happen if error_obj is None, but as a safeguard
+ failed_tasks_log.append(
+ f"Input: {task_orig_fn}, Attempt: {task_attempt_idx}, OutputFile: {final_output_html_filename}, Error: Unknown issue - content or time_taken is None without explicit error."
+ )
+
+ except Exception as e_as_completed:
+ # This catches errors if the future from as_completed fails unexpectedly
+ # (e.g., task cancelled, or an error not caught within generate_code_and_time)
+ # We don't have task_orig_fn or task_attempt_idx here, so logging is more general.
+ print(f"A task in as_completed failed unexpectedly: {e_as_completed}")
+ failed_tasks_log.append(
+ f"Critical Error: A task processing failed - {str(e_as_completed)}"
+ )
+
+ # Write timing data for successful tasks
+ if timing_data:
+ timing_file_path = os.path.join(output_subfolder, "generation_times.txt")
+ try:
+ with open(timing_file_path, "w") as file:
+ file.write("\n".join(timing_data))
+ print(f"Timing data saved to {timing_file_path}")
except Exception as e:
- print(f"Error writing file {output_filepath}: {content} {e}")
- continue
- output_files.append(output_filename)
+ print(f"Error writing timing file {timing_file_path}: {e}")
- # Write timing data to a file
- timing_file_path = os.path.join(output_subfolder, "generation_times.txt")
- try:
- with open(timing_file_path, "w") as file:
- file.write("\n".join(timing_data))
- print(f"Timing data saved to {timing_file_path}")
- except Exception as e:
- print(f"Error writing timing file {timing_file_path}: {e}")
+ # Write log for failed tasks
+ if failed_tasks_log:
+ failed_log_path = os.path.join(output_subfolder, "failed_tasks.txt")
+ try:
+ with open(failed_log_path, "w") as file:
+ file.write("\n".join(failed_tasks_log))
+ print(f"Failed tasks log saved to {failed_log_path}")
+ except Exception as e:
+ print(f"Error writing failed tasks log {failed_log_path}: {e}")
return output_files