Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Route video generation through agentic runner

Abi Raja210e36442b1a
diff --git a/backend/models/gemini.py b/backend/models/gemini.py
index 13365735..0942766f 100644
--- a/backend/models/gemini.py
+++ b/backend/models/gemini.py
@@ -16,6 +16,7 @@ from video.cost_estimation import (
# Set to True to print debug messages for Gemini requests
DEBUG_GEMINI = False
+DEFAULT_VIDEO_FPS = 10
def get_gemini_api_model_name(model: Llm) -> str:
@@ -149,10 +150,23 @@ def convert_message_to_gemini_content(
parts.append({"text": text})
for image_data in image_data_list:
if "data" in image_data:
+ mime_type = image_data["mime_type"]
+ media_bytes = base64.b64decode(image_data["data"])
+ if mime_type.startswith("video/"):
+ # Gemini video inputs require inline video data plus metadata.
+ parts.append(
+ types.Part(
+ inline_data=types.Blob(data=media_bytes, mime_type=mime_type),
+ video_metadata=types.VideoMetadata(fps=DEFAULT_VIDEO_FPS),
+ media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_HIGH,
+ )
+ )
+ continue
+
parts.append(
types.Part.from_bytes(
- data=base64.b64decode(image_data["data"]),
- mime_type=image_data["mime_type"],
+ data=media_bytes,
+ mime_type=mime_type,
media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,
)
)
diff --git a/backend/prompts/__init__.py b/backend/prompts/__init__.py
index 3b504f9e..1eb5cc6c 100644
--- a/backend/prompts/__init__.py
+++ b/backend/prompts/__init__.py
@@ -6,6 +6,7 @@ from image_generation.core import create_alt_url_mapping
from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTS
from prompts.screenshot_system_prompts import SYSTEM_PROMPTS
from prompts.text_prompts import SYSTEM_PROMPTS as TEXT_SYSTEM_PROMPTS
+from prompts.claude_prompts import GEMINI_VIDEO_PROMPT
from prompts.types import Stack, PromptContent
@@ -35,6 +36,7 @@ async def create_prompt(
) -> tuple[list[ChatCompletionMessageParam], dict[str, str]]:
image_cache: dict[str, str] = {}
+ prompt_messages: list[ChatCompletionMessageParam] = []
# If this generation started off with imported code, we need to assemble the prompt differently
if is_imported_from_code:
@@ -53,19 +55,26 @@ async def create_prompt(
elif input_mode == "text":
prompt_messages = assemble_text_prompt(prompt["text"], stack)
elif input_mode == "video":
- # For video mode initial creation, prompt is handled by VideoGenerationStage
- # which sends the video directly to Gemini with its own prompt
if generation_type == "create":
- # Return empty prompt - actual generation is handled by VideoGenerationStage
- prompt_messages = []
+ video_urls = prompt.get("images", [])
+ if not video_urls:
+ raise ValueError("Video mode requires a video to be provided")
+ video_url = video_urls[0]
+ prompt_messages = assemble_video_prompt(
+ video_data_url=video_url,
+ text_prompt=prompt.get("text", ""),
+ )
else:
# For video mode updates, use the screenshot system prompt
# since we're now working with the generated code, not the video
prompt_messages = [
- {
- "role": "system",
- "content": SYSTEM_PROMPTS[stack],
- }
+ cast(
+ ChatCompletionMessageParam,
+ {
+ "role": "system",
+ "content": SYSTEM_PROMPTS[stack],
+ },
+ )
]
else:
# Default to image mode for backward compatibility
@@ -206,3 +215,34 @@ def assemble_text_prompt(
"content": "Generate UI for " + text_prompt,
},
]
+
+
+def assemble_video_prompt(
+ video_data_url: str,
+ text_prompt: str = "",
+) -> list[ChatCompletionMessageParam]:
+ user_text = "Analyze this video and generate the code."
+ if text_prompt.strip():
+ user_text = user_text + "\n\nAdditional instructions: " + text_prompt
+
+ user_content: list[ChatCompletionContentPartParam] = [
+ {
+ "type": "image_url",
+ "image_url": {"url": video_data_url, "detail": "high"},
+ },
+ {
+ "type": "text",
+ "text": user_text,
+ },
+ ]
+
+ return [
+ {
+ "role": "system",
+ "content": GEMINI_VIDEO_PROMPT,
+ },
+ {
+ "role": "user",
+ "content": user_content,
+ },
+ ]
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index fff62262..c212b7e1 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -20,17 +20,7 @@ from config import (
)
from custom_types import InputMode
from llm import (
- Completion,
Llm,
- OPENAI_MODELS,
- ANTHROPIC_MODELS,
- GEMINI_MODELS,
-)
-from models import (
- stream_claude_response,
- stream_openai_response,
- stream_gemini_response,
- stream_gemini_response_video,
)
from fs_logging.core import write_logs
from mock_llm import mock_completion
@@ -63,12 +53,9 @@ MessageType = Literal[
"toolStart",
"toolResult",
]
-from image_generation.core import generate_images
from prompts import create_prompt
from prompts.agentic_instructions import apply_tool_instructions
-from prompts.claude_prompts import GEMINI_VIDEO_PROMPT
from prompts.types import Stack, PromptContent
-from video.utils import get_video_bytes_and_mime_type
from agentic.runner import AgenticRunner
# from utils import pprint_prompt
@@ -78,14 +65,6 @@ from ws.constants import APP_ERROR_WEB_SOCKET_CODE # type: ignore
router = APIRouter()
-class VariantErrorAlreadySent(Exception):
- """Exception that indicates a variantError message has already been sent to frontend"""
-
- def __init__(self, original_error: Exception):
- self.original_error = original_error
- super().__init__(str(original_error))
-
-
@dataclass
class PipelineContext:
"""Context object that carries state through the pipeline"""
@@ -563,291 +542,6 @@ class PostProcessingStage:
# Note: WebSocket closing is handled by the caller
-class ParallelGenerationStage:
- """Handles parallel variant generation with independent processing for each variant"""
-
- def __init__(
- self,
- send_message: Callable[
- [MessageType, str | None, int, Dict[str, Any] | None, str | None],
- Coroutine[Any, Any, None],
- ],
- openai_api_key: str | None,
- openai_base_url: str | None,
- anthropic_api_key: str | None,
- gemini_api_key: str | None,
- should_generate_images: bool,
- input_mode: InputMode,
- generation_type: Literal["create", "update"],
- prompt: PromptContent,
- ):
- self.send_message = send_message
- self.openai_api_key = openai_api_key
- self.openai_base_url = openai_base_url
- self.anthropic_api_key = anthropic_api_key
- self.gemini_api_key = gemini_api_key
- self.should_generate_images = should_generate_images
- self.input_mode = input_mode
- self.generation_type = generation_type
- self.prompt = prompt
-
- async def process_variants(
- self,
- variant_models: List[Llm],
- prompt_messages: List[ChatCompletionMessageParam],
- image_cache: Dict[str, str],
- params: Dict[str, str],
- ) -> Dict[int, str]:
- """Process all variants in parallel and return completions"""
- tasks = self._create_generation_tasks(variant_models, prompt_messages, params)
-
- # Dictionary to track variant tasks and their status
- variant_tasks: Dict[int, asyncio.Task[Completion]] = {}
- variant_completions: Dict[int, str] = {}
-
- # Create tasks for each variant
- for index, task in enumerate(tasks):
- variant_task = asyncio.create_task(task)
- variant_tasks[index] = variant_task
-
- # Process each variant independently
- variant_processors = [
- self._process_variant_completion(
- index, task, variant_models[index], image_cache, variant_completions
- )
- for index, task in variant_tasks.items()
- ]
-
- # Wait for all variants to complete
- await asyncio.gather(*variant_processors, return_exceptions=True)
-
- return variant_completions
-
- def _create_generation_tasks(
- self,
- variant_models: List[Llm],
- prompt_messages: List[ChatCompletionMessageParam],
- params: Dict[str, str],
- ) -> List[Coroutine[Any, Any, Completion]]:
- """Create generation tasks for each variant model"""
- tasks: List[Coroutine[Any, Any, Completion]] = []
-
- # Check if this is a video create request
- is_video_create = (
- self.input_mode == "video" and self.generation_type == "create"
- )
-
- for index, model in enumerate(variant_models):
- # Video create mode: use video-specific Gemini streaming
- if is_video_create and model in GEMINI_MODELS:
- video_data_url = self.prompt.get("images", [None])[0]
- if not video_data_url:
- raise Exception("Video mode requires a video to be provided")
-
- video_bytes, video_mime_type = get_video_bytes_and_mime_type(
- video_data_url
- )
- print(
- f"Using Gemini for video generation (video size: {len(video_bytes)} bytes)"
- )
-
- tasks.append(
- stream_gemini_response_video(
- video_bytes=video_bytes,
- video_mime_type=video_mime_type,
- system_prompt=GEMINI_VIDEO_PROMPT,
- api_key=self.gemini_api_key,
- callback=lambda x, i=index: self._process_chunk(x, i),
- model=model,
- thinking_callback=lambda x, i=index: self._process_thinking(
- x, i
- ),
- )
- )
- elif model in OPENAI_MODELS:
- if self.openai_api_key is None:
- raise Exception("OpenAI API key is missing.")
-
- tasks.append(
- self._stream_openai_with_error_handling(
- prompt_messages,
- model=model,
- index=index,
- )
- )
- elif self.gemini_api_key and model in GEMINI_MODELS:
- tasks.append(
- stream_gemini_response(
- prompt_messages,
- api_key=self.gemini_api_key,
- callback=lambda x, i=index: self._process_chunk(x, i),
- model=model,
- thinking_callback=lambda x, i=index: self._process_thinking(
- x, i
- ),
- )
- )
- elif model in ANTHROPIC_MODELS:
- if self.anthropic_api_key is None:
- raise Exception("Anthropic API key is missing.")
-
- tasks.append(
- stream_claude_response(
- prompt_messages,
- api_key=self.anthropic_api_key,
- callback=lambda x, i=index: self._process_chunk(x, i),
- model_name=model.value,
- thinking_callback=lambda x, i=index: self._process_thinking(
- x, i
- ),
- )
- )
-
- return tasks
-
- async def _process_chunk(self, content: str, variant_index: int):
- """Process streaming chunks"""
- await self.send_message("chunk", content, variant_index)
-
- async def _process_thinking(self, content: str, variant_index: int):
- """Process thinking/reasoning content"""
- await self.send_message("thinking", content, variant_index)
-
- async def _stream_openai_with_error_handling(
- self,
- prompt_messages: List[ChatCompletionMessageParam],
- model: Llm,
- index: int,
- ) -> Completion:
- """Wrap OpenAI streaming with specific error handling"""
- try:
- assert self.openai_api_key is not None
- return await stream_openai_response(
- prompt_messages,
- api_key=self.openai_api_key,
- base_url=self.openai_base_url,
- callback=lambda x: self._process_chunk(x, index),
- model=model,
- )
- except openai.AuthenticationError as e:
- print(f"[VARIANT {index + 1}] OpenAI Authentication failed", e)
- error_message = (
- "Incorrect OpenAI key. Please make sure your OpenAI API key is correct, "
- "or create a new OpenAI API key on your OpenAI dashboard."
- + (
- " Alternatively, you can purchase code generation credits directly on this website."
- if IS_PROD
- else ""
- )
- )
- await self.send_message("variantError", error_message, index)
- raise VariantErrorAlreadySent(e)
- except openai.NotFoundError as e:
- print(f"[VARIANT {index + 1}] OpenAI Model not found", e)
- error_message = (
- e.message
- + ". Please make sure you have followed the instructions correctly to obtain "
- "an OpenAI key with GPT vision access: "
- "https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md"
- + (
- " Alternatively, you can purchase code generation credits directly on this website."
- if IS_PROD
- else ""
- )
- )
- await self.send_message("variantError", error_message, index)
- raise VariantErrorAlreadySent(e)
- except openai.RateLimitError as e:
- print(f"[VARIANT {index + 1}] OpenAI Rate limit exceeded", e)
- error_message = (
- "OpenAI error - 'You exceeded your current quota, please check your plan and billing details.'"
- + (
- " Alternatively, you can purchase code generation credits directly on this website."
- if IS_PROD
- else ""
- )
- )
- await self.send_message("variantError", error_message, index)
- raise VariantErrorAlreadySent(e)
-
- async def _perform_image_generation(
- self,
- completion: str,
- image_cache: dict[str, str],
- ):
- """Generate images for the completion if needed"""
- if not self.should_generate_images:
- return completion
-
- replicate_api_key = REPLICATE_API_KEY
- if replicate_api_key:
- image_generation_model = "flux"
- api_key = replicate_api_key
- else:
- if not self.openai_api_key:
- print(
- "No OpenAI API key and Replicate key found. Skipping image generation."
- )
- return completion
- image_generation_model = "dalle3"
- api_key = self.openai_api_key
-
- print("Generating images with model: ", image_generation_model)
-
- return await generate_images(
- completion,
- api_key=api_key,
- base_url=self.openai_base_url,
- image_cache=image_cache,
- model=image_generation_model,
- )
-
- async def _process_variant_completion(
- self,
- index: int,
- task: asyncio.Task[Completion],
- model: Llm,
- image_cache: Dict[str, str],
- variant_completions: Dict[int, str],
- ):
- """Process a single variant completion including image generation"""
- try:
- completion = await task
-
- print(f"{model.value} completion took {completion['duration']:.2f} seconds")
- variant_completions[index] = completion["code"]
-
- try:
- # Process images for this variant
- processed_html = await self._perform_image_generation(
- completion["code"],
- image_cache,
- )
- # Extract HTML content
- processed_html = extract_html_content(processed_html)
-
- # Send the complete variant back to the client
- await self.send_message("setCode", processed_html, index)
- await self.send_message(
- "variantComplete",
- "Variant generation complete",
- index,
- )
- except Exception as inner_e:
- # If websocket is closed or other error during post-processing
- print(f"Post-processing error for variant {index + 1}: {inner_e}")
- # We still keep the completion in variant_completions
-
- except Exception as e:
- # Handle any errors that occurred during generation
- print(f"Error in variant {index + 1}: {e}")
- traceback.print_exception(type(e), e, e.__traceback__)
-
- # Only send error message if it hasn't been sent already
- if not isinstance(e, VariantErrorAlreadySent):
- await self.send_message("variantError", str(e), index)
-
-
class AgenticGenerationStage:
"""Handles agentic tool-calling generation for each variant."""
@@ -1097,51 +791,22 @@ class CodeGenerationMiddleware(Middleware):
None,
)
- is_video_create = (
- context.extracted_params.input_mode == "video"
- and context.extracted_params.generation_type == "create"
+ generation_stage = AgenticGenerationStage(
+ send_message=context.send_message,
+ openai_api_key=context.extracted_params.openai_api_key,
+ openai_base_url=context.extracted_params.openai_base_url,
+ anthropic_api_key=context.extracted_params.anthropic_api_key,
+ gemini_api_key=GEMINI_API_KEY,
+ should_generate_images=context.extracted_params.should_generate_images,
+ image_cache=context.image_cache,
+ file_state=context.extracted_params.file_state,
+ option_codes=context.extracted_params.option_codes,
)
- if is_video_create:
- generation_stage = ParallelGenerationStage(
- send_message=context.send_message,
- openai_api_key=context.extracted_params.openai_api_key,
- openai_base_url=context.extracted_params.openai_base_url,
- anthropic_api_key=context.extracted_params.anthropic_api_key,
- gemini_api_key=GEMINI_API_KEY,
- should_generate_images=context.extracted_params.should_generate_images,
- input_mode=context.extracted_params.input_mode,
- generation_type=context.extracted_params.generation_type,
- prompt=context.extracted_params.prompt,
- )
-
- context.variant_completions = (
- await generation_stage.process_variants(
- variant_models=context.variant_models,
- prompt_messages=context.prompt_messages,
- image_cache=context.image_cache,
- params=context.params,
- )
- )
- else:
- generation_stage = AgenticGenerationStage(
- send_message=context.send_message,
- openai_api_key=context.extracted_params.openai_api_key,
- openai_base_url=context.extracted_params.openai_base_url,
- anthropic_api_key=context.extracted_params.anthropic_api_key,
- gemini_api_key=GEMINI_API_KEY,
- should_generate_images=context.extracted_params.should_generate_images,
- image_cache=context.image_cache,
- file_state=context.extracted_params.file_state,
- option_codes=context.extracted_params.option_codes,
- )
-
- context.variant_completions = (
- await generation_stage.process_variants(
- variant_models=context.variant_models,
- prompt_messages=context.prompt_messages,
- )
- )
+ context.variant_completions = await generation_stage.process_variants(
+ variant_models=context.variant_models,
+ prompt_messages=context.prompt_messages,
+ )
# Check if all variants failed
if len(context.variant_completions) == 0:
diff --git a/backend/tests/test_prompts.py b/backend/tests/test_prompts.py
index 1b70a874..c417d29b 100644
--- a/backend/tests/test_prompts.py
+++ b/backend/tests/test_prompts.py
@@ -321,9 +321,9 @@ class TestCreatePrompt:
async def test_video_mode_basic_prompt_creation(self) -> None:
"""Test basic video prompt creation in video mode.
- For video mode with generation_type="create", the prompt is empty
- because the actual generation is handled by VideoGenerationStage
- which sends the video directly to Gemini.
+ For video mode with generation_type="create", we now assemble
+ a regular system+user prompt so video generation can run through
+ the agentic runner path.
"""
# Setup test data
video_data_url: str = "data:video/mp4;base64,test_video_data"
@@ -345,9 +345,26 @@ class TestCreatePrompt:
is_imported_from_code=params.get("isImportedFromCode", False),
)
- # For video create mode, prompt is empty - actual generation handled by VideoGenerationStage
expected: ExpectedResult = {
- "messages": [],
+ "messages": [
+ {
+ "role": "system",
+ "content": "<CONTAINS:You will be given a video of a user interacting>",
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": video_data_url, "detail": "high"},
+ },
+ {
+ "type": "text",
+ "text": "Analyze this video and generate the code.",
+ },
+ ],
+ },
+ ],
"image_cache": {}
}