Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add Gemini 3 video-to-code support with direct video input

Claude2267d80a0c54
diff --git a/backend/models/__init__.py b/backend/models/__init__.py
index 3dc0d429..0b54c5fa 100644
--- a/backend/models/__init__.py
+++ b/backend/models/__init__.py
@@ -1,13 +1,14 @@
from .claude import stream_claude_response, stream_claude_response_native
from .openai_client import stream_openai_response
-from .gemini import stream_gemini_response
+from .gemini import stream_gemini_response, stream_gemini_response_video
from llm import Completion
# Re-export the functions for backward compatibility
__all__ = [
- "stream_claude_response",
+ "stream_claude_response",
"stream_claude_response_native",
"stream_openai_response",
"stream_gemini_response",
- "Completion"
+ "stream_gemini_response_video",
+ "Completion",
]
diff --git a/backend/models/gemini.py b/backend/models/gemini.py
index fb0df9ab..530b35e9 100644
--- a/backend/models/gemini.py
+++ b/backend/models/gemini.py
@@ -222,3 +222,107 @@ async def stream_gemini_response(
completion_time = time.time() - start_time
return {"duration": completion_time, "code": full_response}
+
+
+async def stream_gemini_response_video(
+ video_bytes: bytes,
+ video_mime_type: str,
+ system_prompt: str,
+ api_key: str,
+ callback: Callable[[str], Awaitable[None]],
+ model_name: str,
+ thinking_callback: Callable[[str], Awaitable[None]] | None = None,
+) -> Completion:
+ start_time = time.time()
+
+ client = genai.Client(api_key=api_key)
+ full_response = ""
+
+ # Create content with video inline data
+ contents = types.Content(
+ role="user",
+ parts=[
+ types.Part.from_bytes(data=video_bytes, mime_type=video_mime_type),
+ types.Part(text="Analyze this video and generate the code."),
+ ],
+ )
+
+ # Configure based on model
+ if model_name in ["gemini-3-flash-preview (high thinking)", "gemini-3-flash-preview"]:
+ config = types.GenerateContentConfig(
+ temperature=0,
+ max_output_tokens=30000,
+ system_instruction=system_prompt,
+ thinking_config=types.ThinkingConfig(
+ thinking_level="high", include_thoughts=True
+ ),
+ )
+ api_model_name = "gemini-3-flash-preview"
+ elif model_name == "gemini-3-flash-preview (minimal thinking)":
+ config = types.GenerateContentConfig(
+ temperature=0,
+ max_output_tokens=30000,
+ system_instruction=system_prompt,
+ thinking_config=types.ThinkingConfig(
+ thinking_level="minimal", include_thoughts=True
+ ),
+ )
+ api_model_name = "gemini-3-flash-preview"
+ elif model_name in ["gemini-3-pro-preview (high thinking)", "gemini-3-pro-preview"]:
+ config = types.GenerateContentConfig(
+ temperature=0,
+ max_output_tokens=30000,
+ system_instruction=system_prompt,
+ thinking_config=types.ThinkingConfig(
+ thinking_level="high", include_thoughts=True
+ ),
+ )
+ api_model_name = "gemini-3-pro-preview"
+ elif model_name == "gemini-3-pro-preview (low thinking)":
+ config = types.GenerateContentConfig(
+ temperature=0,
+ max_output_tokens=30000,
+ system_instruction=system_prompt,
+ thinking_config=types.ThinkingConfig(
+ thinking_level="low", include_thoughts=True
+ ),
+ )
+ api_model_name = "gemini-3-pro-preview"
+ else:
+ # Default config for other models
+ config = types.GenerateContentConfig(
+ temperature=0,
+ max_output_tokens=20000,
+ system_instruction=system_prompt,
+ )
+ api_model_name = model_name
+
+ if DEBUG_GEMINI:
+ print(f"\n=== Gemini Video Request Debug ({api_model_name}) ===")
+ print(f"Video MIME type: {video_mime_type}")
+ print(f"Video size: {len(video_bytes)} bytes")
+ print(f"System prompt (first 200 chars): {system_prompt[:200]}...")
+ print("=" * 50)
+
+ async for chunk in await client.aio.models.generate_content_stream(
+ model=api_model_name,
+ contents=contents,
+ config=config,
+ ):
+ if chunk.candidates and len(chunk.candidates) > 0:
+ for part in chunk.candidates[0].content.parts:
+ if not part.text:
+ continue
+ elif part.thought:
+ if thinking_callback:
+ await thinking_callback(part.text)
+ else:
+ print(f"\n=== Gemini Video Thinking Summary ({api_model_name}) ===")
+ print(part.text)
+ print("=" * 50)
+ else:
+ full_response += part.text
+ await callback(part.text)
+
+ completion_time = time.time() - start_time
+ return {"duration": completion_time, "code": full_response}
diff --git a/backend/prompts/claude_prompts.py b/backend/prompts/claude_prompts.py
index 7be6bc50..2059df7e 100644
--- a/backend/prompts/claude_prompts.py
+++ b/backend/prompts/claude_prompts.py
@@ -34,7 +34,7 @@ You also have perfect vision and pay great attention to detail.
You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family,
+- Pay close attention to background color, text color, font size, font family,
padding, margin, border, etc. Match the colors and sizes exactly.
- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- If some fuctionality requires a backend call, just mock the data instead.
@@ -51,6 +51,31 @@ Before generating the code for the app, think step-by-step: first, about the use
"""
+GEMINI_VIDEO_PROMPT = """
+You are an expert at building single page, functional apps using HTML, jQuery and Tailwind CSS.
+You also have perfect vision and pay great attention to detail.
+
+You will be given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
+
+- Watch the entire video carefully and understand all the user interactions and UI state changes.
+- Make sure the app looks exactly like what you see in the video.
+- Pay close attention to background color, text color, font size, font family,
+padding, margin, border, etc. Match the colors and sizes exactly.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+- If some functionality requires a backend call, just mock the data instead.
+- MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
+- Use jQuery: <script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>
+
+Before generating the code for the app, think step-by-step: first, about the user flow depicted in the video and then about how you would build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.
+"""
+
+
HTML_TAILWIND_CLAUDE_SYSTEM_PROMPT = """
You have perfect vision and pay great attention to detail which makes you an expert at building single page apps using Tailwind, HTML and JS.
You take screenshots of a reference web page from the user, and then build single page apps
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index 48cfe7f1..e6ced60b 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -30,6 +30,7 @@ from models import (
stream_claude_response_native,
stream_openai_response,
stream_gemini_response,
+ stream_gemini_response_video,
)
from fs_logging.core import write_logs
from mock_llm import mock_completion
@@ -60,8 +61,9 @@ MessageType = Literal[
]
from image_generation.core import generate_images
from prompts import create_prompt
-from prompts.claude_prompts import VIDEO_PROMPT
+from prompts.claude_prompts import VIDEO_PROMPT, GEMINI_VIDEO_PROMPT
from prompts.types import Stack, PromptContent
+from video.utils import get_video_bytes_and_mime_type
# from utils import pprint_prompt
from ws.constants import APP_ERROR_WEB_SOCKET_CODE # type: ignore
@@ -463,7 +465,7 @@ class MockResponseStage:
class VideoGenerationStage:
- """Handles video mode code generation using Claude 3 Opus"""
+ """Handles video mode code generation using Claude 3 Opus or Gemini 3"""
def __init__(
self,
@@ -477,19 +479,74 @@ class VideoGenerationStage:
self,
prompt_messages: List[ChatCompletionMessageParam],
anthropic_api_key: str | None,
+ gemini_api_key: str | None = None,
+ video_data_url: str | None = None,
) -> List[str]:
- """Generate code for video input mode"""
+ """Generate code for video input mode using Claude or Gemini"""
+ # Prefer Gemini if API key is available and video data URL is provided
+ if gemini_api_key and video_data_url:
+ return await self._generate_with_gemini(
+ video_data_url, gemini_api_key
+ )
+
+ # Fall back to Claude
if not anthropic_api_key:
await self.throw_error(
- "Video only works with Anthropic models. No Anthropic API key found. "
- "Please add the environment variable ANTHROPIC_API_KEY to backend/.env "
+ "Video mode requires either a Gemini API key or an Anthropic API key. "
+ "Please add GEMINI_API_KEY or ANTHROPIC_API_KEY to backend/.env "
"or in the settings dialog"
)
- raise Exception("No Anthropic key")
+ raise Exception("No API key for video generation")
- async def process_chunk(content: str, variantIndex: int):
+ return await self._generate_with_claude(prompt_messages, anthropic_api_key)
+
+ async def _generate_with_gemini(
+ self,
+ video_data_url: str,
+ gemini_api_key: str,
+ ) -> List[str]:
+ """Generate code using Gemini 3 with direct video input"""
+ async def process_chunk(content: str, variantIndex: int = 0):
await self.send_message("chunk", content, variantIndex)
+ async def process_thinking(content: str, variantIndex: int = 0):
+ await self.send_message("thinking", content, variantIndex)
+
+ # Get video bytes and mime type
+ video_bytes, video_mime_type = get_video_bytes_and_mime_type(video_data_url)
+
+ print(f"Using Gemini 3 for video generation (video size: {len(video_bytes)} bytes)")
+
+ completion_results = [
+ await stream_gemini_response_video(
+ video_bytes=video_bytes,
+ video_mime_type=video_mime_type,
+ system_prompt=GEMINI_VIDEO_PROMPT,
+ api_key=gemini_api_key,
+ callback=lambda x: process_chunk(x, 0),
+ model_name="gemini-3-flash-preview",
+ thinking_callback=lambda x: process_thinking(x, 0),
+ )
+ ]
+ completions = [result["code"] for result in completion_results]
+
+ # Send the complete variant back to the client
+ await self.send_message("setCode", completions[0], 0)
+ await self.send_message("variantComplete", "Variant generation complete", 0)
+
+ return completions
+
+ async def _generate_with_claude(
+ self,
+ prompt_messages: List[ChatCompletionMessageParam],
+ anthropic_api_key: str,
+ ) -> List[str]:
+ """Generate code using Claude 3 Opus with screenshots extracted from video"""
+ async def process_chunk(content: str, variantIndex: int = 0):
+ await self.send_message("chunk", content, variantIndex)
+
+ print("Using Claude 3 Opus for video generation (screenshot-based)")
+
completion_results = [
await stream_claude_response_native(
system_prompt=VIDEO_PROMPT,
@@ -871,9 +928,13 @@ class CodeGenerationMiddleware(Middleware):
video_stage = VideoGenerationStage(
context.send_message, context.throw_error
)
+ # Get the video data URL from the prompt
+ video_data_url = context.extracted_params.prompt.get("images", [None])[0]
context.completions = await video_stage.generate_video_code(
context.prompt_messages,
context.extracted_params.anthropic_api_key,
+ gemini_api_key=GEMINI_API_KEY,
+ video_data_url=video_data_url,
)
else:
# Select models
diff --git a/backend/video/utils.py b/backend/video/utils.py
index dc772fc3..6b6116e0 100644
--- a/backend/video/utils.py
+++ b/backend/video/utils.py
@@ -132,3 +132,10 @@ def extract_tag_content(tag: str, text: str) -> str:
if start_idx != -1 and end_idx != -1:
return text[start_idx : end_idx + len(tag_end)]
return ""
+
+
+def get_video_bytes_and_mime_type(video_data_url: str) -> tuple[bytes, str]:
+ video_encoded_data = video_data_url.split(",")[1]
+ video_bytes = base64.b64decode(video_encoded_data)
+ mime_type = video_data_url.split(";")[0].split(":")[1]
+ return video_bytes, mime_type