Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Add Gemini 3 video-to-code support with direct video input
Claude2267d80a0c54
diff --git a/backend/models/__init__.py b/backend/models/__init__.pyindex 3dc0d429..0b54c5fa 100644--- a/backend/models/__init__.py+++ b/backend/models/__init__.py@@ -1,13 +1,14 @@from .claude import stream_claude_response, stream_claude_response_nativefrom .openai_client import stream_openai_response-from .gemini import stream_gemini_response+from .gemini import stream_gemini_response, stream_gemini_response_videofrom llm import Completion# Re-export the functions for backward compatibility__all__ = [- "stream_claude_response",+ "stream_claude_response","stream_claude_response_native","stream_openai_response","stream_gemini_response",- "Completion"+ "stream_gemini_response_video",+ "Completion",]diff --git a/backend/models/gemini.py b/backend/models/gemini.pyindex fb0df9ab..530b35e9 100644--- a/backend/models/gemini.py+++ b/backend/models/gemini.py@@ -222,3 +222,107 @@ async def stream_gemini_response(completion_time = time.time() - start_timereturn {"duration": completion_time, "code": full_response}+++async def stream_gemini_response_video(+ video_bytes: bytes,+ video_mime_type: str,+ system_prompt: str,+ api_key: str,+ callback: Callable[[str], Awaitable[None]],+ model_name: str,+ thinking_callback: Callable[[str], Awaitable[None]] | None = None,+) -> Completion:+ start_time = time.time()++ client = genai.Client(api_key=api_key)+ full_response = ""++ # Create content with video inline data+ contents = types.Content(+ role="user",+ parts=[+ types.Part.from_bytes(data=video_bytes, mime_type=video_mime_type),+ types.Part(text="Analyze this video and generate the code."),+ ],+ )++ # Configure based on model+ if model_name in ["gemini-3-flash-preview (high thinking)", "gemini-3-flash-preview"]:+ config = types.GenerateContentConfig(+ temperature=0,+ max_output_tokens=30000,+ system_instruction=system_prompt,+ thinking_config=types.ThinkingConfig(+ thinking_level="high", include_thoughts=True+ ),+ )+ api_model_name = "gemini-3-flash-preview"+ elif model_name == "gemini-3-flash-preview (minimal thinking)":+ config = types.GenerateContentConfig(+ temperature=0,+ max_output_tokens=30000,+ system_instruction=system_prompt,+ thinking_config=types.ThinkingConfig(+ thinking_level="minimal", include_thoughts=True+ ),+ )+ api_model_name = "gemini-3-flash-preview"+ elif model_name in ["gemini-3-pro-preview (high thinking)", "gemini-3-pro-preview"]:+ config = types.GenerateContentConfig(+ temperature=0,+ max_output_tokens=30000,+ system_instruction=system_prompt,+ thinking_config=types.ThinkingConfig(+ thinking_level="high", include_thoughts=True+ ),+ )+ api_model_name = "gemini-3-pro-preview"+ elif model_name == "gemini-3-pro-preview (low thinking)":+ config = types.GenerateContentConfig(+ temperature=0,+ max_output_tokens=30000,+ system_instruction=system_prompt,+ thinking_config=types.ThinkingConfig(+ thinking_level="low", include_thoughts=True+ ),+ )+ api_model_name = "gemini-3-pro-preview"+ else:+ # Default config for other models+ config = types.GenerateContentConfig(+ temperature=0,+ max_output_tokens=20000,+ system_instruction=system_prompt,+ )+ api_model_name = model_name++ if DEBUG_GEMINI:+ print(f"\n=== Gemini Video Request Debug ({api_model_name}) ===")+ print(f"Video MIME type: {video_mime_type}")+ print(f"Video size: {len(video_bytes)} bytes")+ print(f"System prompt (first 200 chars): {system_prompt[:200]}...")+ print("=" * 50)++ async for chunk in await client.aio.models.generate_content_stream(+ model=api_model_name,+ contents=contents,+ config=config,+ ):+ if chunk.candidates and len(chunk.candidates) > 0:+ for part in chunk.candidates[0].content.parts:+ if not part.text:+ continue+ elif part.thought:+ if thinking_callback:+ await thinking_callback(part.text)+ else:+ print(f"\n=== Gemini Video Thinking Summary ({api_model_name}) ===")+ print(part.text)+ print("=" * 50)+ else:+ full_response += part.text+ await callback(part.text)++ completion_time = time.time() - start_time+ return {"duration": completion_time, "code": full_response}diff --git a/backend/prompts/claude_prompts.py b/backend/prompts/claude_prompts.pyindex 7be6bc50..2059df7e 100644--- a/backend/prompts/claude_prompts.py+++ b/backend/prompts/claude_prompts.py@@ -34,7 +34,7 @@ You also have perfect vision and pay great attention to detail.You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.- Make sure the app looks exactly like the screenshot.-- Pay close attention to background color, text color, font size, font family,+- Pay close attention to background color, text color, font size, font family,padding, margin, border, etc. Match the colors and sizes exactly.- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.- If some fuctionality requires a backend call, just mock the data instead.@@ -51,6 +51,31 @@ Before generating the code for the app, think step-by-step: first, about the use"""+GEMINI_VIDEO_PROMPT = """+You are an expert at building single page, functional apps using HTML, jQuery and Tailwind CSS.+You also have perfect vision and pay great attention to detail.++You will be given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.++- Watch the entire video carefully and understand all the user interactions and UI state changes.+- Make sure the app looks exactly like what you see in the video.+- Pay close attention to background color, text color, font size, font family,+padding, margin, border, etc. Match the colors and sizes exactly.+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.+- If some functionality requires a backend call, just mock the data instead.+- MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.++In terms of libraries,++- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>+- You can use Google Fonts+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>+- Use jQuery: <script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>++Before generating the code for the app, think step-by-step: first, about the user flow depicted in the video and then about how you would build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.+"""++HTML_TAILWIND_CLAUDE_SYSTEM_PROMPT = """You have perfect vision and pay great attention to detail which makes you an expert at building single page apps using Tailwind, HTML and JS.You take screenshots of a reference web page from the user, and then build single page appsdiff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.pyindex 48cfe7f1..e6ced60b 100644--- a/backend/routes/generate_code.py+++ b/backend/routes/generate_code.py@@ -30,6 +30,7 @@ from models import (stream_claude_response_native,stream_openai_response,stream_gemini_response,+ stream_gemini_response_video,)from fs_logging.core import write_logsfrom mock_llm import mock_completion@@ -60,8 +61,9 @@ MessageType = Literal[]from image_generation.core import generate_imagesfrom prompts import create_prompt-from prompts.claude_prompts import VIDEO_PROMPT+from prompts.claude_prompts import VIDEO_PROMPT, GEMINI_VIDEO_PROMPTfrom prompts.types import Stack, PromptContent+from video.utils import get_video_bytes_and_mime_type# from utils import pprint_promptfrom ws.constants import APP_ERROR_WEB_SOCKET_CODE # type: ignore@@ -463,7 +465,7 @@ class MockResponseStage:class VideoGenerationStage:- """Handles video mode code generation using Claude 3 Opus"""+ """Handles video mode code generation using Claude 3 Opus or Gemini 3"""def __init__(self,@@ -477,19 +479,74 @@ class VideoGenerationStage:self,prompt_messages: List[ChatCompletionMessageParam],anthropic_api_key: str | None,+ gemini_api_key: str | None = None,+ video_data_url: str | None = None,) -> List[str]:- """Generate code for video input mode"""+ """Generate code for video input mode using Claude or Gemini"""+ # Prefer Gemini if API key is available and video data URL is provided+ if gemini_api_key and video_data_url:+ return await self._generate_with_gemini(+ video_data_url, gemini_api_key+ )++ # Fall back to Claudeif not anthropic_api_key:await self.throw_error(- "Video only works with Anthropic models. No Anthropic API key found. "- "Please add the environment variable ANTHROPIC_API_KEY to backend/.env "+ "Video mode requires either a Gemini API key or an Anthropic API key. "+ "Please add GEMINI_API_KEY or ANTHROPIC_API_KEY to backend/.env ""or in the settings dialog")- raise Exception("No Anthropic key")+ raise Exception("No API key for video generation")- async def process_chunk(content: str, variantIndex: int):+ return await self._generate_with_claude(prompt_messages, anthropic_api_key)++ async def _generate_with_gemini(+ self,+ video_data_url: str,+ gemini_api_key: str,+ ) -> List[str]:+ """Generate code using Gemini 3 with direct video input"""+ async def process_chunk(content: str, variantIndex: int = 0):await self.send_message("chunk", content, variantIndex)+ async def process_thinking(content: str, variantIndex: int = 0):+ await self.send_message("thinking", content, variantIndex)++ # Get video bytes and mime type+ video_bytes, video_mime_type = get_video_bytes_and_mime_type(video_data_url)++ print(f"Using Gemini 3 for video generation (video size: {len(video_bytes)} bytes)")++ completion_results = [+ await stream_gemini_response_video(+ video_bytes=video_bytes,+ video_mime_type=video_mime_type,+ system_prompt=GEMINI_VIDEO_PROMPT,+ api_key=gemini_api_key,+ callback=lambda x: process_chunk(x, 0),+ model_name="gemini-3-flash-preview",+ thinking_callback=lambda x: process_thinking(x, 0),+ )+ ]+ completions = [result["code"] for result in completion_results]++ # Send the complete variant back to the client+ await self.send_message("setCode", completions[0], 0)+ await self.send_message("variantComplete", "Variant generation complete", 0)++ return completions++ async def _generate_with_claude(+ self,+ prompt_messages: List[ChatCompletionMessageParam],+ anthropic_api_key: str,+ ) -> List[str]:+ """Generate code using Claude 3 Opus with screenshots extracted from video"""+ async def process_chunk(content: str, variantIndex: int = 0):+ await self.send_message("chunk", content, variantIndex)++ print("Using Claude 3 Opus for video generation (screenshot-based)")+completion_results = [await stream_claude_response_native(system_prompt=VIDEO_PROMPT,@@ -871,9 +928,13 @@ class CodeGenerationMiddleware(Middleware):video_stage = VideoGenerationStage(context.send_message, context.throw_error)+ # Get the video data URL from the prompt+ video_data_url = context.extracted_params.prompt.get("images", [None])[0]context.completions = await video_stage.generate_video_code(context.prompt_messages,context.extracted_params.anthropic_api_key,+ gemini_api_key=GEMINI_API_KEY,+ video_data_url=video_data_url,)else:# Select modelsdiff --git a/backend/video/utils.py b/backend/video/utils.pyindex dc772fc3..6b6116e0 100644--- a/backend/video/utils.py+++ b/backend/video/utils.py@@ -132,3 +132,10 @@ def extract_tag_content(tag: str, text: str) -> str:if start_idx != -1 and end_idx != -1:return text[start_idx : end_idx + len(tag_end)]return ""+++def get_video_bytes_and_mime_type(video_data_url: str) -> tuple[bytes, str]:+ video_encoded_data = video_data_url.split(",")[1]+ video_bytes = base64.b64decode(video_encoded_data)+ mime_type = video_data_url.split(";")[0].split(":")[1]+ return video_bytes, mime_type