Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Remove Claude Opus 3 video support, use only Gemini for video

Clauded865528937ca
diff --git a/backend/prompts/__init__.py b/backend/prompts/__init__.py
index 58b5e4d3..fa381fc3 100644
--- a/backend/prompts/__init__.py
+++ b/backend/prompts/__init__.py
@@ -7,7 +7,6 @@ from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTS
from prompts.screenshot_system_prompts import SYSTEM_PROMPTS
from prompts.text_prompts import SYSTEM_PROMPTS as TEXT_SYSTEM_PROMPTS
from prompts.types import Stack, PromptContent
-from video.utils import assemble_claude_prompt_video
USER_PROMPT = """
@@ -47,11 +46,11 @@ async def create_prompt(
elif input_mode == "text":
prompt_messages = assemble_text_prompt(prompt["text"], stack)
elif input_mode == "video":
- # For video mode initial creation, extract screenshots from video
- # For updates, we use the history which already contains the generated code
+ # For video mode initial creation, prompt is handled by VideoGenerationStage
+ # which sends the video directly to Gemini with its own prompt
if generation_type == "create":
- video_data_url = prompt["images"][0]
- prompt_messages = await assemble_claude_prompt_video(video_data_url)
+ # Return empty prompt - actual generation is handled by VideoGenerationStage
+ prompt_messages = []
else:
# For video mode updates, use the screenshot system prompt
# since we're now working with the generated code, not the video
diff --git a/backend/prompts/claude_prompts.py b/backend/prompts/claude_prompts.py
index 6902995b..57f1690a 100644
--- a/backend/prompts/claude_prompts.py
+++ b/backend/prompts/claude_prompts.py
@@ -1,56 +1,3 @@
-# Not used yet
-# References:
-# https://github.com/hundredblocks/transcription_demo
-# https://docs.anthropic.com/claude/docs/prompt-engineering
-# https://github.com/anthropics/anthropic-cookbook/blob/main/multimodal/best_practices_for_vision.ipynb
-
-VIDEO_PROMPT = """
-You are an expert at building single page, funtional apps using HTML, Jquery and Tailwind CSS.
-You also have perfect vision and pay great attention to detail.
-
-You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family,
-padding, margin, border, etc. Match the colors and sizes exactly.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-- If some fuctionality requires a backend call, just mock the data instead.
-- MAKE THE APP FUNCTIONAL using Javascript. Allow the user to interact with the app and get the same behavior as the video.
-
-In terms of libraries,
-
-- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-- Use jQuery: <script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>
-
-Before generating the code for the app, think step-by-step: first, about the user flow depicated in the video and then about you how would you build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.
-"""
-
-VIDEO_PROMPT_ALPINE_JS = """
-You are an expert at building single page, funtional apps using HTML, Alpine.js and Tailwind CSS.
-You also have perfect vision and pay great attention to detail.
-
-You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
-
-- Make sure the app looks exactly like the screenshot.
-- Pay close attention to background color, text color, font size, font family,
-padding, margin, border, etc. Match the colors and sizes exactly.
-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
-- If some fuctionality requires a backend call, just mock the data instead.
-- MAKE THE APP FUNCTIONAL using Javascript. Allow the user to interact with the app and get the same behavior as the video.
-
-In terms of libraries,
-
-- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
-- You can use Google Fonts
-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>
-- Use Alpine.js: <script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/cdn.min.js"></script>
-
-Before generating the code for the app, think step-by-step: first, about the user flow depicated in the video and then about you how would you build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.
-"""
-
-
GEMINI_VIDEO_PROMPT = """
You are an expert at building single page, functional apps using HTML, jQuery and Tailwind CSS.
You also have perfect vision and pay great attention to detail.
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index f2446278..45db9766 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -27,7 +27,6 @@ from llm import (
)
from models import (
stream_claude_response,
- stream_claude_response_native,
stream_openai_response,
stream_gemini_response,
stream_gemini_response_video,
@@ -61,7 +60,7 @@ MessageType = Literal[
]
from image_generation.core import generate_images
from prompts import create_prompt
-from prompts.claude_prompts import VIDEO_PROMPT, GEMINI_VIDEO_PROMPT
+from prompts.claude_prompts import GEMINI_VIDEO_PROMPT
from prompts.types import Stack, PromptContent
from video.utils import get_video_bytes_and_mime_type
@@ -465,7 +464,7 @@ class MockResponseStage:
class VideoGenerationStage:
- """Handles video mode code generation using Claude 3 Opus or Gemini 3"""
+ """Handles video mode code generation using Gemini 3"""
def __init__(
self,
@@ -477,28 +476,22 @@ class VideoGenerationStage:
async def generate_video_code(
self,
- prompt_messages: List[ChatCompletionMessageParam],
- anthropic_api_key: str | None,
- gemini_api_key: str | None = None,
+ gemini_api_key: str | None,
video_data_url: str | None = None,
) -> List[str]:
- """Generate code for video input mode using Claude or Gemini"""
- # Prefer Gemini if API key is available and video data URL is provided
- if gemini_api_key and video_data_url:
- return await self._generate_with_gemini(
- video_data_url, gemini_api_key
- )
-
- # Fall back to Claude
- if not anthropic_api_key:
+ """Generate code for video input mode using Gemini"""
+ if not gemini_api_key:
await self.throw_error(
- "Video mode requires either a Gemini API key or an Anthropic API key. "
- "Please add GEMINI_API_KEY or ANTHROPIC_API_KEY to backend/.env "
- "or in the settings dialog"
+ "Video mode requires a Gemini API key. "
+ "Please add GEMINI_API_KEY to backend/.env or in the settings dialog"
)
- raise Exception("No API key for video generation")
+ raise Exception("No Gemini API key for video generation")
+
+ if not video_data_url:
+ await self.throw_error("Video mode requires a video to be provided")
+ raise Exception("No video data URL provided")
- return await self._generate_with_claude(prompt_messages, anthropic_api_key)
+ return await self._generate_with_gemini(video_data_url, gemini_api_key)
async def _generate_with_gemini(
self,
@@ -536,35 +529,6 @@ class VideoGenerationStage:
return completions
- async def _generate_with_claude(
- self,
- prompt_messages: List[ChatCompletionMessageParam],
- anthropic_api_key: str,
- ) -> List[str]:
- """Generate code using Claude 3 Opus with screenshots extracted from video"""
- async def process_chunk(content: str, variantIndex: int = 0):
- await self.send_message("chunk", content, variantIndex)
-
- print("Using Claude 3 Opus for video generation (screenshot-based)")
-
- completion_results = [
- await stream_claude_response_native(
- system_prompt=VIDEO_PROMPT,
- messages=prompt_messages, # type: ignore
- api_key=anthropic_api_key,
- callback=lambda x: process_chunk(x, 0),
- model_name=Llm.CLAUDE_3_OPUS.value,
- include_thinking=True,
- )
- ]
- completions = [result["code"] for result in completion_results]
-
- # Send the complete variant back to the client
- await self.send_message("setCode", completions[0], 0)
- await self.send_message("variantComplete", "Variant generation complete", 0)
-
- return completions
-
class PostProcessingStage:
"""Handles post-processing after code generation completes"""
@@ -937,8 +901,6 @@ class CodeGenerationMiddleware(Middleware):
# Get the video data URL from the prompt
video_data_url = context.extracted_params.prompt.get("images", [None])[0]
context.completions = await video_stage.generate_video_code(
- context.prompt_messages,
- context.extracted_params.anthropic_api_key,
gemini_api_key=GEMINI_API_KEY,
video_data_url=video_data_url,
)
diff --git a/backend/tests/test_prompts.py b/backend/tests/test_prompts.py
index b1e725bf..ea385164 100644
--- a/backend/tests/test_prompts.py
+++ b/backend/tests/test_prompts.py
@@ -319,7 +319,12 @@ class TestCreatePrompt:
@pytest.mark.asyncio
async def test_video_mode_basic_prompt_creation(self) -> None:
- """Test basic video prompt creation in video mode."""
+ """Test basic video prompt creation in video mode.
+
+ For video mode with generation_type="create", the prompt is empty
+ because the actual generation is handled by VideoGenerationStage
+ which sends the video directly to Gemini.
+ """
# Setup test data
video_data_url: str = "data:video/mp4;base64,test_video_data"
params: Dict[str, Any] = {
@@ -329,86 +334,26 @@ class TestCreatePrompt:
},
"generationType": "create"
}
-
- # Mock the video processing function
- mock_video_messages: List[Dict[str, Any]] = [
- {
- "role": "system",
- "content": "Mock Video System Prompt"
- },
- {
- "role": "user",
- "content": [
- {
- "type": "image_url",
- "image_url": {
- "url": "data:image/png;base64,frame1",
- "detail": "high"
- }
- },
- {
- "type": "image_url",
- "image_url": {
- "url": "data:image/png;base64,frame2",
- "detail": "high"
- }
- },
- {
- "type": "text",
- "text": "Create a functional web app from these video frames"
- }
- ]
- }
- ]
-
- with patch('prompts.assemble_claude_prompt_video', return_value=mock_video_messages):
- # Call the function
- messages, image_cache = await create_prompt(
- stack=self.TEST_STACK,
- input_mode="video",
- generation_type=params["generationType"],
- prompt=params["prompt"],
- history=params.get("history", []),
- is_imported_from_code=params.get("isImportedFromCode", False),
- )
-
- # Define expected structure
- expected: ExpectedResult = {
- "messages": [
- {
- "role": "system",
- "content": "Mock Video System Prompt"
- },
- {
- "role": "user",
- "content": [
- {
- "type": "image_url",
- "image_url": {
- "url": "data:image/png;base64,frame1",
- "detail": "high"
- }
- },
- {
- "type": "image_url",
- "image_url": {
- "url": "data:image/png;base64,frame2",
- "detail": "high"
- }
- },
- {
- "type": "text",
- "text": "Create a functional web app from these video frames"
- }
- ]
- }
- ],
- "image_cache": {}
- }
-
- # Assert the structure matches
- actual: ExpectedResult = {"messages": messages, "image_cache": image_cache}
- assert_structure_match(actual, expected)
+
+ # Call the function
+ messages, image_cache = await create_prompt(
+ stack=self.TEST_STACK,
+ input_mode="video",
+ generation_type=params["generationType"],
+ prompt=params["prompt"],
+ history=params.get("history", []),
+ is_imported_from_code=params.get("isImportedFromCode", False),
+ )
+
+ # For video create mode, prompt is empty - actual generation handled by VideoGenerationStage
+ expected: ExpectedResult = {
+ "messages": [],
+ "image_cache": {}
+ }
+
+ # Assert the structure matches
+ actual: ExpectedResult = {"messages": messages, "image_cache": image_cache}
+ assert_structure_match(actual, expected)
@pytest.mark.asyncio
diff --git a/backend/video/utils.py b/backend/video/utils.py
index 6b6116e0..97880f30 100644
--- a/backend/video/utils.py
+++ b/backend/video/utils.py
@@ -1,120 +1,4 @@
-# Extract HTML content from the completion string
import base64
-import io
-import mimetypes
-import os
-import tempfile
-import uuid
-from typing import Any, Union, cast
-from moviepy.editor import VideoFileClip # type: ignore
-from PIL import Image
-import math
-
-
-DEBUG = True
-TARGET_NUM_SCREENSHOTS = (
- 20 # Should be max that Claude supports (20) - reduce to save tokens on testing
-)
-
-
-async def assemble_claude_prompt_video(video_data_url: str) -> list[Any]:
- images = split_video_into_screenshots(video_data_url)
-
- # Save images to tmp if we're debugging
- if DEBUG:
- save_images_to_tmp(images)
-
- # Validate number of images
- print(f"Number of frames extracted from video: {len(images)}")
- if len(images) > 20:
- print(f"Too many screenshots: {len(images)}")
- raise ValueError("Too many screenshots extracted from video")
-
- # Convert images to the message format for Claude
- content_messages: list[dict[str, Union[dict[str, str], str]]] = []
- for image in images:
-
- # Convert Image to buffer
- buffered = io.BytesIO()
- image.save(buffered, format="JPEG")
-
- # Encode bytes as base64
- base64_data = base64.b64encode(buffered.getvalue()).decode("utf-8")
- media_type = "image/jpeg"
-
- content_messages.append(
- {
- "type": "image",
- "source": {
- "type": "base64",
- "media_type": media_type,
- "data": base64_data,
- },
- }
- )
-
- return [
- {
- "role": "user",
- "content": content_messages,
- },
- ]
-
-
-# Returns a list of images/frame (RGB format)
-def split_video_into_screenshots(video_data_url: str) -> list[Image.Image]:
- target_num_screenshots = TARGET_NUM_SCREENSHOTS
-
- # Decode the base64 URL to get the video bytes
- video_encoded_data = video_data_url.split(",")[1]
- video_bytes = base64.b64decode(video_encoded_data)
-
- mime_type = video_data_url.split(";")[0].split(":")[1]
- suffix = mimetypes.guess_extension(mime_type)
-
- with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as temp_video_file:
- print(temp_video_file.name)
- temp_video_file.write(video_bytes)
- temp_video_file.flush()
- clip = VideoFileClip(temp_video_file.name)
- images: list[Image.Image] = []
- total_frames = cast(int, clip.reader.nframes) # type: ignore
-
- # Calculate frame skip interval by dividing total frames by the target number of screenshots
- # Ensuring a minimum skip of 1 frame
- frame_skip = max(1, math.ceil(total_frames / target_num_screenshots))
-
- # Iterate over each frame in the clip
- for i, frame in enumerate(clip.iter_frames()):
- # Save every nth frame
- if i % frame_skip == 0:
- frame_image = Image.fromarray(frame) # type: ignore
- images.append(frame_image)
- # Ensure that we don't capture more than the desired number of frames
- if len(images) >= target_num_screenshots:
- break
-
- # Close the video file to release resources
- clip.close()
-
- return images
-
-
-# Save a list of PIL images to a random temporary directory
-def save_images_to_tmp(images: list[Image.Image]):
-
- # Create a unique temporary directory
- unique_dir_name = f"screenshots_{uuid.uuid4()}"
- tmp_screenshots_dir = os.path.join(tempfile.gettempdir(), unique_dir_name)
- os.makedirs(tmp_screenshots_dir, exist_ok=True)
-
- for idx, image in enumerate(images):
- # Generate a unique image filename using index
- image_filename = f"screenshot_{idx}.jpg"
- tmp_filepath = os.path.join(tmp_screenshots_dir, image_filename)
- image.save(tmp_filepath, format="JPEG")
-
- print("Saved to " + tmp_screenshots_dir)
def extract_tag_content(tag: str, text: str) -> str:
diff --git a/backend/video_to_app.py b/backend/video_to_app.py
deleted file mode 100644
index b8836f2f..00000000
--- a/backend/video_to_app.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# Load environment variables first
-
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-import base64
-import mimetypes
-import time
-import subprocess
-import os
-import asyncio
-from datetime import datetime
-from prompts.claude_prompts import VIDEO_PROMPT
-from utils import pprint_prompt
-from config import ANTHROPIC_API_KEY
-from video.utils import extract_tag_content, assemble_claude_prompt_video
-from llm import Llm
-from models import stream_claude_response_native
-
-STACK = "html_tailwind"
-
-VIDEO_DIR = "./video_evals/videos"
-SCREENSHOTS_DIR = "./video_evals/screenshots"
-OUTPUTS_DIR = "./video_evals/outputs"
-
-
-async def main():
- video_filename = "shortest.mov"
- is_followup = False
-
- if not ANTHROPIC_API_KEY:
- raise ValueError("ANTHROPIC_API_KEY is not set")
-
- # Get previous HTML
- previous_html = ""
- if is_followup:
- previous_html_file = max(
- [
- os.path.join(OUTPUTS_DIR, f)
- for f in os.listdir(OUTPUTS_DIR)
- if f.endswith(".html")
- ],
- key=os.path.getctime,
- )
- with open(previous_html_file, "r") as file:
- previous_html = file.read()
-
- video_file = os.path.join(VIDEO_DIR, video_filename)
- mime_type = mimetypes.guess_type(video_file)[0]
- with open(video_file, "rb") as file:
- video_content = file.read()
- video_data_url = (
- f"data:{mime_type};base64,{base64.b64encode(video_content).decode('utf-8')}"
- )
-
- prompt_messages = await assemble_claude_prompt_video(video_data_url)
-
- # Tell the model to continue
- # {"role": "assistant", "content": SECOND_MESSAGE},
- # {"role": "user", "content": "continue"},
-
- if is_followup:
- prompt_messages += [
- {"role": "assistant", "content": previous_html},
- {
- "role": "user",
- "content": "You've done a good job with a first draft. Improve this further based on the original instructions so that the app is fully functional like in the original video.",
- },
- ] # type: ignore
-
- async def process_chunk(content: str):
- print(content, end="", flush=True)
-
- response_prefix = "<thinking>"
-
- pprint_prompt(prompt_messages) # type: ignore
-
- start_time = time.time()
-
- completion = await stream_claude_response_native(
- system_prompt=VIDEO_PROMPT,
- messages=prompt_messages,
- api_key=ANTHROPIC_API_KEY,
- callback=lambda x: process_chunk(x),
- model_name=Llm.CLAUDE_3_OPUS.value,
- include_thinking=True,
- )
-
- end_time = time.time()
-
- # Prepend the response prefix to the completion
- completion = response_prefix + completion
-
- # Extract the outputs
- html_content = extract_tag_content("html", completion)
- thinking = extract_tag_content("thinking", completion)
-
- print(thinking)
- print(f"Operation took {end_time - start_time} seconds")
-
- os.makedirs(OUTPUTS_DIR, exist_ok=True)
-
- # Generate a unique filename based on the current time
- timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
- filename = f"video_test_output_{timestamp}.html"
- output_path = os.path.join(OUTPUTS_DIR, filename)
-
- # Write the HTML content to the file
- with open(output_path, "w") as file:
- file.write(html_content)
-
- print(f"Output file path: {output_path}")
-
- # Show a notification
- subprocess.run(["osascript", "-e", 'display notification "Coding Complete"'])
-
-
-asyncio.run(main())