Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Remove Claude Opus 3 video support, use only Gemini for video
Clauded865528937ca
diff --git a/backend/prompts/__init__.py b/backend/prompts/__init__.pyindex 58b5e4d3..fa381fc3 100644--- a/backend/prompts/__init__.py+++ b/backend/prompts/__init__.py@@ -7,7 +7,6 @@ from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTSfrom prompts.screenshot_system_prompts import SYSTEM_PROMPTSfrom prompts.text_prompts import SYSTEM_PROMPTS as TEXT_SYSTEM_PROMPTSfrom prompts.types import Stack, PromptContent-from video.utils import assemble_claude_prompt_videoUSER_PROMPT = """@@ -47,11 +46,11 @@ async def create_prompt(elif input_mode == "text":prompt_messages = assemble_text_prompt(prompt["text"], stack)elif input_mode == "video":- # For video mode initial creation, extract screenshots from video- # For updates, we use the history which already contains the generated code+ # For video mode initial creation, prompt is handled by VideoGenerationStage+ # which sends the video directly to Gemini with its own promptif generation_type == "create":- video_data_url = prompt["images"][0]- prompt_messages = await assemble_claude_prompt_video(video_data_url)+ # Return empty prompt - actual generation is handled by VideoGenerationStage+ prompt_messages = []else:# For video mode updates, use the screenshot system prompt# since we're now working with the generated code, not the videodiff --git a/backend/prompts/claude_prompts.py b/backend/prompts/claude_prompts.pyindex 6902995b..57f1690a 100644--- a/backend/prompts/claude_prompts.py+++ b/backend/prompts/claude_prompts.py@@ -1,56 +1,3 @@-# Not used yet-# References:-# https://github.com/hundredblocks/transcription_demo-# https://docs.anthropic.com/claude/docs/prompt-engineering-# https://github.com/anthropics/anthropic-cookbook/blob/main/multimodal/best_practices_for_vision.ipynb--VIDEO_PROMPT = """-You are an expert at building single page, funtional apps using HTML, Jquery and Tailwind CSS.-You also have perfect vision and pay great attention to detail.--You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.--- Make sure the app looks exactly like the screenshot.-- Pay close attention to background color, text color, font size, font family,-padding, margin, border, etc. Match the colors and sizes exactly.-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.-- If some fuctionality requires a backend call, just mock the data instead.-- MAKE THE APP FUNCTIONAL using Javascript. Allow the user to interact with the app and get the same behavior as the video.--In terms of libraries,--- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>-- You can use Google Fonts-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>-- Use jQuery: <script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>--Before generating the code for the app, think step-by-step: first, about the user flow depicated in the video and then about you how would you build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.-"""--VIDEO_PROMPT_ALPINE_JS = """-You are an expert at building single page, funtional apps using HTML, Alpine.js and Tailwind CSS.-You also have perfect vision and pay great attention to detail.--You will be given screenshots in order at consistent intervals from a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.--- Make sure the app looks exactly like the screenshot.-- Pay close attention to background color, text color, font size, font family,-padding, margin, border, etc. Match the colors and sizes exactly.-- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.-- If some fuctionality requires a backend call, just mock the data instead.-- MAKE THE APP FUNCTIONAL using Javascript. Allow the user to interact with the app and get the same behavior as the video.--In terms of libraries,--- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>-- You can use Google Fonts-- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>-- Use Alpine.js: <script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/cdn.min.js"></script>--Before generating the code for the app, think step-by-step: first, about the user flow depicated in the video and then about you how would you build it and how you would structure the code. Do the thinking within <thinking></thinking> tags. Then, provide your code within <html></html> tags.-"""--GEMINI_VIDEO_PROMPT = """You are an expert at building single page, functional apps using HTML, jQuery and Tailwind CSS.You also have perfect vision and pay great attention to detail.diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.pyindex f2446278..45db9766 100644--- a/backend/routes/generate_code.py+++ b/backend/routes/generate_code.py@@ -27,7 +27,6 @@ from llm import ()from models import (stream_claude_response,- stream_claude_response_native,stream_openai_response,stream_gemini_response,stream_gemini_response_video,@@ -61,7 +60,7 @@ MessageType = Literal[]from image_generation.core import generate_imagesfrom prompts import create_prompt-from prompts.claude_prompts import VIDEO_PROMPT, GEMINI_VIDEO_PROMPT+from prompts.claude_prompts import GEMINI_VIDEO_PROMPTfrom prompts.types import Stack, PromptContentfrom video.utils import get_video_bytes_and_mime_type@@ -465,7 +464,7 @@ class MockResponseStage:class VideoGenerationStage:- """Handles video mode code generation using Claude 3 Opus or Gemini 3"""+ """Handles video mode code generation using Gemini 3"""def __init__(self,@@ -477,28 +476,22 @@ class VideoGenerationStage:async def generate_video_code(self,- prompt_messages: List[ChatCompletionMessageParam],- anthropic_api_key: str | None,- gemini_api_key: str | None = None,+ gemini_api_key: str | None,video_data_url: str | None = None,) -> List[str]:- """Generate code for video input mode using Claude or Gemini"""- # Prefer Gemini if API key is available and video data URL is provided- if gemini_api_key and video_data_url:- return await self._generate_with_gemini(- video_data_url, gemini_api_key- )-- # Fall back to Claude- if not anthropic_api_key:+ """Generate code for video input mode using Gemini"""+ if not gemini_api_key:await self.throw_error(- "Video mode requires either a Gemini API key or an Anthropic API key. "- "Please add GEMINI_API_KEY or ANTHROPIC_API_KEY to backend/.env "- "or in the settings dialog"+ "Video mode requires a Gemini API key. "+ "Please add GEMINI_API_KEY to backend/.env or in the settings dialog")- raise Exception("No API key for video generation")+ raise Exception("No Gemini API key for video generation")++ if not video_data_url:+ await self.throw_error("Video mode requires a video to be provided")+ raise Exception("No video data URL provided")- return await self._generate_with_claude(prompt_messages, anthropic_api_key)+ return await self._generate_with_gemini(video_data_url, gemini_api_key)async def _generate_with_gemini(self,@@ -536,35 +529,6 @@ class VideoGenerationStage:return completions- async def _generate_with_claude(- self,- prompt_messages: List[ChatCompletionMessageParam],- anthropic_api_key: str,- ) -> List[str]:- """Generate code using Claude 3 Opus with screenshots extracted from video"""- async def process_chunk(content: str, variantIndex: int = 0):- await self.send_message("chunk", content, variantIndex)-- print("Using Claude 3 Opus for video generation (screenshot-based)")-- completion_results = [- await stream_claude_response_native(- system_prompt=VIDEO_PROMPT,- messages=prompt_messages, # type: ignore- api_key=anthropic_api_key,- callback=lambda x: process_chunk(x, 0),- model_name=Llm.CLAUDE_3_OPUS.value,- include_thinking=True,- )- ]- completions = [result["code"] for result in completion_results]-- # Send the complete variant back to the client- await self.send_message("setCode", completions[0], 0)- await self.send_message("variantComplete", "Variant generation complete", 0)-- return completions-class PostProcessingStage:"""Handles post-processing after code generation completes"""@@ -937,8 +901,6 @@ class CodeGenerationMiddleware(Middleware):# Get the video data URL from the promptvideo_data_url = context.extracted_params.prompt.get("images", [None])[0]context.completions = await video_stage.generate_video_code(- context.prompt_messages,- context.extracted_params.anthropic_api_key,gemini_api_key=GEMINI_API_KEY,video_data_url=video_data_url,)diff --git a/backend/tests/test_prompts.py b/backend/tests/test_prompts.pyindex b1e725bf..ea385164 100644--- a/backend/tests/test_prompts.py+++ b/backend/tests/test_prompts.py@@ -319,7 +319,12 @@ class TestCreatePrompt:@pytest.mark.asyncioasync def test_video_mode_basic_prompt_creation(self) -> None:- """Test basic video prompt creation in video mode."""+ """Test basic video prompt creation in video mode.++ For video mode with generation_type="create", the prompt is empty+ because the actual generation is handled by VideoGenerationStage+ which sends the video directly to Gemini.+ """# Setup test datavideo_data_url: str = "data:video/mp4;base64,test_video_data"params: Dict[str, Any] = {@@ -329,86 +334,26 @@ class TestCreatePrompt:},"generationType": "create"}-- # Mock the video processing function- mock_video_messages: List[Dict[str, Any]] = [- {- "role": "system",- "content": "Mock Video System Prompt"- },- {- "role": "user",- "content": [- {- "type": "image_url",- "image_url": {- "url": "data:image/png;base64,frame1",- "detail": "high"- }- },- {- "type": "image_url",- "image_url": {- "url": "data:image/png;base64,frame2",- "detail": "high"- }- },- {- "type": "text",- "text": "Create a functional web app from these video frames"- }- ]- }- ]-- with patch('prompts.assemble_claude_prompt_video', return_value=mock_video_messages):- # Call the function- messages, image_cache = await create_prompt(- stack=self.TEST_STACK,- input_mode="video",- generation_type=params["generationType"],- prompt=params["prompt"],- history=params.get("history", []),- is_imported_from_code=params.get("isImportedFromCode", False),- )-- # Define expected structure- expected: ExpectedResult = {- "messages": [- {- "role": "system",- "content": "Mock Video System Prompt"- },- {- "role": "user",- "content": [- {- "type": "image_url",- "image_url": {- "url": "data:image/png;base64,frame1",- "detail": "high"- }- },- {- "type": "image_url",- "image_url": {- "url": "data:image/png;base64,frame2",- "detail": "high"- }- },- {- "type": "text",- "text": "Create a functional web app from these video frames"- }- ]- }- ],- "image_cache": {}- }-- # Assert the structure matches- actual: ExpectedResult = {"messages": messages, "image_cache": image_cache}- assert_structure_match(actual, expected)++ # Call the function+ messages, image_cache = await create_prompt(+ stack=self.TEST_STACK,+ input_mode="video",+ generation_type=params["generationType"],+ prompt=params["prompt"],+ history=params.get("history", []),+ is_imported_from_code=params.get("isImportedFromCode", False),+ )++ # For video create mode, prompt is empty - actual generation handled by VideoGenerationStage+ expected: ExpectedResult = {+ "messages": [],+ "image_cache": {}+ }++ # Assert the structure matches+ actual: ExpectedResult = {"messages": messages, "image_cache": image_cache}+ assert_structure_match(actual, expected)@pytest.mark.asynciodiff --git a/backend/video/utils.py b/backend/video/utils.pyindex 6b6116e0..97880f30 100644--- a/backend/video/utils.py+++ b/backend/video/utils.py@@ -1,120 +1,4 @@-# Extract HTML content from the completion stringimport base64-import io-import mimetypes-import os-import tempfile-import uuid-from typing import Any, Union, cast-from moviepy.editor import VideoFileClip # type: ignore-from PIL import Image-import math---DEBUG = True-TARGET_NUM_SCREENSHOTS = (- 20 # Should be max that Claude supports (20) - reduce to save tokens on testing-)---async def assemble_claude_prompt_video(video_data_url: str) -> list[Any]:- images = split_video_into_screenshots(video_data_url)-- # Save images to tmp if we're debugging- if DEBUG:- save_images_to_tmp(images)-- # Validate number of images- print(f"Number of frames extracted from video: {len(images)}")- if len(images) > 20:- print(f"Too many screenshots: {len(images)}")- raise ValueError("Too many screenshots extracted from video")-- # Convert images to the message format for Claude- content_messages: list[dict[str, Union[dict[str, str], str]]] = []- for image in images:-- # Convert Image to buffer- buffered = io.BytesIO()- image.save(buffered, format="JPEG")-- # Encode bytes as base64- base64_data = base64.b64encode(buffered.getvalue()).decode("utf-8")- media_type = "image/jpeg"-- content_messages.append(- {- "type": "image",- "source": {- "type": "base64",- "media_type": media_type,- "data": base64_data,- },- }- )-- return [- {- "role": "user",- "content": content_messages,- },- ]---# Returns a list of images/frame (RGB format)-def split_video_into_screenshots(video_data_url: str) -> list[Image.Image]:- target_num_screenshots = TARGET_NUM_SCREENSHOTS-- # Decode the base64 URL to get the video bytes- video_encoded_data = video_data_url.split(",")[1]- video_bytes = base64.b64decode(video_encoded_data)-- mime_type = video_data_url.split(";")[0].split(":")[1]- suffix = mimetypes.guess_extension(mime_type)-- with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as temp_video_file:- print(temp_video_file.name)- temp_video_file.write(video_bytes)- temp_video_file.flush()- clip = VideoFileClip(temp_video_file.name)- images: list[Image.Image] = []- total_frames = cast(int, clip.reader.nframes) # type: ignore-- # Calculate frame skip interval by dividing total frames by the target number of screenshots- # Ensuring a minimum skip of 1 frame- frame_skip = max(1, math.ceil(total_frames / target_num_screenshots))-- # Iterate over each frame in the clip- for i, frame in enumerate(clip.iter_frames()):- # Save every nth frame- if i % frame_skip == 0:- frame_image = Image.fromarray(frame) # type: ignore- images.append(frame_image)- # Ensure that we don't capture more than the desired number of frames- if len(images) >= target_num_screenshots:- break-- # Close the video file to release resources- clip.close()-- return images---# Save a list of PIL images to a random temporary directory-def save_images_to_tmp(images: list[Image.Image]):-- # Create a unique temporary directory- unique_dir_name = f"screenshots_{uuid.uuid4()}"- tmp_screenshots_dir = os.path.join(tempfile.gettempdir(), unique_dir_name)- os.makedirs(tmp_screenshots_dir, exist_ok=True)-- for idx, image in enumerate(images):- # Generate a unique image filename using index- image_filename = f"screenshot_{idx}.jpg"- tmp_filepath = os.path.join(tmp_screenshots_dir, image_filename)- image.save(tmp_filepath, format="JPEG")-- print("Saved to " + tmp_screenshots_dir)def extract_tag_content(tag: str, text: str) -> str:diff --git a/backend/video_to_app.py b/backend/video_to_app.pydeleted file mode 100644index b8836f2f..00000000--- a/backend/video_to_app.py+++ /dev/null@@ -1,120 +0,0 @@-# Load environment variables first--from dotenv import load_dotenv--load_dotenv()---import base64-import mimetypes-import time-import subprocess-import os-import asyncio-from datetime import datetime-from prompts.claude_prompts import VIDEO_PROMPT-from utils import pprint_prompt-from config import ANTHROPIC_API_KEY-from video.utils import extract_tag_content, assemble_claude_prompt_video-from llm import Llm-from models import stream_claude_response_native--STACK = "html_tailwind"--VIDEO_DIR = "./video_evals/videos"-SCREENSHOTS_DIR = "./video_evals/screenshots"-OUTPUTS_DIR = "./video_evals/outputs"---async def main():- video_filename = "shortest.mov"- is_followup = False-- if not ANTHROPIC_API_KEY:- raise ValueError("ANTHROPIC_API_KEY is not set")-- # Get previous HTML- previous_html = ""- if is_followup:- previous_html_file = max(- [- os.path.join(OUTPUTS_DIR, f)- for f in os.listdir(OUTPUTS_DIR)- if f.endswith(".html")- ],- key=os.path.getctime,- )- with open(previous_html_file, "r") as file:- previous_html = file.read()-- video_file = os.path.join(VIDEO_DIR, video_filename)- mime_type = mimetypes.guess_type(video_file)[0]- with open(video_file, "rb") as file:- video_content = file.read()- video_data_url = (- f"data:{mime_type};base64,{base64.b64encode(video_content).decode('utf-8')}"- )-- prompt_messages = await assemble_claude_prompt_video(video_data_url)-- # Tell the model to continue- # {"role": "assistant", "content": SECOND_MESSAGE},- # {"role": "user", "content": "continue"},-- if is_followup:- prompt_messages += [- {"role": "assistant", "content": previous_html},- {- "role": "user",- "content": "You've done a good job with a first draft. Improve this further based on the original instructions so that the app is fully functional like in the original video.",- },- ] # type: ignore-- async def process_chunk(content: str):- print(content, end="", flush=True)-- response_prefix = "<thinking>"-- pprint_prompt(prompt_messages) # type: ignore-- start_time = time.time()-- completion = await stream_claude_response_native(- system_prompt=VIDEO_PROMPT,- messages=prompt_messages,- api_key=ANTHROPIC_API_KEY,- callback=lambda x: process_chunk(x),- model_name=Llm.CLAUDE_3_OPUS.value,- include_thinking=True,- )-- end_time = time.time()-- # Prepend the response prefix to the completion- completion = response_prefix + completion-- # Extract the outputs- html_content = extract_tag_content("html", completion)- thinking = extract_tag_content("thinking", completion)-- print(thinking)- print(f"Operation took {end_time - start_time} seconds")-- os.makedirs(OUTPUTS_DIR, exist_ok=True)-- # Generate a unique filename based on the current time- timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")- filename = f"video_test_output_{timestamp}.html"- output_path = os.path.join(OUTPUTS_DIR, filename)-- # Write the HTML content to the file- with open(output_path, "w") as file:- file.write(html_content)-- print(f"Output file path: {output_path}")-- # Show a notification- subprocess.run(["osascript", "-e", 'display notification "Coding Complete"'])---asyncio.run(main())