Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Refactor prompt builders by modality
Abi Raja92a8ee53c493
diff --git a/backend/evals/core.py b/backend/evals/core.pyindex ac19f6fb..f19240ab 100644--- a/backend/evals/core.py+++ b/backend/evals/core.py@@ -6,14 +6,18 @@ from config import ()from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELSfrom agent.runner import Agent-from prompts.builders import build_image_prompt_messages+from prompts.image_prompt_builder import build_image_prompt_messagesfrom prompts.prompt_types import Stackfrom openai.types.chat import ChatCompletionMessageParamfrom typing import Anyasync def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> str:- prompt_messages = build_image_prompt_messages([image_url], stack)+ prompt_messages = build_image_prompt_messages(+ image_data_urls=[image_url],+ stack=stack,+ text_prompt="",+ )async def send_message(_: str,__: str | None,diff --git a/backend/prompts/builders.py b/backend/prompts/builders.pyindex 70c00c74..5b9ebab7 100644--- a/backend/prompts/builders.py+++ b/backend/prompts/builders.py@@ -3,20 +3,11 @@ from typing import Any, castfrom openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParamfrom custom_types import InputMode+from prompts.image_prompt_builder import build_image_prompt_messagesfrom prompts.prompt_types import PromptContent, Stack-from prompts.system_prompt import SYSTEM_PROMPT---USER_PROMPT = """-Generate code for a web page that looks exactly like the provided screenshot(s).-If multiple screenshots are provided, organize them meaningfully. If they appear to be-different pages in a website, make them distinct pages and link them. If they look like-different tabs or views in an app, connect them with appropriate navigation. If they-appear unrelated, create a scaffold that separates them into "Screenshot 1", "Screenshot 2",-"Screenshot 3", etc. so it is easy to navigate.-For mobile screenshots, do not include the device frame or browser chrome; focus only on-the actual UI mockups.-"""+from prompts import system_prompt+from prompts.text_prompt_builder import build_text_prompt_messages+from prompts.video_prompt_builder import build_video_prompt_messagesIMPORTED_CODE_INSTRUCTIONS = """You are continuing from an imported codebase. Follow these instructions while updating it:@@ -49,9 +40,16 @@ async def build_prompt_messages(if input_mode == "image":image_urls = prompt.get("images", [])text_prompt = prompt.get("text", "")- prompt_messages = build_image_prompt_messages(image_urls, stack, text_prompt)+ prompt_messages = build_image_prompt_messages(+ image_data_urls=image_urls,+ stack=stack,+ text_prompt=text_prompt,+ )elif input_mode == "text":- prompt_messages = build_text_prompt_messages(prompt["text"], stack)+ prompt_messages = build_text_prompt_messages(+ text_prompt=prompt["text"],+ stack=stack,+ )elif input_mode == "video":if generation_type == "create":video_urls = prompt.get("images", [])@@ -66,13 +64,17 @@ async def build_prompt_messages(prompt_messages = [cast(ChatCompletionMessageParam,- {"role": "system", "content": SYSTEM_PROMPT},+ {"role": "system", "content": system_prompt.SYSTEM_PROMPT},)]else:image_urls = prompt.get("images", [])text_prompt = prompt.get("text", "")- prompt_messages = build_image_prompt_messages(image_urls, stack, text_prompt)+ prompt_messages = build_image_prompt_messages(+ image_data_urls=image_urls,+ stack=stack,+ text_prompt=text_prompt,+ )if generation_type == "update":for index, item in enumerate(history):@@ -127,7 +129,7 @@ def build_imported_code_prompt_messages(user_prompt: PromptContent,) -> list[ChatCompletionMessageParam]:system_content = (- SYSTEM_PROMPT.strip()+ system_prompt.SYSTEM_PROMPT.strip()+ "\n\n"+ IMPORTED_CODE_INSTRUCTIONS.strip()+ f"\n\nSelected stack: {stack}."@@ -195,109 +197,3 @@ def resolve_imported_code_user_prompt("text": "Update the imported code according to the latest request.","images": [],}---def build_image_prompt_messages(- image_data_urls: list[str],- stack: Stack,- text_prompt: str = "",-) -> list[ChatCompletionMessageParam]:- system_content = SYSTEM_PROMPT- user_prompt = USER_PROMPT.strip() + f"\n\nSelected stack: {stack}"-- if text_prompt.strip():- user_prompt = user_prompt + "\n\nAdditional instructions: " + text_prompt-- user_content: list[ChatCompletionContentPartParam] = []- for image_data_url in image_data_urls:- user_content.append(- {- "type": "image_url",- "image_url": {"url": image_data_url, "detail": "high"},- }- )- user_content.append(- {- "type": "text",- "text": user_prompt,- }- )- return [- {- "role": "system",- "content": system_content,- },- {- "role": "user",- "content": user_content,- },- ]---def build_text_prompt_messages(- text_prompt: str,- stack: Stack,-) -> list[ChatCompletionMessageParam]:- return [- {- "role": "system",- "content": SYSTEM_PROMPT,- },- {- "role": "user",- "content": f"""- - Make sure to make it look modern and sleek.- - Use modern, professional fonts and colors.- - Follow UX best practices.- Selected stack: {stack}.- Generate UI for {text_prompt}.- """,- },- ]---def build_video_prompt_messages(- video_data_url: str,- text_prompt: str = "",-) -> list[ChatCompletionMessageParam]:- user_text = """- You have been given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.-- - Watch the entire video carefully and understand all the user interactions and UI state changes.- - Make sure the app looks exactly like what you see in the video.- - Pay close attention to background color, text color, font size, font family,- padding, margin, border, etc. Match the colors and sizes exactly.- - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.- - Put image URLs in HTML (hide / unhide as needed). Do not put image URLs within JavaScript because our parser cannot extract them from JavaScript.- - If some functionality requires a backend call, just mock the data instead.- - MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.- - Use SVGs and interactive 3D elements if needed to match the functionality shown in the video.-- Analyze this video and generate the code.-- Selected stack: HTML, jQuery and Tailwind CSS.- """- if text_prompt.strip():- user_text = user_text + "\n\nAdditional instructions: " + text_prompt-- user_content: list[ChatCompletionContentPartParam] = [- {- "type": "image_url",- "image_url": {"url": video_data_url, "detail": "high"},- },- {- "type": "text",- "text": user_text,- },- ]-- return [- {- "role": "system",- "content": SYSTEM_PROMPT,- },- {- "role": "user",- "content": user_content,- },- ]diff --git a/backend/prompts/image_prompt_builder.py b/backend/prompts/image_prompt_builder.pynew file mode 100644index 00000000..af147189--- /dev/null+++ b/backend/prompts/image_prompt_builder.py@@ -0,0 +1,50 @@+from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam++from prompts.prompt_types import Stack+from prompts import system_prompt++def build_image_prompt_messages(+ image_data_urls: list[str],+ stack: Stack,+ text_prompt: str,+) -> list[ChatCompletionMessageParam]:+ user_prompt = f"""+Generate code for a web page that looks exactly like the provided screenshot(s).+If multiple screenshots are provided, organize them meaningfully. If they appear to be+different pages in a website, make them distinct pages and link them. If they look like+different tabs or views in an app, connect them with appropriate navigation. If they+appear unrelated, create a scaffold that separates them into "Screenshot 1", "Screenshot 2",+"Screenshot 3", etc. so it is easy to navigate.+For mobile screenshots, do not include the device frame or browser chrome; focus only on+the actual UI mockups.++Selected stack: {stack}+""".strip()++ if text_prompt.strip():+ user_prompt = f"{user_prompt}\n\nAdditional instructions: {text_prompt}"++ user_content: list[ChatCompletionContentPartParam] = []+ for image_data_url in image_data_urls:+ user_content.append(+ {+ "type": "image_url",+ "image_url": {"url": image_data_url, "detail": "high"},+ }+ )+ user_content.append(+ {+ "type": "text",+ "text": user_prompt,+ }+ )+ return [+ {+ "role": "system",+ "content": system_prompt.SYSTEM_PROMPT,+ },+ {+ "role": "user",+ "content": user_content,+ },+ ]diff --git a/backend/prompts/text_prompt_builder.py b/backend/prompts/text_prompt_builder.pynew file mode 100644index 00000000..525d2f74--- /dev/null+++ b/backend/prompts/text_prompt_builder.py@@ -0,0 +1,26 @@+from openai.types.chat import ChatCompletionMessageParam++from prompts.prompt_types import Stack+from prompts import system_prompt+++def build_text_prompt_messages(+ text_prompt: str,+ stack: Stack,+) -> list[ChatCompletionMessageParam]:+ return [+ {+ "role": "system",+ "content": system_prompt.SYSTEM_PROMPT,+ },+ {+ "role": "user",+ "content": f"""+ - Make sure to make it look modern and sleek.+ - Use modern, professional fonts and colors.+ - Follow UX best practices.+ Selected stack: {stack}.+ Generate UI for {text_prompt}.+ """,+ },+ ]diff --git a/backend/prompts/video_prompt_builder.py b/backend/prompts/video_prompt_builder.pynew file mode 100644index 00000000..204a3121--- /dev/null+++ b/backend/prompts/video_prompt_builder.py@@ -0,0 +1,49 @@+from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam+from prompts import system_prompt+++def build_video_prompt_messages(+ video_data_url: str,+ text_prompt: str,+) -> list[ChatCompletionMessageParam]:+ user_text = """+ You have been given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.++ - Watch the entire video carefully and understand all the user interactions and UI state changes.+ - Make sure the app looks exactly like what you see in the video.+ - Pay close attention to background color, text color, font size, font family,+ padding, margin, border, etc. Match the colors and sizes exactly.+ - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.+ - Put image URLs in HTML (hide / unhide as needed). Do not put image URLs within JavaScript because our parser cannot extract them from JavaScript.+ - If some functionality requires a backend call, just mock the data instead.+ - MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.+ - Use SVGs and interactive 3D elements if needed to match the functionality shown in the video.++ Analyze this video and generate the code.++ Selected stack: HTML, jQuery and Tailwind CSS.+ """+ if text_prompt.strip():+ user_text = user_text + "\n\nAdditional instructions: " + text_prompt++ user_content: list[ChatCompletionContentPartParam] = [+ {+ "type": "image_url",+ "image_url": {"url": video_data_url, "detail": "high"},+ },+ {+ "type": "text",+ "text": user_text,+ },+ ]++ return [+ {+ "role": "system",+ "content": system_prompt.SYSTEM_PROMPT,+ },+ {+ "role": "user",+ "content": user_content,+ },+ ]diff --git a/backend/tests/test_prompts.py b/backend/tests/test_prompts.pyindex 83058bd9..66222ecd 100644--- a/backend/tests/test_prompts.py+++ b/backend/tests/test_prompts.py@@ -91,7 +91,7 @@ class TestCreatePrompt:"generationType": "create",}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -145,7 +145,7 @@ class TestCreatePrompt:],}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -199,7 +199,7 @@ class TestCreatePrompt:},"generationType": "create"}- with patch('prompts.builders.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):+ with patch('prompts.system_prompt.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -246,7 +246,7 @@ class TestCreatePrompt:{"text": "Now add a navigation menu"} # User's new request]}- with patch('prompts.builders.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):+ with patch('prompts.system_prompt.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -361,7 +361,7 @@ class TestCreatePrompt:]}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -433,7 +433,7 @@ class TestCreatePrompt:]}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -510,7 +510,7 @@ class TestCreatePrompt:]}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -566,7 +566,7 @@ class TestCreatePrompt:]}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -620,7 +620,7 @@ class TestCreatePrompt:],}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):messages = await build_prompt_messages(stack=self.TEST_STACK,input_mode="image",diff --git a/backend/tests/test_prompts_additional.py b/backend/tests/test_prompts_additional.pyindex cd24a4d2..502f6f3c 100644--- a/backend/tests/test_prompts_additional.py+++ b/backend/tests/test_prompts_additional.py@@ -97,7 +97,7 @@ class TestCreatePromptImageSupport:]}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -174,7 +174,7 @@ class TestCreatePromptImageSupport:]}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -230,7 +230,7 @@ class TestCreatePromptImageSupport:]}- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,