Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Refactor prompt builders by modality

Abi Raja92a8ee53c493
diff --git a/backend/evals/core.py b/backend/evals/core.py
index ac19f6fb..f19240ab 100644
--- a/backend/evals/core.py
+++ b/backend/evals/core.py
@@ -6,14 +6,18 @@ from config import (
)
from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS
from agent.runner import Agent
-from prompts.builders import build_image_prompt_messages
+from prompts.image_prompt_builder import build_image_prompt_messages
from prompts.prompt_types import Stack
from openai.types.chat import ChatCompletionMessageParam
from typing import Any
async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> str:
- prompt_messages = build_image_prompt_messages([image_url], stack)
+ prompt_messages = build_image_prompt_messages(
+ image_data_urls=[image_url],
+ stack=stack,
+ text_prompt="",
+ )
async def send_message(
_: str,
__: str | None,
diff --git a/backend/prompts/builders.py b/backend/prompts/builders.py
index 70c00c74..5b9ebab7 100644
--- a/backend/prompts/builders.py
+++ b/backend/prompts/builders.py
@@ -3,20 +3,11 @@ from typing import Any, cast
from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam
from custom_types import InputMode
+from prompts.image_prompt_builder import build_image_prompt_messages
from prompts.prompt_types import PromptContent, Stack
-from prompts.system_prompt import SYSTEM_PROMPT
-
-
-USER_PROMPT = """
-Generate code for a web page that looks exactly like the provided screenshot(s).
-If multiple screenshots are provided, organize them meaningfully. If they appear to be
-different pages in a website, make them distinct pages and link them. If they look like
-different tabs or views in an app, connect them with appropriate navigation. If they
-appear unrelated, create a scaffold that separates them into "Screenshot 1", "Screenshot 2",
-"Screenshot 3", etc. so it is easy to navigate.
-For mobile screenshots, do not include the device frame or browser chrome; focus only on
-the actual UI mockups.
-"""
+from prompts import system_prompt
+from prompts.text_prompt_builder import build_text_prompt_messages
+from prompts.video_prompt_builder import build_video_prompt_messages
IMPORTED_CODE_INSTRUCTIONS = """
You are continuing from an imported codebase. Follow these instructions while updating it:
@@ -49,9 +40,16 @@ async def build_prompt_messages(
if input_mode == "image":
image_urls = prompt.get("images", [])
text_prompt = prompt.get("text", "")
- prompt_messages = build_image_prompt_messages(image_urls, stack, text_prompt)
+ prompt_messages = build_image_prompt_messages(
+ image_data_urls=image_urls,
+ stack=stack,
+ text_prompt=text_prompt,
+ )
elif input_mode == "text":
- prompt_messages = build_text_prompt_messages(prompt["text"], stack)
+ prompt_messages = build_text_prompt_messages(
+ text_prompt=prompt["text"],
+ stack=stack,
+ )
elif input_mode == "video":
if generation_type == "create":
video_urls = prompt.get("images", [])
@@ -66,13 +64,17 @@ async def build_prompt_messages(
prompt_messages = [
cast(
ChatCompletionMessageParam,
- {"role": "system", "content": SYSTEM_PROMPT},
+ {"role": "system", "content": system_prompt.SYSTEM_PROMPT},
)
]
else:
image_urls = prompt.get("images", [])
text_prompt = prompt.get("text", "")
- prompt_messages = build_image_prompt_messages(image_urls, stack, text_prompt)
+ prompt_messages = build_image_prompt_messages(
+ image_data_urls=image_urls,
+ stack=stack,
+ text_prompt=text_prompt,
+ )
if generation_type == "update":
for index, item in enumerate(history):
@@ -127,7 +129,7 @@ def build_imported_code_prompt_messages(
user_prompt: PromptContent,
) -> list[ChatCompletionMessageParam]:
system_content = (
- SYSTEM_PROMPT.strip()
+ system_prompt.SYSTEM_PROMPT.strip()
+ "\n\n"
+ IMPORTED_CODE_INSTRUCTIONS.strip()
+ f"\n\nSelected stack: {stack}."
@@ -195,109 +197,3 @@ def resolve_imported_code_user_prompt(
"text": "Update the imported code according to the latest request.",
"images": [],
}
-
-
-def build_image_prompt_messages(
- image_data_urls: list[str],
- stack: Stack,
- text_prompt: str = "",
-) -> list[ChatCompletionMessageParam]:
- system_content = SYSTEM_PROMPT
- user_prompt = USER_PROMPT.strip() + f"\n\nSelected stack: {stack}"
-
- if text_prompt.strip():
- user_prompt = user_prompt + "\n\nAdditional instructions: " + text_prompt
-
- user_content: list[ChatCompletionContentPartParam] = []
- for image_data_url in image_data_urls:
- user_content.append(
- {
- "type": "image_url",
- "image_url": {"url": image_data_url, "detail": "high"},
- }
- )
- user_content.append(
- {
- "type": "text",
- "text": user_prompt,
- }
- )
- return [
- {
- "role": "system",
- "content": system_content,
- },
- {
- "role": "user",
- "content": user_content,
- },
- ]
-
-
-def build_text_prompt_messages(
- text_prompt: str,
- stack: Stack,
-) -> list[ChatCompletionMessageParam]:
- return [
- {
- "role": "system",
- "content": SYSTEM_PROMPT,
- },
- {
- "role": "user",
- "content": f"""
- - Make sure to make it look modern and sleek.
- - Use modern, professional fonts and colors.
- - Follow UX best practices.
- Selected stack: {stack}.
- Generate UI for {text_prompt}.
- """,
- },
- ]
-
-
-def build_video_prompt_messages(
- video_data_url: str,
- text_prompt: str = "",
-) -> list[ChatCompletionMessageParam]:
- user_text = """
- You have been given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
-
- - Watch the entire video carefully and understand all the user interactions and UI state changes.
- - Make sure the app looks exactly like what you see in the video.
- - Pay close attention to background color, text color, font size, font family,
- padding, margin, border, etc. Match the colors and sizes exactly.
- - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
- - Put image URLs in HTML (hide / unhide as needed). Do not put image URLs within JavaScript because our parser cannot extract them from JavaScript.
- - If some functionality requires a backend call, just mock the data instead.
- - MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.
- - Use SVGs and interactive 3D elements if needed to match the functionality shown in the video.
-
- Analyze this video and generate the code.
-
- Selected stack: HTML, jQuery and Tailwind CSS.
- """
- if text_prompt.strip():
- user_text = user_text + "\n\nAdditional instructions: " + text_prompt
-
- user_content: list[ChatCompletionContentPartParam] = [
- {
- "type": "image_url",
- "image_url": {"url": video_data_url, "detail": "high"},
- },
- {
- "type": "text",
- "text": user_text,
- },
- ]
-
- return [
- {
- "role": "system",
- "content": SYSTEM_PROMPT,
- },
- {
- "role": "user",
- "content": user_content,
- },
- ]
diff --git a/backend/prompts/image_prompt_builder.py b/backend/prompts/image_prompt_builder.py
new file mode 100644
index 00000000..af147189
--- /dev/null
+++ b/backend/prompts/image_prompt_builder.py
@@ -0,0 +1,50 @@
+from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam
+
+from prompts.prompt_types import Stack
+from prompts import system_prompt
+
+def build_image_prompt_messages(
+ image_data_urls: list[str],
+ stack: Stack,
+ text_prompt: str,
+) -> list[ChatCompletionMessageParam]:
+ user_prompt = f"""
+Generate code for a web page that looks exactly like the provided screenshot(s).
+If multiple screenshots are provided, organize them meaningfully. If they appear to be
+different pages in a website, make them distinct pages and link them. If they look like
+different tabs or views in an app, connect them with appropriate navigation. If they
+appear unrelated, create a scaffold that separates them into "Screenshot 1", "Screenshot 2",
+"Screenshot 3", etc. so it is easy to navigate.
+For mobile screenshots, do not include the device frame or browser chrome; focus only on
+the actual UI mockups.
+
+Selected stack: {stack}
+""".strip()
+
+ if text_prompt.strip():
+ user_prompt = f"{user_prompt}\n\nAdditional instructions: {text_prompt}"
+
+ user_content: list[ChatCompletionContentPartParam] = []
+ for image_data_url in image_data_urls:
+ user_content.append(
+ {
+ "type": "image_url",
+ "image_url": {"url": image_data_url, "detail": "high"},
+ }
+ )
+ user_content.append(
+ {
+ "type": "text",
+ "text": user_prompt,
+ }
+ )
+ return [
+ {
+ "role": "system",
+ "content": system_prompt.SYSTEM_PROMPT,
+ },
+ {
+ "role": "user",
+ "content": user_content,
+ },
+ ]
diff --git a/backend/prompts/text_prompt_builder.py b/backend/prompts/text_prompt_builder.py
new file mode 100644
index 00000000..525d2f74
--- /dev/null
+++ b/backend/prompts/text_prompt_builder.py
@@ -0,0 +1,26 @@
+from openai.types.chat import ChatCompletionMessageParam
+
+from prompts.prompt_types import Stack
+from prompts import system_prompt
+
+
+def build_text_prompt_messages(
+ text_prompt: str,
+ stack: Stack,
+) -> list[ChatCompletionMessageParam]:
+ return [
+ {
+ "role": "system",
+ "content": system_prompt.SYSTEM_PROMPT,
+ },
+ {
+ "role": "user",
+ "content": f"""
+ - Make sure to make it look modern and sleek.
+ - Use modern, professional fonts and colors.
+ - Follow UX best practices.
+ Selected stack: {stack}.
+ Generate UI for {text_prompt}.
+ """,
+ },
+ ]
diff --git a/backend/prompts/video_prompt_builder.py b/backend/prompts/video_prompt_builder.py
new file mode 100644
index 00000000..204a3121
--- /dev/null
+++ b/backend/prompts/video_prompt_builder.py
@@ -0,0 +1,49 @@
+from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParam
+from prompts import system_prompt
+
+
+def build_video_prompt_messages(
+ video_data_url: str,
+ text_prompt: str,
+) -> list[ChatCompletionMessageParam]:
+ user_text = """
+ You have been given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.
+
+ - Watch the entire video carefully and understand all the user interactions and UI state changes.
+ - Make sure the app looks exactly like what you see in the video.
+ - Pay close attention to background color, text color, font size, font family,
+ padding, margin, border, etc. Match the colors and sizes exactly.
+ - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.
+ - Put image URLs in HTML (hide / unhide as needed). Do not put image URLs within JavaScript because our parser cannot extract them from JavaScript.
+ - If some functionality requires a backend call, just mock the data instead.
+ - MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.
+ - Use SVGs and interactive 3D elements if needed to match the functionality shown in the video.
+
+ Analyze this video and generate the code.
+
+ Selected stack: HTML, jQuery and Tailwind CSS.
+ """
+ if text_prompt.strip():
+ user_text = user_text + "\n\nAdditional instructions: " + text_prompt
+
+ user_content: list[ChatCompletionContentPartParam] = [
+ {
+ "type": "image_url",
+ "image_url": {"url": video_data_url, "detail": "high"},
+ },
+ {
+ "type": "text",
+ "text": user_text,
+ },
+ ]
+
+ return [
+ {
+ "role": "system",
+ "content": system_prompt.SYSTEM_PROMPT,
+ },
+ {
+ "role": "user",
+ "content": user_content,
+ },
+ ]
diff --git a/backend/tests/test_prompts.py b/backend/tests/test_prompts.py
index 83058bd9..66222ecd 100644
--- a/backend/tests/test_prompts.py
+++ b/backend/tests/test_prompts.py
@@ -91,7 +91,7 @@ class TestCreatePrompt:
"generationType": "create",
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -145,7 +145,7 @@ class TestCreatePrompt:
],
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -199,7 +199,7 @@ class TestCreatePrompt:
},
"generationType": "create"
}
- with patch('prompts.builders.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):
+ with patch('prompts.system_prompt.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -246,7 +246,7 @@ class TestCreatePrompt:
{"text": "Now add a navigation menu"} # User's new request
]
}
- with patch('prompts.builders.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):
+ with patch('prompts.system_prompt.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -361,7 +361,7 @@ class TestCreatePrompt:
]
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -433,7 +433,7 @@ class TestCreatePrompt:
]
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -510,7 +510,7 @@ class TestCreatePrompt:
]
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -566,7 +566,7 @@ class TestCreatePrompt:
]
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -620,7 +620,7 @@ class TestCreatePrompt:
],
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
messages = await build_prompt_messages(
stack=self.TEST_STACK,
input_mode="image",
diff --git a/backend/tests/test_prompts_additional.py b/backend/tests/test_prompts_additional.py
index cd24a4d2..502f6f3c 100644
--- a/backend/tests/test_prompts_additional.py
+++ b/backend/tests/test_prompts_additional.py
@@ -97,7 +97,7 @@ class TestCreatePromptImageSupport:
]
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -174,7 +174,7 @@ class TestCreatePromptImageSupport:
]
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,
@@ -230,7 +230,7 @@ class TestCreatePromptImageSupport:
]
}
- with patch("prompts.builders.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
+ with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):
# Call the function
messages = await build_prompt_messages(
stack=self.TEST_STACK,