Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Update agent image generation policy
Abi Raja55e1e8253efe
diff --git a/backend/agent/runner.py b/backend/agent/runner.pyindex 758391e0..788a9fa2 100644--- a/backend/agent/runner.py+++ b/backend/agent/runner.py@@ -59,6 +59,7 @@ class Agent:self.openai_base_url = openai_base_urlself.anthropic_api_key = anthropic_api_keyself.gemini_api_key = gemini_api_key+ self.should_generate_images = should_generate_imagesself.file_state = AgentFileState()if initial_file_state and initial_file_state.get("content"):@@ -262,7 +263,9 @@ class Agent:raise Exception("OpenAI API key is missing.")client = AsyncOpenAI(api_key=self.openai_api_key, base_url=self.openai_base_url)- canonical_tools = canonical_tool_definitions()+ canonical_tools = canonical_tool_definitions(+ image_generation_enabled=self.should_generate_images+ )try:tools = serialize_openai_responses_tools(canonical_tools)@@ -285,7 +288,9 @@ class Agent:raise Exception("Anthropic API key is missing.")client = AsyncAnthropic(api_key=self.anthropic_api_key)- canonical_tools = canonical_tool_definitions()+ canonical_tools = canonical_tool_definitions(+ image_generation_enabled=self.should_generate_images+ )tools = serialize_anthropic_tools(canonical_tools)try:@@ -308,7 +313,9 @@ class Agent:raise Exception("Gemini API key is missing.")client = genai.Client(api_key=self.gemini_api_key)- canonical_tools = canonical_tool_definitions()+ canonical_tools = canonical_tool_definitions(+ image_generation_enabled=self.should_generate_images+ )tools = serialize_gemini_tools(canonical_tools)adapter: ProviderAdapter = GeminiAdapter(client=client,diff --git a/backend/agent/tools.py b/backend/agent/tools.pyindex b9ccfaab..c94bf5c4 100644--- a/backend/agent/tools.py+++ b/backend/agent/tools.py@@ -230,8 +230,10 @@ def _retrieve_option_schema() -> Dict[str, Any]:}-def canonical_tool_definitions() -> List[CanonicalToolDefinition]:- return [+def canonical_tool_definitions(+ image_generation_enabled: bool = True,+) -> List[CanonicalToolDefinition]:+ tools: List[CanonicalToolDefinition] = [CanonicalToolDefinition(name="create_file",description=(@@ -248,31 +250,39 @@ def canonical_tool_definitions() -> List[CanonicalToolDefinition]:),parameters=_edit_schema(),),- CanonicalToolDefinition(- name="generate_images",- description=(- "Generate image URLs from prompts. Use to replace placeholder images. "- "You can pass multiple prompts at once."- ),- parameters=_image_schema(),- ),- CanonicalToolDefinition(- name="remove_background",- description=(- "Remove the background from an image. Returns a URL to the processed "- "image with transparent background."+ ]+ if image_generation_enabled:+ tools.append(+ CanonicalToolDefinition(+ name="generate_images",+ description=(+ "Generate image URLs from prompts. Use to replace placeholder images. "+ "You can pass multiple prompts at once."+ ),+ parameters=_image_schema(),+ )+ )+ tools.extend(+ [+ CanonicalToolDefinition(+ name="remove_background",+ description=(+ "Remove the background from an image. Returns a URL to the processed "+ "image with transparent background."+ ),+ parameters=_remove_background_schema(),),- parameters=_remove_background_schema(),- ),- CanonicalToolDefinition(- name="retrieve_option",- description=(- "Retrieve the full HTML for a specific option (variant) so you can "- "reference it."+ CanonicalToolDefinition(+ name="retrieve_option",+ description=(+ "Retrieve the full HTML for a specific option (variant) so you can "+ "reference it."+ ),+ parameters=_retrieve_option_schema(),),- parameters=_retrieve_option_schema(),- ),- ]+ ]+ )+ return toolsdef _copy_schema(schema: Dict[str, Any]) -> Dict[str, Any]:diff --git a/backend/evals/core.py b/backend/evals/core.pyindex 00822490..eaba7189 100644--- a/backend/evals/core.py+++ b/backend/evals/core.py@@ -17,6 +17,7 @@ async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> simage_data_urls=[image_url],stack=stack,text_prompt="",+ image_generation_enabled=True,)async def send_message(_: str,diff --git a/backend/prompts/create/__init__.py b/backend/prompts/create/__init__.pyindex a3520be8..58ab525a 100644--- a/backend/prompts/create/__init__.py+++ b/backend/prompts/create/__init__.py@@ -10,6 +10,7 @@ def build_create_prompt_from_input(input_mode: InputMode,stack: Stack,prompt: UserTurnInput,+ image_generation_enabled: bool,) -> Prompt:if input_mode == "image":image_urls = prompt.get("images", [])@@ -18,11 +19,13 @@ def build_create_prompt_from_input(image_data_urls=image_urls,stack=stack,text_prompt=text_prompt,+ image_generation_enabled=image_generation_enabled,)if input_mode == "text":return build_text_prompt_messages(text_prompt=prompt["text"],stack=stack,+ image_generation_enabled=image_generation_enabled,)if input_mode == "video":video_urls = prompt.get("videos", [])@@ -33,6 +36,7 @@ def build_create_prompt_from_input(video_data_url=video_url,stack=stack,text_prompt=prompt.get("text", ""),+ image_generation_enabled=image_generation_enabled,)raise ValueError(f"Unsupported input mode: {input_mode}")diff --git a/backend/prompts/create/image.py b/backend/prompts/create/image.pyindex 82f05c97..0bed6900 100644--- a/backend/prompts/create/image.py+++ b/backend/prompts/create/image.py@@ -2,12 +2,15 @@ from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessfrom prompts.prompt_types import Stackfrom prompts import system_prompt+from prompts.policies import build_user_image_policydef build_image_prompt_messages(image_data_urls: list[str],stack: Stack,text_prompt: str,+ image_generation_enabled: bool,) -> list[ChatCompletionMessageParam]:+ image_policy = build_user_image_policy(image_generation_enabled)user_prompt = f"""Generate code for a web page that looks exactly like the provided screenshot(s).@@ -20,7 +23,7 @@ Selected stack: {stack}padding, margin, border, etc. Match the colors and sizes exactly.- Use the exact text from the screenshot.- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.-- For images, generate images using the generate_images tool.+- {image_policy}## Multiple screenshotsdiff --git a/backend/prompts/create/text.py b/backend/prompts/create/text.pyindex 9e67d010..419e49c6 100644--- a/backend/prompts/create/text.py+++ b/backend/prompts/create/text.py@@ -2,12 +2,15 @@ from openai.types.chat import ChatCompletionMessageParamfrom prompts.prompt_types import Stackfrom prompts import system_prompt+from prompts.policies import build_user_image_policydef build_text_prompt_messages(text_prompt: str,stack: Stack,+ image_generation_enabled: bool,) -> list[ChatCompletionMessageParam]:+ image_policy = build_user_image_policy(image_generation_enabled)USER_PROMPT = f"""Generate UI for {text_prompt}.@@ -17,7 +20,8 @@ Selected stack: {stack}.- Make sure to make it look modern and sleek.- Use modern, professional fonts and colors.-- Follow UX best practices."""+- Follow UX best practices.+- {image_policy}"""return [{diff --git a/backend/prompts/create/video.py b/backend/prompts/create/video.pyindex 9691c1f2..26b4ee4e 100644--- a/backend/prompts/create/video.py+++ b/backend/prompts/create/video.py@@ -1,13 +1,16 @@from openai.types.chat import ChatCompletionContentPartParam, ChatCompletionMessageParamfrom prompts.prompt_types import Stackfrom prompts import system_prompt+from prompts.policies import build_user_image_policydef build_video_prompt_messages(video_data_url: str,stack: Stack,text_prompt: str,+ image_generation_enabled: bool,) -> list[ChatCompletionMessageParam]:+ image_policy = build_user_image_policy(image_generation_enabled)user_text = f"""You have been given a video of a user interacting with a web app. You need to re-create the same app exactly such that the same user interactions will produce the same results in the app you build.@@ -15,7 +18,7 @@ def build_video_prompt_messages(- Make sure the app looks exactly like what you see in the video.- Pay close attention to background color, text color, font size, font family,padding, margin, border, etc. Match the colors and sizes exactly.- - If there are images in the app, generate images using the generate_images tool.+ - {image_policy}- If some functionality requires a backend call, just mock the data instead.- MAKE THE APP FUNCTIONAL using JavaScript. Allow the user to interact with the app and get the same behavior as shown in the video.- Use SVGs and interactive 3D elements if needed to match the functionality shown in the video.diff --git a/backend/prompts/pipeline.py b/backend/prompts/pipeline.pyindex 3035e406..3d3b3eba 100644--- a/backend/prompts/pipeline.py+++ b/backend/prompts/pipeline.py@@ -16,6 +16,7 @@ async def build_prompt_messages(prompt: UserTurnInput,history: list[PromptHistoryMessage],file_state: dict[str, str] | None = None,+ image_generation_enabled: bool = True,) -> Prompt:plan = derive_prompt_construction_plan(stack=stack,@@ -27,12 +28,22 @@ async def build_prompt_messages(strategy = plan["construction_strategy"]if strategy == "update_from_history":- return build_update_prompt_from_history(stack=stack, history=history)+ return build_update_prompt_from_history(+ stack=stack,+ history=history,+ image_generation_enabled=image_generation_enabled,+ )if strategy == "update_from_file_snapshot":assert file_state is not Nonereturn build_update_prompt_from_file_snapshot(stack=stack,prompt=prompt,file_state=file_state,+ image_generation_enabled=image_generation_enabled,)- return build_create_prompt_from_input(input_mode, stack, prompt)+ return build_create_prompt_from_input(+ input_mode,+ stack,+ prompt,+ image_generation_enabled,+ )diff --git a/backend/prompts/policies.py b/backend/prompts/policies.pynew file mode 100644index 00000000..2d71fe85--- /dev/null+++ b/backend/prompts/policies.py@@ -0,0 +1,11 @@+def build_user_image_policy(image_generation_enabled: bool) -> str:+ if image_generation_enabled:+ return (+ "Image generation is enabled for this request. Use generate_images for "+ "missing assets when needed."+ )++ return (+ "Image generation is disabled for this request. Do not call generate_images. "+ "Use provided media, CSS effects, or placeholder URLs (https://placehold.co)."+ )diff --git a/backend/prompts/system_prompt.py b/backend/prompts/system_prompt.pyindex b24c18b3..c5791f55 100644--- a/backend/prompts/system_prompt.py+++ b/backend/prompts/system_prompt.py@@ -9,12 +9,12 @@ You are a coding agent that's an expert at building front-ends.# Tooling instructions-- You have access to tools: create_file, edit_file, generate_images, remove_background, retrieve_option.+- You have access to tools for file creation, file editing, image handling, and option retrieval.- The main file is a single HTML file. Use path "index.html" unless told otherwise.- For a brand new app, call create_file exactly once with the full HTML.- For updates, call edit_file using exact string replacements. Do NOT regenerate the entire file.- Do not output raw HTML in chat. Any code changes must go through tools.-- Use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background.+- When available, use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background.- Use remove_background to remove the background from a provided image URL when needed.- Use retrieve_option to fetch the full HTML for a specific option (1-based option_number) when a user references another option.diff --git a/backend/prompts/update/from_file_snapshot.py b/backend/prompts/update/from_file_snapshot.pyindex a8a43fbc..276c437c 100644--- a/backend/prompts/update/from_file_snapshot.py+++ b/backend/prompts/update/from_file_snapshot.py@@ -3,6 +3,7 @@ from typing import castfrom openai.types.chat import ChatCompletionMessageParamfrom prompts import system_prompt+from prompts.policies import build_user_image_policyfrom prompts.prompt_types import Stack, UserTurnInputfrom prompts.message_builder import Prompt, build_history_message@@ -11,11 +12,15 @@ def build_update_prompt_from_file_snapshot(stack: Stack,prompt: UserTurnInput,file_state: dict[str, str],+ image_generation_enabled: bool,) -> Prompt:path = file_state.get("path", "index.html")request_text = prompt.get("text", "").strip() or "Apply the requested update."+ image_policy = build_user_image_policy(image_generation_enabled)bootstrap_text = f"""Selected stack: {stack}.+{image_policy}+You are editing an existing file.<current_file path="{path}">diff --git a/backend/prompts/update/from_history.py b/backend/prompts/update/from_history.pyindex 157ee18f..8329989d 100644--- a/backend/prompts/update/from_history.py+++ b/backend/prompts/update/from_history.py@@ -3,6 +3,7 @@ from typing import castfrom openai.types.chat import ChatCompletionMessageParamfrom prompts import system_prompt+from prompts.policies import build_user_image_policyfrom prompts.prompt_types import PromptHistoryMessage, Stackfrom prompts.message_builder import Prompt, build_history_message@@ -10,6 +11,7 @@ from prompts.message_builder import Prompt, build_history_messagedef build_update_prompt_from_history(stack: Stack,history: list[PromptHistoryMessage],+ image_generation_enabled: bool,) -> Prompt:first_user_index = next((index for index, item in enumerate(history) if item["role"] == "user"),@@ -27,9 +29,12 @@ def build_update_prompt_from_history(},)]+ image_policy = build_user_image_policy(image_generation_enabled)for index, item in enumerate(history):if index == first_user_index:- stack_prefix = f"Selected stack: {stack}."+ stack_prefix = f"""Selected stack: {stack}.++{image_policy}"""user_text = item.get("text", "")prefixed_text = (f"{stack_prefix}\n\n{user_text}" if user_text.strip() else stack_prefixdiff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.pyindex 3675f8e6..f0575fbf 100644--- a/backend/routes/generate_code.py+++ b/backend/routes/generate_code.py@@ -466,6 +466,7 @@ class PromptCreationStage:prompt=extracted_params.prompt,history=extracted_params.history,file_state=extracted_params.file_state,+ image_generation_enabled=extracted_params.should_generate_images,)print_prompt_preview(prompt_messages)diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.pynew file mode 100644index 00000000..e85e1fa5--- /dev/null+++ b/backend/tests/test_agent_tools.py@@ -0,0 +1,11 @@+from agent.tools import canonical_tool_definitions+++def test_canonical_tool_definitions_include_generate_images_when_enabled() -> None:+ tool_names = [tool.name for tool in canonical_tool_definitions(True)]+ assert "generate_images" in tool_names+++def test_canonical_tool_definitions_exclude_generate_images_when_disabled() -> None:+ tool_names = [tool.name for tool in canonical_tool_definitions(False)]+ assert "generate_images" not in tool_namesdiff --git a/backend/tests/test_prompts.py b/backend/tests/test_prompts.pyindex 50f1d2fa..74cdcea6 100644--- a/backend/tests/test_prompts.py+++ b/backend/tests/test_prompts.py@@ -82,6 +82,10 @@ class TestCreatePrompt:RESULT_IMAGE_URL: str = "data:image/png;base64,result_image_data"MOCK_SYSTEM_PROMPT: str = "Mock HTML Tailwind system prompt"TEST_STACK: Stack = "html_tailwind"+ ENABLED_IMAGE_POLICY: str = (+ "Image generation is enabled for this request. Use generate_images for "+ "missing assets when needed."+ )def test_plan_create_uses_create_from_input(self) -> None:plan = derive_prompt_construction_plan(@@ -122,7 +126,10 @@ class TestCreatePrompt:"generationType": "create",}- with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -159,6 +166,42 @@ class TestCreatePrompt:actual: ExpectedResult = {"messages": messages}assert_structure_match(actual, expected)+ @pytest.mark.asyncio+ async def test_image_mode_create_with_image_generation_disabled(self) -> None:+ params: Dict[str, Any] = {+ "prompt": {"text": "", "images": [self.TEST_IMAGE_URL]},+ "generationType": "create",+ }++ with patch("prompts.system_prompt.SYSTEM_PROMPT", new=self.MOCK_SYSTEM_PROMPT):+ messages = await build_prompt_messages(+ stack=self.TEST_STACK,+ input_mode="image",+ generation_type=params["generationType"],+ prompt=params["prompt"],+ history=[],+ image_generation_enabled=False,+ )++ system_content = messages[0].get("content")+ assert isinstance(system_content, str)+ assert system_content == self.MOCK_SYSTEM_PROMPT++ user_content = messages[1].get("content")+ assert isinstance(user_content, list)+ text_part = next(+ (+ part+ for part in user_content+ if isinstance(part, dict) and part.get("type") == "text"+ ),+ None,+ )+ assert isinstance(text_part, dict)+ user_text = text_part.get("text")+ assert isinstance(user_text, str)+ assert "Image generation is disabled for this request. Do not call generate_images." in user_text+@pytest.mark.asyncioasync def test_image_mode_update_with_history(self) -> None:@@ -175,7 +218,10 @@ class TestCreatePrompt:],}- with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -195,7 +241,11 @@ class TestCreatePrompt:{"role": "assistant", "content": "<html>Initial code</html>"},{"role": "user",- "content": f"Selected stack: {self.TEST_STACK}.\n\nMake the background blue",+ "content": (+ f"Selected stack: {self.TEST_STACK}.\n\n"+ f"{self.ENABLED_IMAGE_POLICY}\n\n"+ "Make the background blue"+ ),},{"role": "assistant", "content": "<html>Updated code</html>"},{"role": "user", "content": "Add a header"},@@ -206,6 +256,32 @@ class TestCreatePrompt:actual: ExpectedResult = {"messages": messages}assert_structure_match(actual, expected)+ @pytest.mark.asyncio+ async def test_update_history_with_image_generation_disabled(self) -> None:+ with patch("prompts.system_prompt.SYSTEM_PROMPT", new=self.MOCK_SYSTEM_PROMPT):+ messages = await build_prompt_messages(+ stack=self.TEST_STACK,+ input_mode="image",+ generation_type="update",+ prompt={"text": "", "images": [self.TEST_IMAGE_URL], "videos": []},+ history=[+ {"role": "assistant", "text": "<html>Initial code</html>", "images": [], "videos": []},+ {"role": "user", "text": "Make the background blue", "images": [], "videos": []},+ {"role": "assistant", "text": "<html>Updated code</html>", "images": [], "videos": []},+ ],+ image_generation_enabled=False,+ )++ system_content = messages[0].get("content")+ assert isinstance(system_content, str)+ assert system_content == self.MOCK_SYSTEM_PROMPT++ first_user_content = messages[2].get("content")+ assert isinstance(first_user_content, str)+ assert "Selected stack: html_tailwind." in first_user_content+ assert "Image generation is disabled for this request. Do not call generate_images." in first_user_content+ assert "Make the background blue" in first_user_content+@pytest.mark.asyncioasync def test_text_mode_create_generation(self) -> None:"""Test create generation from text description in text mode."""@@ -218,7 +294,10 @@ class TestCreatePrompt:},"generationType": "create"}- with patch('prompts.system_prompt.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -264,7 +343,10 @@ class TestCreatePrompt:{"role": "user", "text": "Now add a navigation menu", "images": [], "videos": []},]}- with patch('prompts.system_prompt.SYSTEM_PROMPT', self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -287,7 +369,11 @@ class TestCreatePrompt:},{"role": "user",- "content": f"Selected stack: {self.TEST_STACK}.\n\nAdd a sidebar",+ "content": (+ f"Selected stack: {self.TEST_STACK}.\n\n"+ f"{self.ENABLED_IMAGE_POLICY}\n\n"+ "Add a sidebar"+ ),},{"role": "assistant",@@ -390,7 +476,10 @@ class TestCreatePrompt:]}- with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -420,7 +509,11 @@ class TestCreatePrompt:},{"type": "text",- "text": f"Selected stack: {self.TEST_STACK}.\n\nAdd a button",+ "text": (+ f"Selected stack: {self.TEST_STACK}.\n\n"+ f"{self.ENABLED_IMAGE_POLICY}\n\n"+ "Add a button"+ ),},],},@@ -448,7 +541,10 @@ class TestCreatePrompt:]}- with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -485,7 +581,11 @@ class TestCreatePrompt:},{"type": "text",- "text": f"Selected stack: {self.TEST_STACK}.\n\nStyle like these examples",+ "text": (+ f"Selected stack: {self.TEST_STACK}.\n\n"+ f"{self.ENABLED_IMAGE_POLICY}\n\n"+ "Style like these examples"+ ),},],},@@ -511,7 +611,10 @@ class TestCreatePrompt:]}- with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):# Call the functionmessages = await build_prompt_messages(stack=self.TEST_STACK,@@ -531,7 +634,11 @@ class TestCreatePrompt:{"role": "assistant", "content": "<html>Initial code</html>"},{"role": "user",- "content": f"Selected stack: {self.TEST_STACK}.\n\nMake it blue",+ "content": (+ f"Selected stack: {self.TEST_STACK}.\n\n"+ f"{self.ENABLED_IMAGE_POLICY}\n\n"+ "Make it blue"+ ),}, # Text-only message{"role": "assistant", "content": "<html>Blue code</html>"},],@@ -555,7 +662,10 @@ class TestCreatePrompt:},}- with patch("prompts.system_prompt.SYSTEM_PROMPT", self.MOCK_SYSTEM_PROMPT):+ with patch(+ "prompts.system_prompt.SYSTEM_PROMPT",+ new=self.MOCK_SYSTEM_PROMPT,+ ):messages = await build_prompt_messages(stack=self.TEST_STACK,input_mode="image",