Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
add experimental support for SVG
Abi Raja9b728d034bf2
diff --git a/backend/eval.py b/backend/eval.pyindex ac286bf5..000c6fe9 100644--- a/backend/eval.py+++ b/backend/eval.py@@ -49,7 +49,7 @@ async def main():for filename in evals:filepath = os.path.join(INPUT_DIR, filename)data_url = await image_to_data_url(filepath)- task = generate_code_core(data_url, "html_tailwind")+ task = generate_code_core(data_url, "svg")tasks.append(task)results = await asyncio.gather(*tasks)diff --git a/backend/imported_code_prompts.py b/backend/imported_code_prompts.pyindex 748c7706..a8bfa6a0 100644--- a/backend/imported_code_prompts.py+++ b/backend/imported_code_prompts.py@@ -78,3 +78,15 @@ In terms of libraries,Return only the full code in <html></html> tags.Do not include markdown "```" or "```html" at the start or end."""++IMPORTED_CODE_SVG_SYSTEM_PROMPT = """+You are an expert at building SVGs.++- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.+- You can use Google Fonts++Return only the full code in <svg></svg> tags.+Do not include markdown "```" or "```svg" at the start or end.+"""diff --git a/backend/prompts.py b/backend/prompts.pyindex f0539640..1fe4f9bb 100644--- a/backend/prompts.py+++ b/backend/prompts.py@@ -7,12 +7,14 @@ from imported_code_prompts import (IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT,IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT,IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT,+ IMPORTED_CODE_SVG_SYSTEM_PROMPT,)from screenshot_system_prompts import (BOOTSTRAP_SYSTEM_PROMPT,IONIC_TAILWIND_SYSTEM_PROMPT,REACT_TAILWIND_SYSTEM_PROMPT,TAILWIND_SYSTEM_PROMPT,+ SVG_SYSTEM_PROMPT,)@@ -20,6 +22,10 @@ USER_PROMPT = """Generate code for a web page that looks exactly like this."""+SVG_USER_PROMPT = """+Generate code for a SVG that looks exactly like this.+"""+def assemble_imported_code_prompt(code: str, stack: str, result_image_data_url: Union[str, None] = None@@ -33,9 +39,16 @@ def assemble_imported_code_prompt(system_content = IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPTelif stack == "ionic_tailwind":system_content = IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT+ elif stack == "svg":+ system_content = IMPORTED_CODE_SVG_SYSTEM_PROMPTelse:raise Exception("Code config is not one of available options")+ user_content = (+ "Here is the code of the app: " + code+ if stack != "svg"+ else "Here is the code of the SVG: " + code+ )return [{"role": "system",@@ -43,7 +56,7 @@ def assemble_imported_code_prompt(},{"role": "user",- "content": "Here is the code of the app: " + code,+ "content": user_content,},]# TODO: Use result_image_data_url@@ -64,9 +77,13 @@ def assemble_prompt(system_content = BOOTSTRAP_SYSTEM_PROMPTelif generated_code_config == "ionic_tailwind":system_content = IONIC_TAILWIND_SYSTEM_PROMPT+ elif generated_code_config == "svg":+ system_content = SVG_SYSTEM_PROMPTelse:raise Exception("Code config is not one of available options")+ user_prompt = USER_PROMPT if generated_code_config != "svg" else SVG_USER_PROMPT+user_content: List[ChatCompletionContentPartParam] = [{"type": "image_url",@@ -74,7 +91,7 @@ def assemble_prompt(},{"type": "text",- "text": USER_PROMPT,+ "text": user_prompt,},]diff --git a/backend/screenshot_system_prompts.py b/backend/screenshot_system_prompts.pyindex a48adaa7..3308d03e 100644--- a/backend/screenshot_system_prompts.py+++ b/backend/screenshot_system_prompts.py@@ -110,3 +110,21 @@ In terms of libraries,Return only the full code in <html></html> tags.Do not include markdown "```" or "```html" at the start or end."""+++SVG_SYSTEM_PROMPT = """+You are an expert at building SVGs.+You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.++- Make sure the SVG looks exactly like the screenshot.+- Pay close attention to background color, text color, font size, font family,+padding, margin, border, etc. Match the colors and sizes exactly.+- Use the exact text from the screenshot.+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.+- You can use Google Fonts++Return only the full code in <svg></svg> tags.+Do not include markdown "```" or "```svg" at the start or end.+"""diff --git a/backend/test_prompts.py b/backend/test_prompts.pyindex 87f3281b..303e3a9b 100644--- a/backend/test_prompts.py+++ b/backend/test_prompts.py@@ -113,6 +113,23 @@ Return only the full code in <html></html> tags.Do not include markdown "```" or "```html" at the start or end."""+SVG_SYSTEM_PROMPT = """+You are an expert at building SVGs.+You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot.++- Make sure the SVG looks exactly like the screenshot.+- Pay close attention to background color, text color, font size, font family,+padding, margin, border, etc. Match the colors and sizes exactly.+- Use the exact text from the screenshot.+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.+- You can use Google Fonts++Return only the full code in <svg></svg> tags.+Do not include markdown "```" or "```svg" at the start or end.+"""+IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """You are an expert Tailwind developer.@@ -194,27 +211,55 @@ Return only the full code in <html></html> tags.Do not include markdown "```" or "```html" at the start or end."""+IMPORTED_CODE_SVG_SYSTEM_PROMPT = """+You are an expert at building SVGs.++- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.+- Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "<!-- Repeat for each news item -->" or bad things will happen.+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later.+- You can use Google Fonts++Return only the full code in <svg></svg> tags.+Do not include markdown "```" or "```svg" at the start or end.+"""++USER_PROMPT = """+Generate code for a web page that looks exactly like this.+"""++SVG_USER_PROMPT = """+Generate code for a SVG that looks exactly like this.+"""+def test_prompts():tailwind_prompt = assemble_prompt("image_data_url", "html_tailwind", "result_image_data_url")assert tailwind_prompt[0]["content"] == TAILWIND_SYSTEM_PROMPT+ assert tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignorereact_tailwind_prompt = assemble_prompt("image_data_url", "react_tailwind", "result_image_data_url")assert react_tailwind_prompt[0]["content"] == REACT_TAILWIND_SYSTEM_PROMPT+ assert react_tailwind_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignorebootstrap_prompt = assemble_prompt("image_data_url", "bootstrap", "result_image_data_url")assert bootstrap_prompt[0]["content"] == BOOTSTRAP_SYSTEM_PROMPT+ assert bootstrap_prompt[1]["content"][2]["text"] == USER_PROMPT # type: ignoreionic_tailwind = assemble_prompt("image_data_url", "ionic_tailwind", "result_image_data_url")assert ionic_tailwind[0]["content"] == IONIC_TAILWIND_SYSTEM_PROMPT+ assert ionic_tailwind[1]["content"][2]["text"] == USER_PROMPT # type: ignore++ svg_prompt = assemble_prompt("image_data_url", "svg", "result_image_data_url")+ assert svg_prompt[0]["content"] == SVG_SYSTEM_PROMPT+ assert svg_prompt[1]["content"][2]["text"] == SVG_USER_PROMPT # type: ignoredef test_imported_code_prompts():@@ -253,3 +298,10 @@ def test_imported_code_prompts():{"role": "user", "content": "Here is the code of the app: code"},]assert ionic_tailwind == expected_ionic_tailwind++ svg = assemble_imported_code_prompt("code", "svg", "result_image_data_url")+ expected_svg = [+ {"role": "system", "content": IMPORTED_CODE_SVG_SYSTEM_PROMPT},+ {"role": "user", "content": "Here is the code of the SVG: code"},+ ]+ assert svg == expected_svgdiff --git a/frontend/src/components/OutputSettingsSection.tsx b/frontend/src/components/OutputSettingsSection.tsxindex fc116ea0..f671b586 100644--- a/frontend/src/components/OutputSettingsSection.tsx+++ b/frontend/src/components/OutputSettingsSection.tsx@@ -6,6 +6,7 @@ import {SelectTrigger,} from "./ui/select";import { GeneratedCodeConfig } from "../types";+import { Badge } from "./ui/badge";function generateDisplayComponent(config: GeneratedCodeConfig) {switch (config) {@@ -36,9 +37,16 @@ function generateDisplayComponent(config: GeneratedCodeConfig) {<span className="font-semibold">Tailwind</span></div>);- default:- // TODO: Should never reach this out. Error out- return config;+ case GeneratedCodeConfig.SVG:+ return (+ <div>+ <span className="font-semibold">SVG</span>+ </div>+ );+ default: {+ const exhaustiveCheck: never = config;+ throw new Error(`Unhandled case: ${exhaustiveCheck}`);+ }}}@@ -83,7 +91,20 @@ function OutputSettingsSection({{generateDisplayComponent(GeneratedCodeConfig.BOOTSTRAP)}</SelectItem><SelectItem value={GeneratedCodeConfig.IONIC_TAILWIND}>- {generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)}+ <div className="flex items-center">+ {generateDisplayComponent(GeneratedCodeConfig.IONIC_TAILWIND)}+ <Badge className="ml-2" variant="secondary">+ Beta+ </Badge>+ </div>+ </SelectItem>+ <SelectItem value={GeneratedCodeConfig.SVG}>+ <div className="flex items-center">+ {generateDisplayComponent(GeneratedCodeConfig.SVG)}+ <Badge className="ml-2" variant="secondary">+ Beta+ </Badge>+ </div></SelectItem></SelectGroup></SelectContent>diff --git a/frontend/src/components/Preview.tsx b/frontend/src/components/Preview.tsxindex 42cc7b06..d2eff16b 100644--- a/frontend/src/components/Preview.tsx+++ b/frontend/src/components/Preview.tsx@@ -1,6 +1,6 @@import { useEffect, useRef } from "react";import classNames from "classnames";-import useThrottle from "../hooks/useThrottle";+// import useThrottle from "../hooks/useThrottle";interface Props {code: string;diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 236116a4..2820427a 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -9,6 +9,7 @@ export enum GeneratedCodeConfig {REACT_TAILWIND = "react_tailwind",BOOTSTRAP = "bootstrap",IONIC_TAILWIND = "ionic_tailwind",+ SVG = "svg",}export interface Settings {