Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
chore: unify prompt data structure
Abi Rajaf09107106b3d
diff --git a/backend/prompts/__init__.py b/backend/prompts/__init__.pyindex 19a6b6a3..d8e06c6e 100644--- a/backend/prompts/__init__.py+++ b/backend/prompts/__init__.py@@ -1,4 +1,4 @@-from typing import Union+from typing import Union, Anyfrom openai.types.chat import ChatCompletionMessageParam, ChatCompletionContentPartParamfrom custom_types import InputMode@@ -6,7 +6,7 @@ from image_generation.core import create_alt_url_mappingfrom prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTSfrom prompts.screenshot_system_prompts import SYSTEM_PROMPTSfrom prompts.text_prompts import SYSTEM_PROMPTS as TEXT_SYSTEM_PROMPTS-from prompts.types import Stack+from prompts.types import Stack, PromptContentfrom video.utils import assemble_claude_prompt_video@@ -20,67 +20,59 @@ Generate code for a SVG that looks exactly like this.async def create_prompt(- params: dict[str, str], stack: Stack, input_mode: InputMode+ params: dict[str, Any], stack: Stack, input_mode: InputMode) -> tuple[list[ChatCompletionMessageParam], dict[str, str]]:image_cache: dict[str, str] = {}# If this generation started off with imported code, we need to assemble the prompt differentlyif params.get("isImportedFromCode"):- original_imported_code = params["history"][0]+ original_imported_code = params["history"][0]["text"]prompt_messages = assemble_imported_code_prompt(original_imported_code, stack)- for index, text in enumerate(params["history"][1:]):- if index % 2 == 0:- message: ChatCompletionMessageParam = {- "role": "user",- "content": text,- }- else:- message: ChatCompletionMessageParam = {- "role": "assistant",- "content": text,- }+ for index, item in enumerate(params["history"][1:]):+ role = "user" if index % 2 == 0 else "assistant"+ message: ChatCompletionMessageParam = {+ "role": role,+ "content": item["text"],+ }prompt_messages.append(message)else:# Assemble the prompt for non-imported code+ prompt: PromptContent = params["prompt"]if input_mode == "image":+ image_url = prompt["images"][0]if params.get("resultImage"):prompt_messages = assemble_prompt(- params["image"], stack, params["resultImage"]+ image_url, stack, params["resultImage"])else:- prompt_messages = assemble_prompt(params["image"], stack)+ prompt_messages = assemble_prompt(image_url, stack)elif input_mode == "text":- prompt_messages = assemble_text_prompt(params["image"], stack)+ prompt_messages = assemble_text_prompt(prompt["text"], stack)else:# Default to image mode for backward compatibility+ image_url = prompt["images"][0]if params.get("resultImage"):prompt_messages = assemble_prompt(- params["image"], stack, params["resultImage"]+ image_url, stack, params["resultImage"])else:- prompt_messages = assemble_prompt(params["image"], stack)+ prompt_messages = assemble_prompt(image_url, stack)if params["generationType"] == "update":# Transform the history tree into message format- # TODO: Move this to frontend- for index, text in enumerate(params["history"]):- if index % 2 == 0:- message: ChatCompletionMessageParam = {- "role": "assistant",- "content": text,- }- else:- message: ChatCompletionMessageParam = {- "role": "user",- "content": text,- }+ for index, item in enumerate(params["history"]):+ role = "assistant" if index % 2 == 0 else "user"+ message: ChatCompletionMessageParam = {+ "role": role,+ "content": item["text"],+ }prompt_messages.append(message)- image_cache = create_alt_url_mapping(params["history"][-2])+ image_cache = create_alt_url_mapping(params["history"][-2]["text"])if input_mode == "video":- video_data_url = params["image"]+ video_data_url = params["prompt"]["images"][0]prompt_messages = await assemble_claude_prompt_video(video_data_url)return prompt_messages, image_cachediff --git a/backend/prompts/types.py b/backend/prompts/types.pyindex 7d4ca2d2..c3c29aed 100644--- a/backend/prompts/types.py+++ b/backend/prompts/types.py@@ -1,4 +1,4 @@-from typing import Literal, TypedDict+from typing import Literal, TypedDict, Listclass SystemPrompts(TypedDict):@@ -11,6 +11,13 @@ class SystemPrompts(TypedDict):svg: str+class PromptContent(TypedDict):+ """Unified structure for prompt text and images."""++ text: str+ images: List[str]++Stack = Literal["html_css","html_tailwind",diff --git a/frontend/src/App.tsx b/frontend/src/App.tsxindex 651de228..eff03559 100644--- a/frontend/src/App.tsx+++ b/frontend/src/App.tsx@@ -198,17 +198,13 @@ function App() {...baseCommitObject,type: "ai_create" as const,parentHash: null,- inputs: { image_url: referenceImages[0] },+ inputs: params.prompt,}: {...baseCommitObject,type: "ai_edit" as const,parentHash: head,- inputs: {- prompt: params.history- ? params.history[params.history.length - 1]- : "",- },+ inputs: params.prompt,};// Create a new commit and set it as the head@@ -258,8 +254,8 @@ function App() {if (referenceImages.length > 0) {doGenerateCode({generationType: "create",- image: referenceImages[0],inputMode,+ prompt: { text: "", images: [referenceImages[0]] },});}}@@ -273,7 +269,7 @@ function App() {doGenerateCode({generationType: "create",inputMode: "text",- image: text,+ prompt: { text, images: [] },});}@@ -314,12 +310,18 @@ function App() {selectedElement.outerHTML;}- const updatedHistory = [...historyTree, modifiedUpdateInstruction];+ const updatedHistory = [+ ...historyTree,+ { text: modifiedUpdateInstruction, images: [] },+ ];doGenerateCode({generationType: "update",inputMode,- image: inputMode === "text" ? initialPrompt : referenceImages[0],+ prompt:+ inputMode === "text"+ ? { text: initialPrompt, images: [] }+ : { text: "", images: [referenceImages[0]] },history: updatedHistory,isImportedFromCode,});diff --git a/frontend/src/components/commits/types.ts b/frontend/src/components/commits/types.tsindex 93959240..f1630be1 100644--- a/frontend/src/components/commits/types.ts+++ b/frontend/src/components/commits/types.ts@@ -19,18 +19,16 @@ export type BaseCommit = {export type CommitType = "ai_create" | "ai_edit" | "code_create";+import { PromptContent } from "../../types";+export type AiCreateCommit = BaseCommit & {type: "ai_create";- inputs: {- image_url: string;- };+ inputs: PromptContent;};export type AiEditCommit = BaseCommit & {type: "ai_edit";- inputs: {- prompt: string;- };+ inputs: PromptContent;};export type CodeCreateCommit = BaseCommit & {diff --git a/frontend/src/components/history/utils.test.ts b/frontend/src/components/history/utils.test.tsindex 7a5aaa29..24063737 100644--- a/frontend/src/components/history/utils.test.ts+++ b/frontend/src/components/history/utils.test.ts@@ -10,9 +10,7 @@ const basicLinearHistory: Record<CommitHash, Commit> = {parentHash: null,variants: [{ code: "<html>1. create</html>" }],selectedVariantIndex: 0,- inputs: {- image_url: "",- },+ inputs: { text: "", images: [""] },},"1": {hash: "1",@@ -22,9 +20,7 @@ const basicLinearHistory: Record<CommitHash, Commit> = {parentHash: "0",variants: [{ code: "<html>2. edit with better icons</html>" }],selectedVariantIndex: 0,- inputs: {- prompt: "use better icons",- },+ inputs: { text: "use better icons", images: [] },},"2": {hash: "2",@@ -34,9 +30,7 @@ const basicLinearHistory: Record<CommitHash, Commit> = {parentHash: "1",variants: [{ code: "<html>3. edit with better icons and red text</html>" }],selectedVariantIndex: 0,- inputs: {- prompt: "make text red",- },+ inputs: { text: "make text red", images: [] },},};@@ -66,9 +60,7 @@ const basicBranchingHistory: Record<CommitHash, Commit> = {{ code: "<html>4. edit with better icons and green text</html>" },],selectedVariantIndex: 0,- inputs: {- prompt: "make text green",- },+ inputs: { text: "make text green", images: [] },},};@@ -84,9 +76,7 @@ const longerBranchingHistory: Record<CommitHash, Commit> = {{ code: "<html>5. edit with better icons and green, bold text</html>" },],selectedVariantIndex: 0,- inputs: {- prompt: "make text bold",- },+ inputs: { text: "make text bold", images: [] },},};@@ -99,9 +89,7 @@ const basicBadHistory: Record<CommitHash, Commit> = {parentHash: null,variants: [{ code: "<html>1. create</html>" }],selectedVariantIndex: 0,- inputs: {- image_url: "",- },+ inputs: { text: "", images: [""] },},"1": {hash: "1",@@ -111,51 +99,52 @@ const basicBadHistory: Record<CommitHash, Commit> = {parentHash: "2", // <- Bad parent hashvariants: [{ code: "<html>2. edit with better icons</html>" }],selectedVariantIndex: 0,- inputs: {- prompt: "use better icons",- },+ inputs: { text: "use better icons", images: [] },},};describe("History Utils", () => {test("should correctly extract the history tree", () => {expect(extractHistory("2", basicLinearHistory)).toEqual([- "<html>1. create</html>",- "use better icons",- "<html>2. edit with better icons</html>",- "make text red",- "<html>3. edit with better icons and red text</html>",+ { text: "<html>1. create</html>", images: [] },+ { text: "use better icons", images: [] },+ { text: "<html>2. edit with better icons</html>", images: [] },+ { text: "make text red", images: [] },+ { text: "<html>3. edit with better icons and red text</html>", images: [] },]);expect(extractHistory("0", basicLinearHistory)).toEqual([- "<html>1. create</html>",+ { text: "<html>1. create</html>", images: [] },]);// Test branchingexpect(extractHistory("3", basicBranchingHistory)).toEqual([- "<html>1. create</html>",- "use better icons",- "<html>2. edit with better icons</html>",- "make text green",- "<html>4. edit with better icons and green text</html>",+ { text: "<html>1. create</html>", images: [] },+ { text: "use better icons", images: [] },+ { text: "<html>2. edit with better icons</html>", images: [] },+ { text: "make text green", images: [] },+ { text: "<html>4. edit with better icons and green text</html>", images: [] },]);expect(extractHistory("4", longerBranchingHistory)).toEqual([- "<html>1. create</html>",- "use better icons",- "<html>2. edit with better icons</html>",- "make text green",- "<html>4. edit with better icons and green text</html>",- "make text bold",- "<html>5. edit with better icons and green, bold text</html>",+ { text: "<html>1. create</html>", images: [] },+ { text: "use better icons", images: [] },+ { text: "<html>2. edit with better icons</html>", images: [] },+ { text: "make text green", images: [] },+ { text: "<html>4. edit with better icons and green text</html>", images: [] },+ { text: "make text bold", images: [] },+ {+ text: "<html>5. edit with better icons and green, bold text</html>",+ images: [],+ },]);expect(extractHistory("2", longerBranchingHistory)).toEqual([- "<html>1. create</html>",- "use better icons",- "<html>2. edit with better icons</html>",- "make text red",- "<html>3. edit with better icons and red text</html>",+ { text: "<html>1. create</html>", images: [] },+ { text: "use better icons", images: [] },+ { text: "<html>2. edit with better icons</html>", images: [] },+ { text: "make text red", images: [] },+ { text: "<html>3. edit with better icons and red text</html>", images: [] },]);// Errorsdiff --git a/frontend/src/components/history/utils.ts b/frontend/src/components/history/utils.tsindex 16fee843..ae30c8fd 100644--- a/frontend/src/components/history/utils.ts+++ b/frontend/src/components/history/utils.ts@@ -1,21 +1,25 @@import { Commit, CommitHash, CommitType } from "../commits/types";+import { PromptContent } from "../../types";export function extractHistory(hash: CommitHash,commits: Record<CommitHash, Commit>-): string[] {- const flatHistory: string[] = [];+): PromptContent[] {+ const flatHistory: PromptContent[] = [];let currentCommitHash: CommitHash | null = hash;while (currentCommitHash !== null) {const commit: Commit | null = commits[currentCommitHash];if (commit) {- flatHistory.unshift(commit.variants[commit.selectedVariantIndex].code);+ flatHistory.unshift({+ text: commit.variants[commit.selectedVariantIndex].code,+ images: [],+ });// For edits, add the prompt to the historyif (commit.type === "ai_edit") {- flatHistory.unshift(commit.inputs.prompt);+ flatHistory.unshift(commit.inputs);}// Move to the parent of the current item@@ -65,7 +69,7 @@ export function summarizeHistoryItem(commit: Commit) {case "ai_create":return "Create";case "ai_edit":- return commit.inputs.prompt;+ return commit.inputs.text;case "code_create":return "Imported from code";default: {diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex a5166547..1e3d0203 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -31,11 +31,16 @@ export enum ScreenRecorderState {FINISHED = "finished",}+export interface PromptContent {+ text: string;+ images: string[]; // Array of data URLs+}+export interface CodeGenerationParams {generationType: "create" | "update";inputMode: "image" | "video" | "text";- image: string;- history?: string[];+ prompt: PromptContent;+ history?: PromptContent[];isImportedFromCode?: boolean;}