Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

chore: unify prompt data structure

Abi Rajaf09107106b3d
diff --git a/backend/prompts/__init__.py b/backend/prompts/__init__.py
index 19a6b6a3..d8e06c6e 100644
--- a/backend/prompts/__init__.py
+++ b/backend/prompts/__init__.py
@@ -1,4 +1,4 @@
-from typing import Union
+from typing import Union, Any
from openai.types.chat import ChatCompletionMessageParam, ChatCompletionContentPartParam
from custom_types import InputMode
@@ -6,7 +6,7 @@ from image_generation.core import create_alt_url_mapping
from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTS
from prompts.screenshot_system_prompts import SYSTEM_PROMPTS
from prompts.text_prompts import SYSTEM_PROMPTS as TEXT_SYSTEM_PROMPTS
-from prompts.types import Stack
+from prompts.types import Stack, PromptContent
from video.utils import assemble_claude_prompt_video
@@ -20,67 +20,59 @@ Generate code for a SVG that looks exactly like this.
async def create_prompt(
- params: dict[str, str], stack: Stack, input_mode: InputMode
+ params: dict[str, Any], stack: Stack, input_mode: InputMode
) -> tuple[list[ChatCompletionMessageParam], dict[str, str]]:
image_cache: dict[str, str] = {}
# If this generation started off with imported code, we need to assemble the prompt differently
if params.get("isImportedFromCode"):
- original_imported_code = params["history"][0]
+ original_imported_code = params["history"][0]["text"]
prompt_messages = assemble_imported_code_prompt(original_imported_code, stack)
- for index, text in enumerate(params["history"][1:]):
- if index % 2 == 0:
- message: ChatCompletionMessageParam = {
- "role": "user",
- "content": text,
- }
- else:
- message: ChatCompletionMessageParam = {
- "role": "assistant",
- "content": text,
- }
+ for index, item in enumerate(params["history"][1:]):
+ role = "user" if index % 2 == 0 else "assistant"
+ message: ChatCompletionMessageParam = {
+ "role": role,
+ "content": item["text"],
+ }
prompt_messages.append(message)
else:
# Assemble the prompt for non-imported code
+ prompt: PromptContent = params["prompt"]
if input_mode == "image":
+ image_url = prompt["images"][0]
if params.get("resultImage"):
prompt_messages = assemble_prompt(
- params["image"], stack, params["resultImage"]
+ image_url, stack, params["resultImage"]
)
else:
- prompt_messages = assemble_prompt(params["image"], stack)
+ prompt_messages = assemble_prompt(image_url, stack)
elif input_mode == "text":
- prompt_messages = assemble_text_prompt(params["image"], stack)
+ prompt_messages = assemble_text_prompt(prompt["text"], stack)
else:
# Default to image mode for backward compatibility
+ image_url = prompt["images"][0]
if params.get("resultImage"):
prompt_messages = assemble_prompt(
- params["image"], stack, params["resultImage"]
+ image_url, stack, params["resultImage"]
)
else:
- prompt_messages = assemble_prompt(params["image"], stack)
+ prompt_messages = assemble_prompt(image_url, stack)
if params["generationType"] == "update":
# Transform the history tree into message format
- # TODO: Move this to frontend
- for index, text in enumerate(params["history"]):
- if index % 2 == 0:
- message: ChatCompletionMessageParam = {
- "role": "assistant",
- "content": text,
- }
- else:
- message: ChatCompletionMessageParam = {
- "role": "user",
- "content": text,
- }
+ for index, item in enumerate(params["history"]):
+ role = "assistant" if index % 2 == 0 else "user"
+ message: ChatCompletionMessageParam = {
+ "role": role,
+ "content": item["text"],
+ }
prompt_messages.append(message)
- image_cache = create_alt_url_mapping(params["history"][-2])
+ image_cache = create_alt_url_mapping(params["history"][-2]["text"])
if input_mode == "video":
- video_data_url = params["image"]
+ video_data_url = params["prompt"]["images"][0]
prompt_messages = await assemble_claude_prompt_video(video_data_url)
return prompt_messages, image_cache
diff --git a/backend/prompts/types.py b/backend/prompts/types.py
index 7d4ca2d2..c3c29aed 100644
--- a/backend/prompts/types.py
+++ b/backend/prompts/types.py
@@ -1,4 +1,4 @@
-from typing import Literal, TypedDict
+from typing import Literal, TypedDict, List
class SystemPrompts(TypedDict):
@@ -11,6 +11,13 @@ class SystemPrompts(TypedDict):
svg: str
+class PromptContent(TypedDict):
+ """Unified structure for prompt text and images."""
+
+ text: str
+ images: List[str]
+
+
Stack = Literal[
"html_css",
"html_tailwind",
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 651de228..eff03559 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -198,17 +198,13 @@ function App() {
...baseCommitObject,
type: "ai_create" as const,
parentHash: null,
- inputs: { image_url: referenceImages[0] },
+ inputs: params.prompt,
}
: {
...baseCommitObject,
type: "ai_edit" as const,
parentHash: head,
- inputs: {
- prompt: params.history
- ? params.history[params.history.length - 1]
- : "",
- },
+ inputs: params.prompt,
};
// Create a new commit and set it as the head
@@ -258,8 +254,8 @@ function App() {
if (referenceImages.length > 0) {
doGenerateCode({
generationType: "create",
- image: referenceImages[0],
inputMode,
+ prompt: { text: "", images: [referenceImages[0]] },
});
}
}
@@ -273,7 +269,7 @@ function App() {
doGenerateCode({
generationType: "create",
inputMode: "text",
- image: text,
+ prompt: { text, images: [] },
});
}
@@ -314,12 +310,18 @@ function App() {
selectedElement.outerHTML;
}
- const updatedHistory = [...historyTree, modifiedUpdateInstruction];
+ const updatedHistory = [
+ ...historyTree,
+ { text: modifiedUpdateInstruction, images: [] },
+ ];
doGenerateCode({
generationType: "update",
inputMode,
- image: inputMode === "text" ? initialPrompt : referenceImages[0],
+ prompt:
+ inputMode === "text"
+ ? { text: initialPrompt, images: [] }
+ : { text: "", images: [referenceImages[0]] },
history: updatedHistory,
isImportedFromCode,
});
diff --git a/frontend/src/components/commits/types.ts b/frontend/src/components/commits/types.ts
index 93959240..f1630be1 100644
--- a/frontend/src/components/commits/types.ts
+++ b/frontend/src/components/commits/types.ts
@@ -19,18 +19,16 @@ export type BaseCommit = {
export type CommitType = "ai_create" | "ai_edit" | "code_create";
+import { PromptContent } from "../../types";
+
export type AiCreateCommit = BaseCommit & {
type: "ai_create";
- inputs: {
- image_url: string;
- };
+ inputs: PromptContent;
};
export type AiEditCommit = BaseCommit & {
type: "ai_edit";
- inputs: {
- prompt: string;
- };
+ inputs: PromptContent;
};
export type CodeCreateCommit = BaseCommit & {
diff --git a/frontend/src/components/history/utils.test.ts b/frontend/src/components/history/utils.test.ts
index 7a5aaa29..24063737 100644
--- a/frontend/src/components/history/utils.test.ts
+++ b/frontend/src/components/history/utils.test.ts
@@ -10,9 +10,7 @@ const basicLinearHistory: Record<CommitHash, Commit> = {
parentHash: null,
variants: [{ code: "<html>1. create</html>" }],
selectedVariantIndex: 0,
- inputs: {
- image_url: "",
- },
+ inputs: { text: "", images: [""] },
},
"1": {
hash: "1",
@@ -22,9 +20,7 @@ const basicLinearHistory: Record<CommitHash, Commit> = {
parentHash: "0",
variants: [{ code: "<html>2. edit with better icons</html>" }],
selectedVariantIndex: 0,
- inputs: {
- prompt: "use better icons",
- },
+ inputs: { text: "use better icons", images: [] },
},
"2": {
hash: "2",
@@ -34,9 +30,7 @@ const basicLinearHistory: Record<CommitHash, Commit> = {
parentHash: "1",
variants: [{ code: "<html>3. edit with better icons and red text</html>" }],
selectedVariantIndex: 0,
- inputs: {
- prompt: "make text red",
- },
+ inputs: { text: "make text red", images: [] },
},
};
@@ -66,9 +60,7 @@ const basicBranchingHistory: Record<CommitHash, Commit> = {
{ code: "<html>4. edit with better icons and green text</html>" },
],
selectedVariantIndex: 0,
- inputs: {
- prompt: "make text green",
- },
+ inputs: { text: "make text green", images: [] },
},
};
@@ -84,9 +76,7 @@ const longerBranchingHistory: Record<CommitHash, Commit> = {
{ code: "<html>5. edit with better icons and green, bold text</html>" },
],
selectedVariantIndex: 0,
- inputs: {
- prompt: "make text bold",
- },
+ inputs: { text: "make text bold", images: [] },
},
};
@@ -99,9 +89,7 @@ const basicBadHistory: Record<CommitHash, Commit> = {
parentHash: null,
variants: [{ code: "<html>1. create</html>" }],
selectedVariantIndex: 0,
- inputs: {
- image_url: "",
- },
+ inputs: { text: "", images: [""] },
},
"1": {
hash: "1",
@@ -111,51 +99,52 @@ const basicBadHistory: Record<CommitHash, Commit> = {
parentHash: "2", // <- Bad parent hash
variants: [{ code: "<html>2. edit with better icons</html>" }],
selectedVariantIndex: 0,
- inputs: {
- prompt: "use better icons",
- },
+ inputs: { text: "use better icons", images: [] },
},
};
describe("History Utils", () => {
test("should correctly extract the history tree", () => {
expect(extractHistory("2", basicLinearHistory)).toEqual([
- "<html>1. create</html>",
- "use better icons",
- "<html>2. edit with better icons</html>",
- "make text red",
- "<html>3. edit with better icons and red text</html>",
+ { text: "<html>1. create</html>", images: [] },
+ { text: "use better icons", images: [] },
+ { text: "<html>2. edit with better icons</html>", images: [] },
+ { text: "make text red", images: [] },
+ { text: "<html>3. edit with better icons and red text</html>", images: [] },
]);
expect(extractHistory("0", basicLinearHistory)).toEqual([
- "<html>1. create</html>",
+ { text: "<html>1. create</html>", images: [] },
]);
// Test branching
expect(extractHistory("3", basicBranchingHistory)).toEqual([
- "<html>1. create</html>",
- "use better icons",
- "<html>2. edit with better icons</html>",
- "make text green",
- "<html>4. edit with better icons and green text</html>",
+ { text: "<html>1. create</html>", images: [] },
+ { text: "use better icons", images: [] },
+ { text: "<html>2. edit with better icons</html>", images: [] },
+ { text: "make text green", images: [] },
+ { text: "<html>4. edit with better icons and green text</html>", images: [] },
]);
expect(extractHistory("4", longerBranchingHistory)).toEqual([
- "<html>1. create</html>",
- "use better icons",
- "<html>2. edit with better icons</html>",
- "make text green",
- "<html>4. edit with better icons and green text</html>",
- "make text bold",
- "<html>5. edit with better icons and green, bold text</html>",
+ { text: "<html>1. create</html>", images: [] },
+ { text: "use better icons", images: [] },
+ { text: "<html>2. edit with better icons</html>", images: [] },
+ { text: "make text green", images: [] },
+ { text: "<html>4. edit with better icons and green text</html>", images: [] },
+ { text: "make text bold", images: [] },
+ {
+ text: "<html>5. edit with better icons and green, bold text</html>",
+ images: [],
+ },
]);
expect(extractHistory("2", longerBranchingHistory)).toEqual([
- "<html>1. create</html>",
- "use better icons",
- "<html>2. edit with better icons</html>",
- "make text red",
- "<html>3. edit with better icons and red text</html>",
+ { text: "<html>1. create</html>", images: [] },
+ { text: "use better icons", images: [] },
+ { text: "<html>2. edit with better icons</html>", images: [] },
+ { text: "make text red", images: [] },
+ { text: "<html>3. edit with better icons and red text</html>", images: [] },
]);
// Errors
diff --git a/frontend/src/components/history/utils.ts b/frontend/src/components/history/utils.ts
index 16fee843..ae30c8fd 100644
--- a/frontend/src/components/history/utils.ts
+++ b/frontend/src/components/history/utils.ts
@@ -1,21 +1,25 @@
import { Commit, CommitHash, CommitType } from "../commits/types";
+import { PromptContent } from "../../types";
export function extractHistory(
hash: CommitHash,
commits: Record<CommitHash, Commit>
-): string[] {
- const flatHistory: string[] = [];
+): PromptContent[] {
+ const flatHistory: PromptContent[] = [];
let currentCommitHash: CommitHash | null = hash;
while (currentCommitHash !== null) {
const commit: Commit | null = commits[currentCommitHash];
if (commit) {
- flatHistory.unshift(commit.variants[commit.selectedVariantIndex].code);
+ flatHistory.unshift({
+ text: commit.variants[commit.selectedVariantIndex].code,
+ images: [],
+ });
// For edits, add the prompt to the history
if (commit.type === "ai_edit") {
- flatHistory.unshift(commit.inputs.prompt);
+ flatHistory.unshift(commit.inputs);
}
// Move to the parent of the current item
@@ -65,7 +69,7 @@ export function summarizeHistoryItem(commit: Commit) {
case "ai_create":
return "Create";
case "ai_edit":
- return commit.inputs.prompt;
+ return commit.inputs.text;
case "code_create":
return "Imported from code";
default: {
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index a5166547..1e3d0203 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -31,11 +31,16 @@ export enum ScreenRecorderState {
FINISHED = "finished",
}
+export interface PromptContent {
+ text: string;
+ images: string[]; // Array of data URLs
+}
+
export interface CodeGenerationParams {
generationType: "create" | "update";
inputMode: "image" | "video" | "text";
- image: string;
- history?: string[];
+ prompt: PromptContent;
+ history?: PromptContent[];
isImportedFromCode?: boolean;
}