Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

feat: add frontend support for images in follow-up edits

Abi Rajabbda0b37fecb
diff --git a/backend/prompts/screenshot_system_prompts.py b/backend/prompts/screenshot_system_prompts.py
index d05a3575..f9a41c16 100644
--- a/backend/prompts/screenshot_system_prompts.py
+++ b/backend/prompts/screenshot_system_prompts.py
@@ -5,8 +5,6 @@ HTML_TAILWIND_SYSTEM_PROMPT = """
You are an expert Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Tailwind, HTML and JS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
@@ -30,8 +28,6 @@ HTML_CSS_SYSTEM_PROMPT = """
You are an expert CSS developer
You take screenshots of a reference web page from the user, and then build single page apps
using CSS, HTML and JS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
@@ -54,8 +50,6 @@ BOOTSTRAP_SYSTEM_PROMPT = """
You are an expert Bootstrap developer
You take screenshots of a reference web page from the user, and then build single page apps
using Bootstrap, HTML and JS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
@@ -79,8 +73,6 @@ REACT_TAILWIND_SYSTEM_PROMPT = """
You are an expert React/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using React and Tailwind CSS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
@@ -108,8 +100,6 @@ IONIC_TAILWIND_SYSTEM_PROMPT = """
You are an expert Ionic/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Ionic and Tailwind CSS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
@@ -142,8 +132,6 @@ VUE_TAILWIND_SYSTEM_PROMPT = """
You are an expert Vue/Tailwind developer
You take screenshots of a reference web page from the user, and then build single page apps
using Vue and Tailwind CSS.
-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to
-update it to look more like the reference image(The first image).
- Make sure the app looks exactly like the screenshot.
- Pay close attention to background color, text color, font size, font family,
diff --git a/backend/tests/test_prompt_summary.py b/backend/tests/test_prompt_summary.py
index dc71a8f3..14547811 100644
--- a/backend/tests/test_prompt_summary.py
+++ b/backend/tests/test_prompt_summary.py
@@ -1,4 +1,3 @@
-import pytest
from utils import format_prompt_summary
@@ -9,8 +8,14 @@ def test_format_prompt_summary():
"role": "user",
"content": [
{"type": "text", "text": "hello world"},
- {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}},
- {"type": "image_url", "image_url": {"url": "data:image/png;base64,BBB"}},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,AAA"},
+ },
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,BBB"},
+ },
],
},
]
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index f71188d1..4e871d2d 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -57,6 +57,8 @@ function App() {
const {
disableInSelectAndEditMode,
setUpdateInstruction,
+ updateImage,
+ setUpdateImage,
appState,
setAppState,
} = useAppStore();
@@ -314,7 +316,7 @@ function App() {
const updatedHistory = [
...historyTree,
- { text: modifiedUpdateInstruction, images: [] },
+ { text: modifiedUpdateInstruction, images: updateImage ? [updateImage] : [] },
];
doGenerateCode({
@@ -329,6 +331,7 @@ function App() {
});
setUpdateInstruction("");
+ setUpdateImage(null);
}
const handleTermDialogOpenChange = (open: boolean) => {
diff --git a/frontend/src/components/UpdateImageUpload.tsx b/frontend/src/components/UpdateImageUpload.tsx
new file mode 100644
index 00000000..baa3fb16
--- /dev/null
+++ b/frontend/src/components/UpdateImageUpload.tsx
@@ -0,0 +1,106 @@
+import { useState, useCallback } from "react";
+import { useDropzone } from "react-dropzone";
+import { toast } from "react-hot-toast";
+import { Cross2Icon, UploadIcon } from "@radix-ui/react-icons";
+import { Button } from "./ui/button";
+
+// Helper function to convert file to data URL
+function fileToDataURL(file: File): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = (error) => reject(error);
+ reader.readAsDataURL(file);
+ });
+}
+
+interface Props {
+ updateImage: string | null;
+ setUpdateImage: (image: string | null) => void;
+}
+
+function UpdateImageUpload({ updateImage, setUpdateImage }: Props) {
+ const [isDragActive, setIsDragActive] = useState(false);
+
+ const onDrop = useCallback((acceptedFiles: File[]) => {
+ if (acceptedFiles.length > 0) {
+ const file = acceptedFiles[0];
+ fileToDataURL(file)
+ .then((dataUrl) => {
+ setUpdateImage(dataUrl);
+ })
+ .catch((error) => {
+ toast.error("Error reading image file");
+ console.error("Error reading file:", error);
+ });
+ }
+ }, [setUpdateImage]);
+
+ const { getRootProps, getInputProps, isDragAccept, isDragReject } = useDropzone({
+ onDrop,
+ accept: {
+ "image/png": [".png"],
+ "image/jpeg": [".jpeg", ".jpg"],
+ },
+ maxFiles: 1,
+ maxSize: 1024 * 1024 * 20, // 20MB
+ onDragEnter: () => setIsDragActive(true),
+ onDragLeave: () => setIsDragActive(false),
+ onDropRejected: (rejectedFiles) => {
+ setIsDragActive(false);
+ toast.error(rejectedFiles[0].errors[0].message);
+ },
+ });
+
+ const removeImage = () => {
+ setUpdateImage(null);
+ setIsDragActive(false);
+ };
+
+ if (updateImage) {
+ return (
+ <div className="relative mb-2">
+ <img
+ src={updateImage}
+ alt="Update reference"
+ className="w-full max-w-[300px] border border-gray-200 rounded-md"
+ />
+ <Button
+ variant="destructive"
+ size="sm"
+ className="absolute top-1 right-1 h-6 w-6 p-0"
+ onClick={removeImage}
+ >
+ <Cross2Icon className="h-3 w-3" />
+ </Button>
+ <div className="text-gray-400 uppercase text-xs text-center mt-1">
+ Reference Image
+ </div>
+ </div>
+ );
+ }
+
+ return (
+ <div
+ {...getRootProps()}
+ className={`
+ border-2 border-dashed rounded-md p-4 text-center cursor-pointer transition-colors mb-2
+ ${isDragAccept ? "border-green-400 bg-green-50" : ""}
+ ${isDragReject ? "border-red-400 bg-red-50" : ""}
+ ${isDragActive ? "border-blue-400 bg-blue-50" : "border-gray-300"}
+ hover:border-gray-400 hover:bg-gray-50
+ `}
+ >
+ <input {...getInputProps()} />
+ <UploadIcon className="h-8 w-8 mx-auto mb-2 text-gray-400" />
+ <p className="text-sm text-gray-600">
+ Add reference image (optional)
+ </p>
+ <p className="text-xs text-gray-400 mt-1">
+ PNG, JPG up to 20MB
+ </p>
+ </div>
+ );
+}
+
+export default UpdateImageUpload;
diff --git a/frontend/src/components/sidebar/Sidebar.tsx b/frontend/src/components/sidebar/Sidebar.tsx
index ed0a24f5..eb8d5795 100644
--- a/frontend/src/components/sidebar/Sidebar.tsx
+++ b/frontend/src/components/sidebar/Sidebar.tsx
@@ -11,6 +11,7 @@ import { Textarea } from "../ui/textarea";
import { useEffect, useRef, useState } from "react";
import HistoryDisplay from "../history/HistoryDisplay";
import Variants from "../variants/Variants";
+import UpdateImageUpload from "../UpdateImageUpload";
interface SidebarProps {
showSelectAndEditFeature: boolean;
@@ -28,7 +29,7 @@ function Sidebar({
const textareaRef = useRef<HTMLTextAreaElement>(null);
const [isErrorExpanded, setIsErrorExpanded] = useState(false);
- const { appState, updateInstruction, setUpdateInstruction } = useAppStore();
+ const { appState, updateInstruction, setUpdateInstruction, updateImage, setUpdateImage } = useAppStore();
const { inputMode, referenceImages, head, commits } = useProjectStore();
@@ -137,6 +138,10 @@ function Sidebar({
!isSelectedVariantError && (
<div>
<div className="grid w-full gap-2">
+ <UpdateImageUpload
+ updateImage={updateImage}
+ setUpdateImage={setUpdateImage}
+ />
<Textarea
ref={textareaRef}
placeholder="Tell the AI what to change..."
diff --git a/frontend/src/store/app-store.ts b/frontend/src/store/app-store.ts
index e9afce6b..bc5b2d54 100644
--- a/frontend/src/store/app-store.ts
+++ b/frontend/src/store/app-store.ts
@@ -10,6 +10,10 @@ interface AppStore {
updateInstruction: string;
setUpdateInstruction: (instruction: string) => void;
+ // Update image support (restricted to single image)
+ updateImage: string | null;
+ setUpdateImage: (image: string | null) => void;
+
inSelectAndEditMode: boolean;
toggleInSelectAndEditMode: () => void;
disableInSelectAndEditMode: () => void;
@@ -24,6 +28,10 @@ export const useAppStore = create<AppStore>((set) => ({
setUpdateInstruction: (instruction: string) =>
set({ updateInstruction: instruction }),
+ // Update image support
+ updateImage: null,
+ setUpdateImage: (image: string | null) => set({ updateImage: image }),
+
inSelectAndEditMode: false,
toggleInSelectAndEditMode: () =>
set((state) => ({ inSelectAndEditMode: !state.inSelectAndEditMode })),