Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
feat: add frontend support for images in follow-up edits
Abi Rajabbda0b37fecb
diff --git a/backend/prompts/screenshot_system_prompts.py b/backend/prompts/screenshot_system_prompts.pyindex d05a3575..f9a41c16 100644--- a/backend/prompts/screenshot_system_prompts.py+++ b/backend/prompts/screenshot_system_prompts.py@@ -5,8 +5,6 @@ HTML_TAILWIND_SYSTEM_PROMPT = """You are an expert Tailwind developerYou take screenshots of a reference web page from the user, and then build single page appsusing Tailwind, HTML and JS.-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to-update it to look more like the reference image(The first image).- Make sure the app looks exactly like the screenshot.- Pay close attention to background color, text color, font size, font family,@@ -30,8 +28,6 @@ HTML_CSS_SYSTEM_PROMPT = """You are an expert CSS developerYou take screenshots of a reference web page from the user, and then build single page appsusing CSS, HTML and JS.-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to-update it to look more like the reference image(The first image).- Make sure the app looks exactly like the screenshot.- Pay close attention to background color, text color, font size, font family,@@ -54,8 +50,6 @@ BOOTSTRAP_SYSTEM_PROMPT = """You are an expert Bootstrap developerYou take screenshots of a reference web page from the user, and then build single page appsusing Bootstrap, HTML and JS.-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to-update it to look more like the reference image(The first image).- Make sure the app looks exactly like the screenshot.- Pay close attention to background color, text color, font size, font family,@@ -79,8 +73,6 @@ REACT_TAILWIND_SYSTEM_PROMPT = """You are an expert React/Tailwind developerYou take screenshots of a reference web page from the user, and then build single page appsusing React and Tailwind CSS.-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to-update it to look more like the reference image(The first image).- Make sure the app looks exactly like the screenshot.- Pay close attention to background color, text color, font size, font family,@@ -108,8 +100,6 @@ IONIC_TAILWIND_SYSTEM_PROMPT = """You are an expert Ionic/Tailwind developerYou take screenshots of a reference web page from the user, and then build single page appsusing Ionic and Tailwind CSS.-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to-update it to look more like the reference image(The first image).- Make sure the app looks exactly like the screenshot.- Pay close attention to background color, text color, font size, font family,@@ -142,8 +132,6 @@ VUE_TAILWIND_SYSTEM_PROMPT = """You are an expert Vue/Tailwind developerYou take screenshots of a reference web page from the user, and then build single page appsusing Vue and Tailwind CSS.-You might also be given a screenshot(The second image) of a web page that you have already built, and asked to-update it to look more like the reference image(The first image).- Make sure the app looks exactly like the screenshot.- Pay close attention to background color, text color, font size, font family,diff --git a/backend/tests/test_prompt_summary.py b/backend/tests/test_prompt_summary.pyindex dc71a8f3..14547811 100644--- a/backend/tests/test_prompt_summary.py+++ b/backend/tests/test_prompt_summary.py@@ -1,4 +1,3 @@-import pytestfrom utils import format_prompt_summary@@ -9,8 +8,14 @@ def test_format_prompt_summary():"role": "user","content": [{"type": "text", "text": "hello world"},- {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}},- {"type": "image_url", "image_url": {"url": "data:image/png;base64,BBB"}},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,AAA"},+ },+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,BBB"},+ },],},]diff --git a/frontend/src/App.tsx b/frontend/src/App.tsxindex f71188d1..4e871d2d 100644--- a/frontend/src/App.tsx+++ b/frontend/src/App.tsx@@ -57,6 +57,8 @@ function App() {const {disableInSelectAndEditMode,setUpdateInstruction,+ updateImage,+ setUpdateImage,appState,setAppState,} = useAppStore();@@ -314,7 +316,7 @@ function App() {const updatedHistory = [...historyTree,- { text: modifiedUpdateInstruction, images: [] },+ { text: modifiedUpdateInstruction, images: updateImage ? [updateImage] : [] },];doGenerateCode({@@ -329,6 +331,7 @@ function App() {});setUpdateInstruction("");+ setUpdateImage(null);}const handleTermDialogOpenChange = (open: boolean) => {diff --git a/frontend/src/components/UpdateImageUpload.tsx b/frontend/src/components/UpdateImageUpload.tsxnew file mode 100644index 00000000..baa3fb16--- /dev/null+++ b/frontend/src/components/UpdateImageUpload.tsx@@ -0,0 +1,106 @@+import { useState, useCallback } from "react";+import { useDropzone } from "react-dropzone";+import { toast } from "react-hot-toast";+import { Cross2Icon, UploadIcon } from "@radix-ui/react-icons";+import { Button } from "./ui/button";++// Helper function to convert file to data URL+function fileToDataURL(file: File): Promise<string> {+ return new Promise((resolve, reject) => {+ const reader = new FileReader();+ reader.onload = () => resolve(reader.result as string);+ reader.onerror = (error) => reject(error);+ reader.readAsDataURL(file);+ });+}++interface Props {+ updateImage: string | null;+ setUpdateImage: (image: string | null) => void;+}++function UpdateImageUpload({ updateImage, setUpdateImage }: Props) {+ const [isDragActive, setIsDragActive] = useState(false);++ const onDrop = useCallback((acceptedFiles: File[]) => {+ if (acceptedFiles.length > 0) {+ const file = acceptedFiles[0];+ fileToDataURL(file)+ .then((dataUrl) => {+ setUpdateImage(dataUrl);+ })+ .catch((error) => {+ toast.error("Error reading image file");+ console.error("Error reading file:", error);+ });+ }+ }, [setUpdateImage]);++ const { getRootProps, getInputProps, isDragAccept, isDragReject } = useDropzone({+ onDrop,+ accept: {+ "image/png": [".png"],+ "image/jpeg": [".jpeg", ".jpg"],+ },+ maxFiles: 1,+ maxSize: 1024 * 1024 * 20, // 20MB+ onDragEnter: () => setIsDragActive(true),+ onDragLeave: () => setIsDragActive(false),+ onDropRejected: (rejectedFiles) => {+ setIsDragActive(false);+ toast.error(rejectedFiles[0].errors[0].message);+ },+ });++ const removeImage = () => {+ setUpdateImage(null);+ setIsDragActive(false);+ };++ if (updateImage) {+ return (+ <div className="relative mb-2">+ <img+ src={updateImage}+ alt="Update reference"+ className="w-full max-w-[300px] border border-gray-200 rounded-md"+ />+ <Button+ variant="destructive"+ size="sm"+ className="absolute top-1 right-1 h-6 w-6 p-0"+ onClick={removeImage}+ >+ <Cross2Icon className="h-3 w-3" />+ </Button>+ <div className="text-gray-400 uppercase text-xs text-center mt-1">+ Reference Image+ </div>+ </div>+ );+ }++ return (+ <div+ {...getRootProps()}+ className={`+ border-2 border-dashed rounded-md p-4 text-center cursor-pointer transition-colors mb-2+ ${isDragAccept ? "border-green-400 bg-green-50" : ""}+ ${isDragReject ? "border-red-400 bg-red-50" : ""}+ ${isDragActive ? "border-blue-400 bg-blue-50" : "border-gray-300"}+ hover:border-gray-400 hover:bg-gray-50+ `}+ >+ <input {...getInputProps()} />+ <UploadIcon className="h-8 w-8 mx-auto mb-2 text-gray-400" />+ <p className="text-sm text-gray-600">+ Add reference image (optional)+ </p>+ <p className="text-xs text-gray-400 mt-1">+ PNG, JPG up to 20MB+ </p>+ </div>+ );+}++export default UpdateImageUpload;diff --git a/frontend/src/components/sidebar/Sidebar.tsx b/frontend/src/components/sidebar/Sidebar.tsxindex ed0a24f5..eb8d5795 100644--- a/frontend/src/components/sidebar/Sidebar.tsx+++ b/frontend/src/components/sidebar/Sidebar.tsx@@ -11,6 +11,7 @@ import { Textarea } from "../ui/textarea";import { useEffect, useRef, useState } from "react";import HistoryDisplay from "../history/HistoryDisplay";import Variants from "../variants/Variants";+import UpdateImageUpload from "../UpdateImageUpload";interface SidebarProps {showSelectAndEditFeature: boolean;@@ -28,7 +29,7 @@ function Sidebar({const textareaRef = useRef<HTMLTextAreaElement>(null);const [isErrorExpanded, setIsErrorExpanded] = useState(false);- const { appState, updateInstruction, setUpdateInstruction } = useAppStore();+ const { appState, updateInstruction, setUpdateInstruction, updateImage, setUpdateImage } = useAppStore();const { inputMode, referenceImages, head, commits } = useProjectStore();@@ -137,6 +138,10 @@ function Sidebar({!isSelectedVariantError && (<div><div className="grid w-full gap-2">+ <UpdateImageUpload+ updateImage={updateImage}+ setUpdateImage={setUpdateImage}+ /><Textarearef={textareaRef}placeholder="Tell the AI what to change..."diff --git a/frontend/src/store/app-store.ts b/frontend/src/store/app-store.tsindex e9afce6b..bc5b2d54 100644--- a/frontend/src/store/app-store.ts+++ b/frontend/src/store/app-store.ts@@ -10,6 +10,10 @@ interface AppStore {updateInstruction: string;setUpdateInstruction: (instruction: string) => void;+ // Update image support (restricted to single image)+ updateImage: string | null;+ setUpdateImage: (image: string | null) => void;+inSelectAndEditMode: boolean;toggleInSelectAndEditMode: () => void;disableInSelectAndEditMode: () => void;@@ -24,6 +28,10 @@ export const useAppStore = create<AppStore>((set) => ({setUpdateInstruction: (instruction: string) =>set({ updateInstruction: instruction }),+ // Update image support+ updateImage: null,+ setUpdateImage: (image: string | null) => set({ updateImage: image }),+inSelectAndEditMode: false,toggleInSelectAndEditMode: () =>set((state) => ({ inSelectAndEditMode: !state.inSelectAndEditMode })),