Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
add text generation
Abi Rajac8651dd93908
diff --git a/backend/custom_types.py b/backend/custom_types.pyindex b6c9feee..795eb187 100644--- a/backend/custom_types.py+++ b/backend/custom_types.py@@ -4,4 +4,5 @@ from typing import LiteralInputMode = Literal["image","video",+ "text",]diff --git a/backend/prompts/__init__.py b/backend/prompts/__init__.pyindex 955502b6..19a6b6a3 100644--- a/backend/prompts/__init__.py+++ b/backend/prompts/__init__.py@@ -5,6 +5,7 @@ from custom_types import InputModefrom image_generation.core import create_alt_url_mappingfrom prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTSfrom prompts.screenshot_system_prompts import SYSTEM_PROMPTS+from prompts.text_prompts import SYSTEM_PROMPTS as TEXT_SYSTEM_PROMPTSfrom prompts.types import Stackfrom video.utils import assemble_claude_prompt_video@@ -42,12 +43,23 @@ async def create_prompt(prompt_messages.append(message)else:# Assemble the prompt for non-imported code- if params.get("resultImage"):- prompt_messages = assemble_prompt(- params["image"], stack, params["resultImage"]- )+ if input_mode == "image":+ if params.get("resultImage"):+ prompt_messages = assemble_prompt(+ params["image"], stack, params["resultImage"]+ )+ else:+ prompt_messages = assemble_prompt(params["image"], stack)+ elif input_mode == "text":+ prompt_messages = assemble_text_prompt(params["image"], stack)else:- prompt_messages = assemble_prompt(params["image"], stack)+ # Default to image mode for backward compatibility+ if params.get("resultImage"):+ prompt_messages = assemble_prompt(+ params["image"], stack, params["resultImage"]+ )+ else:+ prompt_messages = assemble_prompt(params["image"], stack)if params["generationType"] == "update":# Transform the history tree into message format@@ -132,3 +144,22 @@ def assemble_prompt("content": user_content,},]+++def assemble_text_prompt(+ text_prompt: str,+ stack: Stack,+) -> list[ChatCompletionMessageParam]:++ system_content = TEXT_SYSTEM_PROMPTS[stack]++ return [+ {+ "role": "system",+ "content": system_content,+ },+ {+ "role": "user",+ "content": "Generate UI for " + text_prompt,+ },+ ]diff --git a/backend/prompts/text_prompts.py b/backend/prompts/text_prompts.pynew file mode 100644index 00000000..b885643f--- /dev/null+++ b/backend/prompts/text_prompts.py@@ -0,0 +1,161 @@+from prompts.types import SystemPrompts++GENERAL_INSTRUCTIONS = """+- Make sure to make it look modern and sleek.+- Use modern, professional fonts and colors.+- Follow UX best practices.+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later."""++LIBRARY_INSTRUCTIONS = """+- You can use Google Fonts+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>"""++FORMAT_INSTRUCTIONS = """+Return only the full code in <html></html> tags.+Do not include markdown "```" or "```html" at the start or end.+Reply with only the code, and no text/explanation before and after the code.+"""++HTML_TAILWIND_SYSTEM_PROMPT = f"""+You are an expert Tailwind developer.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>+{LIBRARY_INSTRUCTIONS}++{FORMAT_INSTRUCTIONS}"""++HTML_TAILWIND_SYSTEM_PROMPT_WITH_THINKING = f"""+<thinking>+First, I need to understand what the user is asking for. They want me to create a UI based on their text description. I should think about:+1. What type of UI component or page they're describing+2. What layout would work best+3. What interactive elements might be needed+4. How to make it look modern and professional+5. What colors, fonts, and spacing to use++Then I'll create the HTML with Tailwind CSS classes following the guidelines.+</thinking>++You are an expert Tailwind developer.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>+{LIBRARY_INSTRUCTIONS}++{FORMAT_INSTRUCTIONS}"""++REACT_TAILWIND_SYSTEM_PROMPT = f"""+You are an expert React developer who specializes in Tailwind CSS and creating beautiful, modern web applications.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>+{LIBRARY_INSTRUCTIONS}+- For icons, use Lucide React: `import {{ IconName }} from "lucide-react"`++{FORMAT_INSTRUCTIONS}++Make sure to include "use client" if the component uses any React hooks."""++BOOTSTRAP_SYSTEM_PROMPT = f"""+You are an expert Bootstrap developer.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++- Use this link to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">+- Use this script to include Bootstrap JS: <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>+{LIBRARY_INSTRUCTIONS}++{FORMAT_INSTRUCTIONS}"""++IONIC_REACT_SYSTEM_PROMPT = f"""+You are an expert Ionic React developer who specializes in creating beautiful, modern mobile applications.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++- Use Ionic React components like IonContent, IonHeader, IonToolbar, IonTitle, IonButton, etc.+- Use this CSS import: <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />+- Use this script import: <script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>+{LIBRARY_INSTRUCTIONS}++{FORMAT_INSTRUCTIONS}++Make sure to wrap the content in IonApp, IonContent components as needed."""++SVG_SYSTEM_PROMPT = f"""+You are an expert SVG developer who creates beautiful, modern SVG graphics and icons.++Create clean, scalable SVG graphics with proper structure.+Use appropriate viewBox settings and ensure the SVG is responsive.+Apply modern design principles with clean lines and appropriate colors.++{FORMAT_INSTRUCTIONS}++Return only the SVG code without any wrapper HTML."""++HTML_CSS_SYSTEM_PROMPT = f"""+You are an expert HTML and CSS developer.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++{LIBRARY_INSTRUCTIONS}++{FORMAT_INSTRUCTIONS}"""++VUE_TAILWIND_SYSTEM_PROMPT = f"""+You are an expert Vue developer who specializes in Tailwind CSS and creating beautiful, modern web applications.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>+{LIBRARY_INSTRUCTIONS}+- For icons, use Lucide Vue: `import {{ IconName }} from "lucide-vue-next"`++{FORMAT_INSTRUCTIONS}++Make sure to use Vue 3 composition API syntax."""++IONIC_TAILWIND_SYSTEM_PROMPT = f"""+You are an expert Ionic developer who specializes in Tailwind CSS and creating beautiful, modern mobile applications.++{GENERAL_INSTRUCTIONS}++In terms of libraries,++- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>+- Use Ionic components like ion-content, ion-header, ion-toolbar, ion-title, ion-button, etc.+{LIBRARY_INSTRUCTIONS}++{FORMAT_INSTRUCTIONS}++Make sure to wrap the content in ion-app, ion-content components as needed."""++SYSTEM_PROMPTS: SystemPrompts = {+ "html_css": HTML_CSS_SYSTEM_PROMPT,+ "html_tailwind": HTML_TAILWIND_SYSTEM_PROMPT,+ "html_tailwind_thinking": HTML_TAILWIND_SYSTEM_PROMPT_WITH_THINKING,+ "react_tailwind": REACT_TAILWIND_SYSTEM_PROMPT,+ "bootstrap": BOOTSTRAP_SYSTEM_PROMPT,+ "ionic_tailwind": IONIC_TAILWIND_SYSTEM_PROMPT,+ "ionic_react": IONIC_REACT_SYSTEM_PROMPT,+ "vue_tailwind": VUE_TAILWIND_SYSTEM_PROMPT,+ "svg": SVG_SYSTEM_PROMPT,+}diff --git a/backend/prompts/types.py b/backend/prompts/types.pyindex 7d4ca2d2..57599cb7 100644--- a/backend/prompts/types.py+++ b/backend/prompts/types.py@@ -4,9 +4,11 @@ from typing import Literal, TypedDictclass SystemPrompts(TypedDict):html_css: strhtml_tailwind: str+ html_tailwind_thinking: strreact_tailwind: strbootstrap: strionic_tailwind: str+ ionic_react: strvue_tailwind: strsvg: str@@ -14,9 +16,11 @@ class SystemPrompts(TypedDict):Stack = Literal["html_css","html_tailwind",+ "html_tailwind_thinking","react_tailwind","bootstrap","ionic_tailwind",+ "ionic_react","vue_tailwind","svg",]diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.pyindex e88caf2b..0f4734c3 100644--- a/backend/routes/generate_code.py+++ b/backend/routes/generate_code.py@@ -296,6 +296,7 @@ class ModelSelectionStage:async def select_models(self,generation_type: Literal["create", "update"],+ input_mode: InputMode,openai_api_key: str | None,anthropic_api_key: str | None,gemini_api_key: str | None = None,@@ -304,6 +305,7 @@ class ModelSelectionStage:try:variant_models = self._get_variant_models(generation_type,+ input_mode,NUM_VARIANTS,openai_api_key,anthropic_api_key,@@ -327,6 +329,7 @@ class ModelSelectionStage:def _get_variant_models(self,generation_type: Literal["create", "update"],+ input_mode: InputMode,num_variants: int,openai_api_key: str | None,anthropic_api_key: str | None,@@ -340,18 +343,23 @@ class ModelSelectionStage:else:claude_model = Llm.CLAUDE_3_5_SONNET_2024_06_20- # Gemini only works for create right now- if generation_type == "create":- gemini_model = Llm.GEMINI_2_0_FLASH+ # For text input mode, use Claude 4 Sonnet as third option+ # For other input modes (image/video), use Gemini as third option+ if input_mode == "text":+ third_model = Llm.CLAUDE_4_SONNET_2025_05_14else:- gemini_model = Llm.CLAUDE_3_7_SONNET_2025_02_19+ # Gemini only works for create right now+ if generation_type == "create":+ third_model = Llm.GEMINI_2_0_FLASH+ else:+ third_model = Llm.CLAUDE_3_7_SONNET_2025_02_19# Define models based on available API keys- if openai_api_key and anthropic_api_key and gemini_api_key:+ if openai_api_key and anthropic_api_key and (gemini_api_key or input_mode == "text"):models = [Llm.GPT_4_1_2025_04_14,claude_model,- gemini_model,+ third_model,]elif openai_api_key and anthropic_api_key:models = [claude_model, Llm.GPT_4_1_2025_04_14]@@ -866,6 +874,7 @@ class CodeGenerationMiddleware(Middleware):model_selector = ModelSelectionStage(context.throw_error)context.variant_models = await model_selector.select_models(generation_type=context.extracted_params.generation_type,+ input_mode=context.extracted_params.input_mode,openai_api_key=context.extracted_params.openai_api_key,anthropic_api_key=context.extracted_params.anthropic_api_key,gemini_api_key=GEMINI_API_KEY,diff --git a/frontend/src/App.tsx b/frontend/src/App.tsxindex 7f82adc6..651de228 100644--- a/frontend/src/App.tsx+++ b/frontend/src/App.tsx@@ -23,6 +23,7 @@ import { GenerationSettings } from "./components/settings/GenerationSettings";import StartPane from "./components/start-pane/StartPane";import { Commit } from "./components/commits/types";import { createCommit } from "./components/commits/utils";+import GenerateFromText from "./components/generate-from-text/GenerateFromText";function App() {const {@@ -33,6 +34,8 @@ function App() {setIsImportedFromCode,referenceImages,setReferenceImages,+ initialPrompt,+ setInitialPrompt,head,commits,@@ -139,7 +142,12 @@ function App() {}// Re-run the create- doCreate(referenceImages, inputMode);+ if (inputMode === "image" || inputMode === "video") {+ doCreate(referenceImages, inputMode);+ } else {+ // TODO: Fix this+ doCreateFromText(initialPrompt);+ }};// Used when the user cancels the code generation@@ -256,6 +264,19 @@ function App() {}}+ function doCreateFromText(text: string) {+ // Reset any existing state+ reset();++ setInputMode("text");+ setInitialPrompt(text);+ doGenerateCode({+ generationType: "create",+ inputMode: "text",+ image: text,+ });+ }+// Subsequent updatesasync function doUpdate(updateInstruction: string,@@ -298,7 +319,7 @@ function App() {doGenerateCode({generationType: "update",inputMode,- image: referenceImages[0],+ image: inputMode === "text" ? initialPrompt : referenceImages[0],history: updatedHistory,isImportedFromCode,});@@ -372,6 +393,10 @@ function App() {{IS_RUNNING_ON_CLOUD && !settings.openAiApiKey && <OnboardingNote />}+ {appState === AppState.INITIAL && (+ <GenerateFromText doCreateFromText={doCreateFromText} />+ )}+{/* Rest of the sidebar when we're not in the initial state */}{(appState === AppState.CODING ||appState === AppState.CODE_READY) && (diff --git a/frontend/src/components/generate-from-text/GenerateFromText.tsx b/frontend/src/components/generate-from-text/GenerateFromText.tsxnew file mode 100644index 00000000..8d0bd914--- /dev/null+++ b/frontend/src/components/generate-from-text/GenerateFromText.tsx@@ -0,0 +1,72 @@+import { useState, useRef, useEffect } from "react";+import { Button } from "../ui/button";+import { Textarea } from "../ui/textarea";+import toast from "react-hot-toast";++interface GenerateFromTextProps {+ doCreateFromText: (text: string) => void;+}++function GenerateFromText({ doCreateFromText }: GenerateFromTextProps) {+ const [isOpen, setIsOpen] = useState(false);+ const [text, setText] = useState("");+ const textareaRef = useRef<HTMLTextAreaElement>(null);++ useEffect(() => {+ if (isOpen && textareaRef.current) {+ textareaRef.current.focus();+ }+ }, [isOpen]);++ const handleGenerate = () => {+ if (text.trim() === "") {+ toast.error("Please enter a prompt to generate from");+ return;+ }+ doCreateFromText(text);+ };++ const handleKeyDown = (e: React.KeyboardEvent) => {+ if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {+ e.preventDefault();+ handleGenerate();+ }+ };++ return (+ <div className="mt-4">+ {!isOpen ? (+ <div className="flex justify-center">+ <Button variant="secondary" onClick={() => setIsOpen(true)}>+ Generate from text prompt [BETA]+ </Button>+ </div>+ ) : (+ <>+ <Textarea+ ref={textareaRef}+ rows={2}+ placeholder="A SaaS admin dashboard with charts and user management"+ className="w-full mb-4"+ value={text}+ onChange={(e) => setText(e.target.value)}+ onKeyDown={handleKeyDown}+ />+ <div className="flex justify-between items-center">+ <span className="text-sm text-gray-500">+ Press Cmd/Ctrl + Enter to generate+ </span>+ <div className="flex gap-2">+ <Button variant="outline" onClick={() => setIsOpen(false)}>+ Cancel+ </Button>+ <Button onClick={handleGenerate}>Generate</Button>+ </div>+ </div>+ </>+ )}+ </div>+ );+}++export default GenerateFromText;diff --git a/frontend/src/store/project-store.ts b/frontend/src/store/project-store.tsindex 7ae0bda0..9bb62feb 100644--- a/frontend/src/store/project-store.ts+++ b/frontend/src/store/project-store.ts@@ -4,12 +4,14 @@ import { Commit, CommitHash, VariantStatus } from "../components/commits/types";// Store for app-wide stateinterface ProjectStore {// Inputs- inputMode: "image" | "video";- setInputMode: (mode: "image" | "video") => void;+ inputMode: "image" | "video" | "text";+ setInputMode: (mode: "image" | "video" | "text") => void;isImportedFromCode: boolean;setIsImportedFromCode: (imported: boolean) => void;referenceImages: string[];setReferenceImages: (images: string[]) => void;+ initialPrompt: string;+ setInitialPrompt: (prompt: string) => void;// Outputscommits: Record<string, Commit>;@@ -50,6 +52,8 @@ export const useProjectStore = create<ProjectStore>((set) => ({setIsImportedFromCode: (imported) => set({ isImportedFromCode: imported }),referenceImages: [],setReferenceImages: (images) => set({ referenceImages: images }),+ initialPrompt: "",+ setInitialPrompt: (prompt) => set({ initialPrompt: prompt }),// Outputscommits: {},diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 909bfdc8..a5166547 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -33,7 +33,7 @@ export enum ScreenRecorderState {export interface CodeGenerationParams {generationType: "create" | "update";- inputMode: "image" | "video";+ inputMode: "image" | "video" | "text";image: string;history?: string[];isImportedFromCode?: boolean;