Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

add text generation

Abi Rajac8651dd93908
diff --git a/backend/custom_types.py b/backend/custom_types.py
index b6c9feee..795eb187 100644
--- a/backend/custom_types.py
+++ b/backend/custom_types.py
@@ -4,4 +4,5 @@ from typing import Literal
InputMode = Literal[
"image",
"video",
+ "text",
]
diff --git a/backend/prompts/__init__.py b/backend/prompts/__init__.py
index 955502b6..19a6b6a3 100644
--- a/backend/prompts/__init__.py
+++ b/backend/prompts/__init__.py
@@ -5,6 +5,7 @@ from custom_types import InputMode
from image_generation.core import create_alt_url_mapping
from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTS
from prompts.screenshot_system_prompts import SYSTEM_PROMPTS
+from prompts.text_prompts import SYSTEM_PROMPTS as TEXT_SYSTEM_PROMPTS
from prompts.types import Stack
from video.utils import assemble_claude_prompt_video
@@ -42,12 +43,23 @@ async def create_prompt(
prompt_messages.append(message)
else:
# Assemble the prompt for non-imported code
- if params.get("resultImage"):
- prompt_messages = assemble_prompt(
- params["image"], stack, params["resultImage"]
- )
+ if input_mode == "image":
+ if params.get("resultImage"):
+ prompt_messages = assemble_prompt(
+ params["image"], stack, params["resultImage"]
+ )
+ else:
+ prompt_messages = assemble_prompt(params["image"], stack)
+ elif input_mode == "text":
+ prompt_messages = assemble_text_prompt(params["image"], stack)
else:
- prompt_messages = assemble_prompt(params["image"], stack)
+ # Default to image mode for backward compatibility
+ if params.get("resultImage"):
+ prompt_messages = assemble_prompt(
+ params["image"], stack, params["resultImage"]
+ )
+ else:
+ prompt_messages = assemble_prompt(params["image"], stack)
if params["generationType"] == "update":
# Transform the history tree into message format
@@ -132,3 +144,22 @@ def assemble_prompt(
"content": user_content,
},
]
+
+
+def assemble_text_prompt(
+ text_prompt: str,
+ stack: Stack,
+) -> list[ChatCompletionMessageParam]:
+
+ system_content = TEXT_SYSTEM_PROMPTS[stack]
+
+ return [
+ {
+ "role": "system",
+ "content": system_content,
+ },
+ {
+ "role": "user",
+ "content": "Generate UI for " + text_prompt,
+ },
+ ]
diff --git a/backend/prompts/text_prompts.py b/backend/prompts/text_prompts.py
new file mode 100644
index 00000000..b885643f
--- /dev/null
+++ b/backend/prompts/text_prompts.py
@@ -0,0 +1,161 @@
+from prompts.types import SystemPrompts
+
+GENERAL_INSTRUCTIONS = """
+- Make sure to make it look modern and sleek.
+- Use modern, professional fonts and colors.
+- Follow UX best practices.
+- Do not add comments in the code such as "<!-- Add other navigation links as needed -->" and "<!-- ... other news items ... -->" in place of writing the full code. WRITE THE FULL CODE.
+- For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later."""
+
+LIBRARY_INSTRUCTIONS = """
+- You can use Google Fonts
+- Font Awesome for icons: <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css"></link>"""
+
+FORMAT_INSTRUCTIONS = """
+Return only the full code in <html></html> tags.
+Do not include markdown "```" or "```html" at the start or end.
+Reply with only the code, and no text/explanation before and after the code.
+"""
+
+HTML_TAILWIND_SYSTEM_PROMPT = f"""
+You are an expert Tailwind developer.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+{LIBRARY_INSTRUCTIONS}
+
+{FORMAT_INSTRUCTIONS}"""
+
+HTML_TAILWIND_SYSTEM_PROMPT_WITH_THINKING = f"""
+<thinking>
+First, I need to understand what the user is asking for. They want me to create a UI based on their text description. I should think about:
+1. What type of UI component or page they're describing
+2. What layout would work best
+3. What interactive elements might be needed
+4. How to make it look modern and professional
+5. What colors, fonts, and spacing to use
+
+Then I'll create the HTML with Tailwind CSS classes following the guidelines.
+</thinking>
+
+You are an expert Tailwind developer.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+{LIBRARY_INSTRUCTIONS}
+
+{FORMAT_INSTRUCTIONS}"""
+
+REACT_TAILWIND_SYSTEM_PROMPT = f"""
+You are an expert React developer who specializes in Tailwind CSS and creating beautiful, modern web applications.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+{LIBRARY_INSTRUCTIONS}
+- For icons, use Lucide React: `import {{ IconName }} from "lucide-react"`
+
+{FORMAT_INSTRUCTIONS}
+
+Make sure to include "use client" if the component uses any React hooks."""
+
+BOOTSTRAP_SYSTEM_PROMPT = f"""
+You are an expert Bootstrap developer.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+- Use this link to include Bootstrap: <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
+- Use this script to include Bootstrap JS: <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
+{LIBRARY_INSTRUCTIONS}
+
+{FORMAT_INSTRUCTIONS}"""
+
+IONIC_REACT_SYSTEM_PROMPT = f"""
+You are an expert Ionic React developer who specializes in creating beautiful, modern mobile applications.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+- Use Ionic React components like IonContent, IonHeader, IonToolbar, IonTitle, IonButton, etc.
+- Use this CSS import: <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@ionic/core/css/ionic.bundle.css" />
+- Use this script import: <script type="module" src="https://cdn.jsdelivr.net/npm/@ionic/core/dist/ionic/ionic.esm.js"></script>
+{LIBRARY_INSTRUCTIONS}
+
+{FORMAT_INSTRUCTIONS}
+
+Make sure to wrap the content in IonApp, IonContent components as needed."""
+
+SVG_SYSTEM_PROMPT = f"""
+You are an expert SVG developer who creates beautiful, modern SVG graphics and icons.
+
+Create clean, scalable SVG graphics with proper structure.
+Use appropriate viewBox settings and ensure the SVG is responsive.
+Apply modern design principles with clean lines and appropriate colors.
+
+{FORMAT_INSTRUCTIONS}
+
+Return only the SVG code without any wrapper HTML."""
+
+HTML_CSS_SYSTEM_PROMPT = f"""
+You are an expert HTML and CSS developer.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+{LIBRARY_INSTRUCTIONS}
+
+{FORMAT_INSTRUCTIONS}"""
+
+VUE_TAILWIND_SYSTEM_PROMPT = f"""
+You are an expert Vue developer who specializes in Tailwind CSS and creating beautiful, modern web applications.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+{LIBRARY_INSTRUCTIONS}
+- For icons, use Lucide Vue: `import {{ IconName }} from "lucide-vue-next"`
+
+{FORMAT_INSTRUCTIONS}
+
+Make sure to use Vue 3 composition API syntax."""
+
+IONIC_TAILWIND_SYSTEM_PROMPT = f"""
+You are an expert Ionic developer who specializes in Tailwind CSS and creating beautiful, modern mobile applications.
+
+{GENERAL_INSTRUCTIONS}
+
+In terms of libraries,
+
+- Use this script to include Tailwind: <script src="https://cdn.tailwindcss.com"></script>
+- Use Ionic components like ion-content, ion-header, ion-toolbar, ion-title, ion-button, etc.
+{LIBRARY_INSTRUCTIONS}
+
+{FORMAT_INSTRUCTIONS}
+
+Make sure to wrap the content in ion-app, ion-content components as needed."""
+
+SYSTEM_PROMPTS: SystemPrompts = {
+ "html_css": HTML_CSS_SYSTEM_PROMPT,
+ "html_tailwind": HTML_TAILWIND_SYSTEM_PROMPT,
+ "html_tailwind_thinking": HTML_TAILWIND_SYSTEM_PROMPT_WITH_THINKING,
+ "react_tailwind": REACT_TAILWIND_SYSTEM_PROMPT,
+ "bootstrap": BOOTSTRAP_SYSTEM_PROMPT,
+ "ionic_tailwind": IONIC_TAILWIND_SYSTEM_PROMPT,
+ "ionic_react": IONIC_REACT_SYSTEM_PROMPT,
+ "vue_tailwind": VUE_TAILWIND_SYSTEM_PROMPT,
+ "svg": SVG_SYSTEM_PROMPT,
+}
diff --git a/backend/prompts/types.py b/backend/prompts/types.py
index 7d4ca2d2..57599cb7 100644
--- a/backend/prompts/types.py
+++ b/backend/prompts/types.py
@@ -4,9 +4,11 @@ from typing import Literal, TypedDict
class SystemPrompts(TypedDict):
html_css: str
html_tailwind: str
+ html_tailwind_thinking: str
react_tailwind: str
bootstrap: str
ionic_tailwind: str
+ ionic_react: str
vue_tailwind: str
svg: str
@@ -14,9 +16,11 @@ class SystemPrompts(TypedDict):
Stack = Literal[
"html_css",
"html_tailwind",
+ "html_tailwind_thinking",
"react_tailwind",
"bootstrap",
"ionic_tailwind",
+ "ionic_react",
"vue_tailwind",
"svg",
]
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index e88caf2b..0f4734c3 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -296,6 +296,7 @@ class ModelSelectionStage:
async def select_models(
self,
generation_type: Literal["create", "update"],
+ input_mode: InputMode,
openai_api_key: str | None,
anthropic_api_key: str | None,
gemini_api_key: str | None = None,
@@ -304,6 +305,7 @@ class ModelSelectionStage:
try:
variant_models = self._get_variant_models(
generation_type,
+ input_mode,
NUM_VARIANTS,
openai_api_key,
anthropic_api_key,
@@ -327,6 +329,7 @@ class ModelSelectionStage:
def _get_variant_models(
self,
generation_type: Literal["create", "update"],
+ input_mode: InputMode,
num_variants: int,
openai_api_key: str | None,
anthropic_api_key: str | None,
@@ -340,18 +343,23 @@ class ModelSelectionStage:
else:
claude_model = Llm.CLAUDE_3_5_SONNET_2024_06_20
- # Gemini only works for create right now
- if generation_type == "create":
- gemini_model = Llm.GEMINI_2_0_FLASH
+ # For text input mode, use Claude 4 Sonnet as third option
+ # For other input modes (image/video), use Gemini as third option
+ if input_mode == "text":
+ third_model = Llm.CLAUDE_4_SONNET_2025_05_14
else:
- gemini_model = Llm.CLAUDE_3_7_SONNET_2025_02_19
+ # Gemini only works for create right now
+ if generation_type == "create":
+ third_model = Llm.GEMINI_2_0_FLASH
+ else:
+ third_model = Llm.CLAUDE_3_7_SONNET_2025_02_19
# Define models based on available API keys
- if openai_api_key and anthropic_api_key and gemini_api_key:
+ if openai_api_key and anthropic_api_key and (gemini_api_key or input_mode == "text"):
models = [
Llm.GPT_4_1_2025_04_14,
claude_model,
- gemini_model,
+ third_model,
]
elif openai_api_key and anthropic_api_key:
models = [claude_model, Llm.GPT_4_1_2025_04_14]
@@ -866,6 +874,7 @@ class CodeGenerationMiddleware(Middleware):
model_selector = ModelSelectionStage(context.throw_error)
context.variant_models = await model_selector.select_models(
generation_type=context.extracted_params.generation_type,
+ input_mode=context.extracted_params.input_mode,
openai_api_key=context.extracted_params.openai_api_key,
anthropic_api_key=context.extracted_params.anthropic_api_key,
gemini_api_key=GEMINI_API_KEY,
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 7f82adc6..651de228 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -23,6 +23,7 @@ import { GenerationSettings } from "./components/settings/GenerationSettings";
import StartPane from "./components/start-pane/StartPane";
import { Commit } from "./components/commits/types";
import { createCommit } from "./components/commits/utils";
+import GenerateFromText from "./components/generate-from-text/GenerateFromText";
function App() {
const {
@@ -33,6 +34,8 @@ function App() {
setIsImportedFromCode,
referenceImages,
setReferenceImages,
+ initialPrompt,
+ setInitialPrompt,
head,
commits,
@@ -139,7 +142,12 @@ function App() {
}
// Re-run the create
- doCreate(referenceImages, inputMode);
+ if (inputMode === "image" || inputMode === "video") {
+ doCreate(referenceImages, inputMode);
+ } else {
+ // TODO: Fix this
+ doCreateFromText(initialPrompt);
+ }
};
// Used when the user cancels the code generation
@@ -256,6 +264,19 @@ function App() {
}
}
+ function doCreateFromText(text: string) {
+ // Reset any existing state
+ reset();
+
+ setInputMode("text");
+ setInitialPrompt(text);
+ doGenerateCode({
+ generationType: "create",
+ inputMode: "text",
+ image: text,
+ });
+ }
+
// Subsequent updates
async function doUpdate(
updateInstruction: string,
@@ -298,7 +319,7 @@ function App() {
doGenerateCode({
generationType: "update",
inputMode,
- image: referenceImages[0],
+ image: inputMode === "text" ? initialPrompt : referenceImages[0],
history: updatedHistory,
isImportedFromCode,
});
@@ -372,6 +393,10 @@ function App() {
{IS_RUNNING_ON_CLOUD && !settings.openAiApiKey && <OnboardingNote />}
+ {appState === AppState.INITIAL && (
+ <GenerateFromText doCreateFromText={doCreateFromText} />
+ )}
+
{/* Rest of the sidebar when we're not in the initial state */}
{(appState === AppState.CODING ||
appState === AppState.CODE_READY) && (
diff --git a/frontend/src/components/generate-from-text/GenerateFromText.tsx b/frontend/src/components/generate-from-text/GenerateFromText.tsx
new file mode 100644
index 00000000..8d0bd914
--- /dev/null
+++ b/frontend/src/components/generate-from-text/GenerateFromText.tsx
@@ -0,0 +1,72 @@
+import { useState, useRef, useEffect } from "react";
+import { Button } from "../ui/button";
+import { Textarea } from "../ui/textarea";
+import toast from "react-hot-toast";
+
+interface GenerateFromTextProps {
+ doCreateFromText: (text: string) => void;
+}
+
+function GenerateFromText({ doCreateFromText }: GenerateFromTextProps) {
+ const [isOpen, setIsOpen] = useState(false);
+ const [text, setText] = useState("");
+ const textareaRef = useRef<HTMLTextAreaElement>(null);
+
+ useEffect(() => {
+ if (isOpen && textareaRef.current) {
+ textareaRef.current.focus();
+ }
+ }, [isOpen]);
+
+ const handleGenerate = () => {
+ if (text.trim() === "") {
+ toast.error("Please enter a prompt to generate from");
+ return;
+ }
+ doCreateFromText(text);
+ };
+
+ const handleKeyDown = (e: React.KeyboardEvent) => {
+ if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+ e.preventDefault();
+ handleGenerate();
+ }
+ };
+
+ return (
+ <div className="mt-4">
+ {!isOpen ? (
+ <div className="flex justify-center">
+ <Button variant="secondary" onClick={() => setIsOpen(true)}>
+ Generate from text prompt [BETA]
+ </Button>
+ </div>
+ ) : (
+ <>
+ <Textarea
+ ref={textareaRef}
+ rows={2}
+ placeholder="A SaaS admin dashboard with charts and user management"
+ className="w-full mb-4"
+ value={text}
+ onChange={(e) => setText(e.target.value)}
+ onKeyDown={handleKeyDown}
+ />
+ <div className="flex justify-between items-center">
+ <span className="text-sm text-gray-500">
+ Press Cmd/Ctrl + Enter to generate
+ </span>
+ <div className="flex gap-2">
+ <Button variant="outline" onClick={() => setIsOpen(false)}>
+ Cancel
+ </Button>
+ <Button onClick={handleGenerate}>Generate</Button>
+ </div>
+ </div>
+ </>
+ )}
+ </div>
+ );
+}
+
+export default GenerateFromText;
diff --git a/frontend/src/store/project-store.ts b/frontend/src/store/project-store.ts
index 7ae0bda0..9bb62feb 100644
--- a/frontend/src/store/project-store.ts
+++ b/frontend/src/store/project-store.ts
@@ -4,12 +4,14 @@ import { Commit, CommitHash, VariantStatus } from "../components/commits/types";
// Store for app-wide state
interface ProjectStore {
// Inputs
- inputMode: "image" | "video";
- setInputMode: (mode: "image" | "video") => void;
+ inputMode: "image" | "video" | "text";
+ setInputMode: (mode: "image" | "video" | "text") => void;
isImportedFromCode: boolean;
setIsImportedFromCode: (imported: boolean) => void;
referenceImages: string[];
setReferenceImages: (images: string[]) => void;
+ initialPrompt: string;
+ setInitialPrompt: (prompt: string) => void;
// Outputs
commits: Record<string, Commit>;
@@ -50,6 +52,8 @@ export const useProjectStore = create<ProjectStore>((set) => ({
setIsImportedFromCode: (imported) => set({ isImportedFromCode: imported }),
referenceImages: [],
setReferenceImages: (images) => set({ referenceImages: images }),
+ initialPrompt: "",
+ setInitialPrompt: (prompt) => set({ initialPrompt: prompt }),
// Outputs
commits: {},
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 909bfdc8..a5166547 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -33,7 +33,7 @@ export enum ScreenRecorderState {
export interface CodeGenerationParams {
generationType: "create" | "update";
- inputMode: "image" | "video";
+ inputMode: "image" | "video" | "text";
image: string;
history?: string[];
isImportedFromCode?: boolean;