Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Pass file state for agent edits

Abi Rajaa4789322315c
diff --git a/backend/agentic/runner.py b/backend/agentic/runner.py
index 279b0361..036945ed 100644
--- a/backend/agentic/runner.py
+++ b/backend/agentic/runner.py
@@ -295,6 +295,7 @@ class AgenticRunner:
gemini_api_key: Optional[str],
should_generate_images: bool,
image_cache: Dict[str, str],
+ initial_file_state: Optional[Dict[str, str]] = None,
):
self.send_message = send_message
self.variant_index = variant_index
@@ -306,6 +307,9 @@ class AgenticRunner:
self.image_cache = image_cache
self.file_state = AgentFileState()
+ if initial_file_state and initial_file_state.get("content"):
+ self.file_state.path = initial_file_state.get("path") or "index.html"
+ self.file_state.content = initial_file_state["content"]
self.toolbox = AgentToolbox(
self.file_state,
self.image_cache,
@@ -319,6 +323,49 @@ class AgenticRunner:
def _next_event_id(self, prefix: str) -> str:
return f"{prefix}-{self.variant_index}-{uuid.uuid4().hex[:8]}"
+ def _extract_text_content(self, message: ChatCompletionMessageParam) -> str:
+ content = message.get("content", "")
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ for part in content:
+ if isinstance(part, dict) and part.get("type") == "text":
+ return _ensure_str(part.get("text"))
+ return ""
+
+ def _seed_file_state_from_messages(
+ self, prompt_messages: List[ChatCompletionMessageParam]
+ ) -> None:
+ if self.file_state.content:
+ return
+ for message in reversed(prompt_messages):
+ if message.get("role") != "assistant":
+ continue
+ raw_text = self._extract_text_content(message)
+ if not raw_text:
+ continue
+ extracted = extract_html_content(raw_text)
+ self.file_state.content = extracted or raw_text
+ if not self.file_state.path:
+ self.file_state.path = "index.html"
+ return
+ if prompt_messages:
+ system_message = prompt_messages[0]
+ if system_message.get("role") == "system":
+ system_text = self._extract_text_content(system_message)
+ markers = [
+ "Here is the code of the app:",
+ "Here is the code of the SVG:",
+ ]
+ for marker in markers:
+ if marker in system_text:
+ raw_text = system_text.split(marker, 1)[1].strip()
+ extracted = extract_html_content(raw_text)
+ self.file_state.content = extracted or raw_text
+ if not self.file_state.path:
+ self.file_state.path = "index.html"
+ return
+
async def _send(
self,
msg_type: str,
@@ -505,6 +552,7 @@ class AgenticRunner:
return openai_tools, anthropic_tools, gemini_tools
async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:
+ self._seed_file_state_from_messages(prompt_messages)
if model in OPENAI_MODELS:
return await self._run_openai(model, prompt_messages)
if model in ANTHROPIC_MODELS:
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index 601311ab..ef6807fe 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -246,6 +246,7 @@ class ExtractedParams:
prompt: PromptContent
history: List[Dict[str, Any]]
is_imported_from_code: bool
+ file_state: Dict[str, str] | None
class ParameterExtractionStage:
@@ -310,6 +311,15 @@ class ParameterExtractionStage:
# Extract imported code flag
is_imported_from_code = params.get("isImportedFromCode", False)
+ # Extract file state for agentic edits
+ raw_file_state = params.get("fileState")
+ file_state: Dict[str, str] | None = None
+ if isinstance(raw_file_state, dict):
+ content = raw_file_state.get("content")
+ if isinstance(content, str) and content.strip():
+ path = raw_file_state.get("path") or "index.html"
+ file_state = {"path": path, "content": content}
+
return ExtractedParams(
stack=validated_stack,
input_mode=validated_input_mode,
@@ -321,6 +331,7 @@ class ParameterExtractionStage:
prompt=prompt,
history=history,
is_imported_from_code=is_imported_from_code,
+ file_state=file_state,
)
def _get_from_settings_dialog_or_env(
@@ -832,6 +843,7 @@ class AgenticGenerationStage:
gemini_api_key: str | None,
should_generate_images: bool,
image_cache: Dict[str, str],
+ file_state: Dict[str, str] | None,
):
self.send_message = send_message
self.openai_api_key = openai_api_key
@@ -840,6 +852,7 @@ class AgenticGenerationStage:
self.gemini_api_key = gemini_api_key
self.should_generate_images = should_generate_images
self.image_cache = image_cache
+ self.file_state = file_state
async def process_variants(
self,
@@ -881,6 +894,7 @@ class AgenticGenerationStage:
gemini_api_key=self.gemini_api_key,
should_generate_images=self.should_generate_images,
image_cache=self.image_cache,
+ initial_file_state=self.file_state,
)
completion = await runner.run(model, prompt_messages)
if completion:
@@ -1091,6 +1105,7 @@ class CodeGenerationMiddleware(Middleware):
gemini_api_key=GEMINI_API_KEY,
should_generate_images=context.extracted_params.should_generate_images,
image_cache=context.image_cache,
+ file_state=context.extracted_params.file_state,
)
context.variant_completions = (
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 48cb2831..5f0b16c4 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -384,6 +384,10 @@ function App() {
throw new Error("Update called with no head");
}
+ const currentCommit = commits[head];
+ const currentCode =
+ currentCommit?.variants[currentCommit.selectedVariantIndex]?.code || "";
+
let historyTree;
try {
historyTree = extractHistory(head, commits);
@@ -418,6 +422,12 @@ function App() {
: { text: "", images: [referenceImages[0]] },
history: updatedHistory,
isImportedFromCode,
+ fileState: currentCode
+ ? {
+ path: "index.html",
+ content: currentCode,
+ }
+ : undefined,
});
setUpdateInstruction("");
diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsx
index 39b28606..ebf47b01 100644
--- a/frontend/src/components/agent/AgentActivity.tsx
+++ b/frontend/src/components/agent/AgentActivity.tsx
@@ -33,12 +33,13 @@ function formatDuration(startedAt?: number, endedAt?: number): string {
function formatTotalDuration(events: AgentEvent[]): string {
if (events.length === 0) return "";
- const start = Math.min(...events.map((event) => event.startedAt));
const now = Date.now();
- const end = Math.max(
- ...events.map((event) => event.endedAt ?? now)
- );
- const seconds = Math.max(1, Math.round((end - start) / 1000));
+ const totalMs = events.reduce((sum, event) => {
+ const end = event.endedAt ?? now;
+ const duration = Math.max(0, end - event.startedAt);
+ return sum + duration;
+ }, 0);
+ const seconds = Math.max(1, Math.round(totalMs / 1000));
return `${seconds}s`;
}
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 1e3d0203..1fe01eff 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -42,6 +42,10 @@ export interface CodeGenerationParams {
prompt: PromptContent;
history?: PromptContent[];
isImportedFromCode?: boolean;
+ fileState?: {
+ path: string;
+ content: string;
+ };
}
export type FullGenerationSettings = CodeGenerationParams & Settings;