Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Add screenshot preview tool
Abi Rajac86e453aa34a
diff --git a/backend/Dockerfile b/backend/Dockerfileindex 82c2fbbb..265b6302 100644--- a/backend/Dockerfile+++ b/backend/Dockerfile@@ -1,4 +1,4 @@-FROM python:3.12.3-slim-bullseye+FROM python:3.12.3-slim-bookwormENV POETRY_VERSION 1.8.0@@ -17,5 +17,8 @@ RUN poetry config virtualenvs.create false# Install dependenciesRUN poetry install+# Install Chromium and Linux libraries needed by Playwright screenshot previews.+RUN playwright install --with-deps chromium+# Copy the current directory contents into the container at /appCOPY ./ /app/diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.pyindex 17b37d8c..dc301e41 100644--- a/backend/agent/tools/definitions.py+++ b/backend/agent/tools/definitions.py@@ -145,6 +145,13 @@ def _extract_assets_schema() -> Dict[str, Any]:}+def _screenshot_preview_schema() -> Dict[str, Any]:+ return {+ "type": "object",+ "properties": {},+ }++def _retrieve_option_schema() -> Dict[str, Any]:return {"type": "object",@@ -162,6 +169,7 @@ def canonical_tool_definitions(image_generation_enabled: bool = True,image_editing_enabled: bool = True,asset_extraction_enabled: bool = True,+ screenshot_enabled: bool = True,) -> List[CanonicalToolDefinition]:tools: List[CanonicalToolDefinition] = [CanonicalToolDefinition(@@ -231,6 +239,20 @@ def canonical_tool_definitions(parameters=_extract_assets_schema(),))+ if screenshot_enabled:+ tools.append(+ CanonicalToolDefinition(+ name="screenshot_preview",+ description=(+ "Render the current HTML file in a headless browser and return "+ "full-page desktop and mobile screenshots so you can visually "+ "verify your work. Use after creating or substantially editing "+ "the file to check layout, spacing, and fidelity to the "+ "requested design. Screenshots are returned as attached images."+ ),+ parameters=_screenshot_preview_schema(),+ )+ )tools.extend([SAVE_ASSETS_TOOL_DEFINITION,diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.pyindex f17846b4..3c219f79 100644--- a/backend/agent/tools/runtime.py+++ b/backend/agent/tools/runtime.py@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union, castfrom codegen.utils import extract_html_contentfrom config import LOCAL_ASSET_DIR, REPLICATE_API_KEYfrom agent.tools.extract_assets import run_extract_assets+from agent.tools.screenshot_preview import run_screenshot_previewfrom image_generation.generation import process_tasksfrom image_generation.replicate import (P_IMAGE_EDIT_ASPECT_RATIOS,@@ -105,6 +106,12 @@ class AgentToolRuntime:asset_base_url=self.asset_base_url,user_id=self.user_id,)+ if tool_call.name == "screenshot_preview":+ return await run_screenshot_preview(+ tool_call.arguments,+ file_state=self.file_state,+ asset_base_url=self.asset_base_url,+ )if tool_call.name == "save_assets":return await run_save_assets(tool_call.arguments, user_id=self.user_id)if tool_call.name == "retrieve_option":diff --git a/backend/agent/tools/screenshot_preview.py b/backend/agent/tools/screenshot_preview.pynew file mode 100644index 00000000..e2802108--- /dev/null+++ b/backend/agent/tools/screenshot_preview.py@@ -0,0 +1,110 @@+import base64+from typing import Any, Dict++from preview_screenshot import capture_preview_screenshot++from agent.state import AgentFileState+from agent.tools.types import ToolExecutionResult, ToolMultimodalPart+from uploaded_assets.store import (+ persist_data_url_as_temporary_asset,+ promote_temporary_asset_id,+)+++PREVIEW_VIEWPORTS = ("desktop", "mobile")+++async def run_screenshot_preview(+ _args: Dict[str, Any],+ *,+ file_state: AgentFileState,+ asset_base_url: str,+) -> ToolExecutionResult:+ """Render the current HTML in headless Chromium and return screenshots.++ PNGs are attached as multimodal parts so the model can visually verify+ its work; public URLs are kept in the summary for the UI.+ """+ if not file_state.content:+ return ToolExecutionResult(+ ok=False,+ result={"error": "No file exists yet. Call create_file first."},+ summary={"error": "No file to screenshot"},+ )++ screenshots: list[Dict[str, Any]] = []+ multimodal_parts: list[ToolMultimodalPart] = []+ try:+ for viewport in PREVIEW_VIEWPORTS:+ image_bytes = await capture_preview_screenshot(+ file_state.content,+ device=viewport,+ full_page=True,+ )+ display_name = f"preview_{viewport}.png"+ image_part_index = len(multimodal_parts)+ encoded_image = base64.b64encode(image_bytes).decode("ascii")+ data_url = f"data:image/png;base64,{encoded_image}"+ temporary_asset = persist_data_url_as_temporary_asset(+ data_url,+ asset_base_url,+ )+ if not temporary_asset:+ raise Exception(f"Could not persist {viewport} screenshot")+ saved_asset = await promote_temporary_asset_id(temporary_asset.asset_id)+ if not saved_asset:+ raise Exception(f"Could not publish {viewport} screenshot")+ screenshots.append(+ {+ "viewport": viewport,+ "full_page": True,+ "image_part_index": image_part_index,+ "image_display_name": display_name,+ "image_bytes": len(image_bytes),+ "image_url": saved_asset.public_url,+ "status": "ok",+ }+ )+ multimodal_parts.append(+ ToolMultimodalPart(+ display_name=display_name,+ mime_type="image/png",+ data=image_bytes,+ )+ )+ except Exception as exc:+ print(f"Preview screenshot failed: {exc}")+ return ToolExecutionResult(+ ok=False,+ result={"error": f"Screenshot failed: {exc}"},+ summary={"error": "Screenshot failed"},+ )++ result: Dict[str, Any] = {+ "content": (+ "Full-page desktop and mobile screenshots of the current preview "+ "are attached."+ ),+ "details": {+ "screenshots": [+ {+ "viewport": screenshot["viewport"],+ "full_page": screenshot["full_page"],+ "image_part_index": screenshot["image_part_index"],+ "image_display_name": screenshot["image_display_name"],+ "image_bytes": screenshot["image_bytes"],+ }+ for screenshot in screenshots+ ],+ },+ }+ summary: Dict[str, Any] = {+ "screenshots": screenshots,+ "status": "ok",+ }+ return ToolExecutionResult(+ ok=True,+ result=result,+ summary=summary,+ multimodal_parts=multimodal_parts,+ )diff --git a/backend/poetry.lock b/backend/poetry.lockindex 9bc1ef87..68106f21 100644--- a/backend/poetry.lock+++ b/backend/poetry.lock@@ -706,6 +706,99 @@ protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4[package.extras]grpc = ["grpcio (>=1.44.0,<2.0.0)"]+[[package]]+name = "greenlet"+version = "3.5.1"+description = "Lightweight in-process concurrent programming"+optional = false+python-versions = ">=3.10"+groups = ["main"]+files = [+ {file = "greenlet-3.5.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7eacb17a9d41538a2bc4912eba5ef13823c83cb69e4d141d0813debe7163187f"},+ {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5cc9606aa5f4e0bde0d3bd502b44f743864c3ffa5cfa1011b1e30f5aa02366f"},+ {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c3d35f87c7253b715d13d679e0783d845910144f282cb939fe1ba4ac8616269c"},+ {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:00929c98ec525fd9bf075875d8c5f6a983a90906cdf78a66e6de2d8e466c2a19"},+ {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:540dae7b956209af4d70a3be35927b4055f617763771e5e84a5255bea934d2f5"},+ {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:001775efe7b8e758861294c7a27c28af87f3f3f1c20468a2bc618c45b346c061"},+ {file = "greenlet-3.5.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed8cdb691169715a9a492844a83246f090182247d1a5031dc78a403f68ba1e97"},+ {file = "greenlet-3.5.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9d59e840387076a51016777a9328b3f2c427c6f9208a6e958bad251be50a648d"},+ {file = "greenlet-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:b9152fca4a6466e114aaec745ae61cba739903a109754a9d4e1262f01e9259b1"},+ {file = "greenlet-3.5.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:73f78f9b9f0a5c06e5c946ba1e8e36f5114923b6be109ee618c54f079c3ea14f"},+ {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0cbed8bb44e23c5b199f888f4e4ce096b45ad9f25ff74a7ad0213875e936bb2"},+ {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a203a8bd0acb0701653d3bbb26e404854a68674139ed5cbb778830f42b09bb33"},+ {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ebeb75c81211f5c702576cf81f315e77e23cfdb2c7c6fcb9dd143e6de35c360"},+ {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a271fcd66c74615cda6a964fda3f304267a12e50a084472218a39bb0376f563"},+ {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:017a544f0385d441e88714160d089d6900ef46c9eff9d99b6715a5ef2d127747"},+ {file = "greenlet-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ded7b068c7c31c1a8657d4fd42d886b3e051ae29f88b80c5ff9d502257b0f071"},+ {file = "greenlet-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0932b81d72f552ded9d810d00021b64d89f2195a91ce115b893f943b7a4ab3c"},+ {file = "greenlet-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:88e300d136eac057b2397aa1cfd7328b4c87c7eb66a09c7bc6a1292234db474e"},+ {file = "greenlet-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:cc6ab7e555c8a112ad3a76e368e86e12a2754bcae1652a5602e133ec7b635523"},+ {file = "greenlet-3.5.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:fa4f98af3a528f0c3fd592a26df7f376f93329c8f4d987f6bb979057af8bf5e2"},+ {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffea73584b216150eab159b6d12348fb253e68757974de1e2c40d8a318ac89ed"},+ {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1072b4f9edcc1e192d9283a66a3e68d6b84c561de33a83d7858beb9ba1effe10"},+ {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89101bfd5011e069be974903cb3a4e4523845e4ece2d62dcd8d358933c0ef249"},+ {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:add5217d68b31130f0beca584d7fef4878327d2e31642b66618a14eef312b63b"},+ {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:e6cd99ea59dd5d89f0c956606571d79bfe6f68c9eb7f4a4083a41a7f1587edee"},+ {file = "greenlet-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a5ea42a752d47a145eae922b605cd1634665ac3d5ec1e72402d5048e8d60d207"},+ {file = "greenlet-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5551170cf4f5ff5623e9af81323751979fee2c731e2287b61f73cd27257b823"},+ {file = "greenlet-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3c8bb982ad117d29478ef8f5533e97df21f1e2befd17a299257b0c96d1371c0b"},+ {file = "greenlet-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:80eb4b04dadc4e67df3fae179a32c4706a3f495bc7f22fc8a81115d5f5512188"},+ {file = "greenlet-3.5.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:51518ff74664078fc51bffcc6fc529b0df5ae58da192691cee765d45ce944a2b"},+ {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ffdb3c0bb002c99cd8f298957e046c3dbf6006b5b7cdf11a4e19194624a0a0a"},+ {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7715a5a2c3378ba602c3a440558261e13a820bb53a82693aacd7b7f6d964e283"},+ {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d40a890035c0058cadbdc4af7569800fd28a0e527a0fdbb7b5f9418f176846ce"},+ {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc71ff466927a201b08305acac451ebe1aedfcea002f62f1f2f2ac2ac1e6a135"},+ {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:67821bb03e4e98664490edb787ff6af501194c29bbee0f5c1dfdcf1dc3d9d436"},+ {file = "greenlet-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cd443683db272ebaaca03af98c0b063ab30db70ea8a31a1559f35e3f7b744ccd"},+ {file = "greenlet-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:089fff7a6ce8d9316d1f65ebc00273a56be258c1725b32b94de90a3a979557e1"},+ {file = "greenlet-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:110a1ca7b49b014b097f6078272c3f4ed31af45b254de5228b79adba879f6af9"},+ {file = "greenlet-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f16ba1efc0715b680a18b8123d90dad887c6112ae3555b4b5c32c149540c6b4e"},+ {file = "greenlet-3.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8ab31c9de8651a2facdd5c5bb0011f2380dd1a7af78ce2adf4b56095294fc07"},+ {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e300185139abc337ade480c327183adf42a875ac7181bfe66d7d4efea31fbea"},+ {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7ffdb990dcaa0234cf9845aead5df2e3c3a8b6507d409274dd87e0d5ab05ffc2"},+ {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c09df69dc1712d131332054a858a3e5cca400967fa3a672e2324fbb0971448c"},+ {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f82b3597e9d83b63408affed0b48fd0f54935edac4302237b9a837be0dae33c"},+ {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:a4764e0bfc6a4d114c865b32520805c16a990ef5f286a514413b05d5ecd6a23d"},+ {file = "greenlet-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0141e37414c10164e702b8fb1473304221ad98f71600850c6ef7ff4880feba0"},+ {file = "greenlet-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:50ae25a67bea74ea41fb14b960bc532df73eb713417b2d61892dced82fe8d3bc"},+ {file = "greenlet-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:8a17c42330e261299766b75ac1ea32caa437a9453c8f65d16a13140db378ecd3"},+ {file = "greenlet-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:7b5f5fae05b8ac6d176a61b60c394a8cbdc2b5b91b81793066e68745cf165e54"},+ {file = "greenlet-3.5.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:ea8da1e900d758d078810d4255d8c6aa572181896a31ec79d779eb79c3adc9ad"},+ {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a19570c52a21420dcbc94e661994bc325c0b5b11304540fed514586da5dc8f2e"},+ {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3d955c89b75eeca4723d7cc14135f393cd47c32e2a6cb4a8e4c6e760a26b0986"},+ {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea37d5a157eb9493820d3792ac4ece28619a394391d2b9f2f78057d396ff0f0f"},+ {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2daaaebd1a5aa88c49045b6baf9310b3263796bd88db713edf37cf53e7bb4e"},+ {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:8d8a23250ea3ec7b36de8fa4b541e9e2db3ee82915cc060ab0631609ad8b28de"},+ {file = "greenlet-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bfbd69cc349e43bf3a8ae1c85548ff0718efc887615c2db16c3833d7b0b072d"},+ {file = "greenlet-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4378720dd888136c27215a0214d32a4d37c3852765d45bc37aad0623423cfd78"},+ {file = "greenlet-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:45718441607f9325d948db98cbc691276059316d0358c188c246da4e1d4d23d2"},+ {file = "greenlet-3.5.1-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2baee5ca02031757ffe8cc3d69f0cc0aec7065ce362622da74f32d3bcab1c541"},+ {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b1ec3274918a81d3ea778b9e75b56b72b33f300edb6cf7f3a7fe1dae56683de"},+ {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:111e2390ffffc47d5840b01711dd7fac07d4c09283d0283e7f3264b14e284c64"},+ {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:10a9a1c0bfbc93d41156ffcb90c75fbc05544054faf15dcc1fdf9765f8b607f0"},+ {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e630136e905fe5ff43e86945ae41220b6d1470956a39220e708110ac48d01ea5"},+ {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:ef08c1567c78074b22d1a200183d52d04a14df447bf70bcbb6a3507a48e776fc"},+ {file = "greenlet-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:975eac34b44a7077ca4d421348455b94f0f518246a7f14bc6d2fdcfe5b584368"},+ {file = "greenlet-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9ab3c3a0b2ae6198e67c898dad5215a49f9ae0d0081b3c3ec59f333e39eeca26"},+ {file = "greenlet-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:cbfc69be86e10dcfef5b1e6269d1d6926552aa89ee39e1de3353360c1b6989ab"},+ {file = "greenlet-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:92fd6d44ac5e5a887c8a5dc4a8ba0ba908527c31c12f78c6bc7dcfe8aab279f6"},+ {file = "greenlet-3.5.1-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:a6fdf2433a5441ef9a95464f7c3e674775da1c8c1177fff311cee1acad4626ed"},+ {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7546556f0d649f99f6a361098a55f761181bb2ea12ff150bb16d26092ad88244"},+ {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5ee3ea898009fa898f85f9982255d35278c477bebe185beca249cab42d4526c"},+ {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a57b0d05a0448eed231d59c0ceb287dde984551e54cbc51ac2d4865712838e9c"},+ {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5c81f74d204d3edd136ebfd50dce53acbb776995d721a0fe801626cfc93b8cd"},+ {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:b0703c2cef53e01baec47f7a3868009913ad71ec678bbecb42a6f40895e4ce62"},+ {file = "greenlet-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:2c18ef16bf6d4dd410e4dd52996888ea1497be26892fe5bbc73580aba4287b8e"},+ {file = "greenlet-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:17d86354f0ae6b61bf9be5148d0dd34e06c3cb7c602c671f79f29ac3b150e659"},+ {file = "greenlet-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e7516cf6ae6b8a582c2770a0caed47b8a48373ed732c33d69a72913ae6ac923e"},+ {file = "greenlet-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:5028648bf2253ec4745add746129d3904121fa7fe871a76bed23c5720573ce0a"},+ {file = "greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829"},+]++[package.extras]+docs = ["Sphinx", "furo"]+test = ["objgraph", "psutil", "setuptools"]+[[package]]name = "grpcio"version = "1.73.0"@@ -1589,6 +1682,28 @@ docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-atest = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]type = ["mypy (>=1.11.2)"]+[[package]]+name = "playwright"+version = "1.60.0"+description = "A high-level API to automate web browsers"+optional = false+python-versions = ">=3.9"+groups = ["main"]+files = [+ {file = "playwright-1.60.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6a8cd0fec171fb3089e95e898c8bc8a6f35dea0b78b399e12fcc19427e91b1d7"},+ {file = "playwright-1.60.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:39b5420ba6145045b69ced4c5c47d4d9fe5bddfc8ff816c518913afcb25ec7a5"},+ {file = "playwright-1.60.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:2581d0e6a3392c71f91b27460c7fd093356818dc430f48153896c8aeeaef7705"},+ {file = "playwright-1.60.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:1c2bfae7884fb3fb05b853290eab8f343d524e5016f2f1def702acbbdf14c93e"},+ {file = "playwright-1.60.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43e66564125ee31b07a58cefb21e256d62d67d8d1713e6858df7a3019d8ed353"},+ {file = "playwright-1.60.0-py3-none-win32.whl", hash = "sha256:ec94e416ea320711e0ad4bf185dcbf41833672961e90773e1885255d7db7b7e7"},+ {file = "playwright-1.60.0-py3-none-win_amd64.whl", hash = "sha256:9566821ce6030a1f9e7146a24e19355ab0d98805fd0f9be50bb3d8fef1750c02"},+ {file = "playwright-1.60.0-py3-none-win_arm64.whl", hash = "sha256:6e4f6700a4c2250efff8e690a81d66e3855754fb587b6b87cf5c784014f91537"},+]++[package.dependencies]+greenlet = ">=3.1.1,<4.0.0"+pyee = ">=13,<14"+[[package]]name = "pluggy"version = "1.5.0"@@ -1913,6 +2028,24 @@ files = [[package.dependencies]typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"+[[package]]+name = "pyee"+version = "13.0.1"+description = "A rough port of Node.js's EventEmitter to Python with a few tricks of its own"+optional = false+python-versions = ">=3.8"+groups = ["main"]+files = [+ {file = "pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228"},+ {file = "pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8"},+]++[package.dependencies]+typing-extensions = "*"++[package.extras]+dev = ["black", "build", "flake8", "flake8-black", "isort", "jupyter-console", "mkdocs", "mkdocs-include-markdown-plugin", "mkdocstrings[python]", "mypy", "pytest", "pytest-asyncio ; python_version >= \"3.4\"", "pytest-trio ; python_version >= \"3.7\"", "sphinx", "toml", "tox", "trio", "trio ; python_version > \"3.6\"", "trio-typing ; python_version > \"3.6\"", "twine", "twisted", "validate-pyproject[all]"]+[[package]]name = "pyright"version = "1.1.408"@@ -2606,4 +2739,4 @@ type = ["pytest-mypy"][metadata]lock-version = "2.1"python-versions = "^3.10"-content-hash = "b86b4d62ab193770370654df1c016d050d0d0c440a09d9e557b6da6d94a473a5"+content-hash = "aa170f15569774ed36faf04840cbb8c3bae016143383838d67a4ddee22a04f12"diff --git a/backend/preview_screenshot.py b/backend/preview_screenshot.pynew file mode 100644index 00000000..6a1e0bc4--- /dev/null+++ b/backend/preview_screenshot.py@@ -0,0 +1,73 @@+import asyncio+from typing import Dict, Optional, Tuple++from playwright.async_api import (+ Browser,+ Playwright,+ TimeoutError as PlaywrightTimeoutError,+ async_playwright,+)++VIEWPORT_SIZES: Dict[str, Tuple[int, int]] = {+ "desktop": (1280, 832),+ "mobile": (342, 684),+}++PAGE_LOAD_TIMEOUT_MS = 15000+RENDER_SETTLE_MS = 250++_playwright: Optional[Playwright] = None+_browser: Optional[Browser] = None+_browser_lock = asyncio.Lock()+++async def _get_browser() -> Browser:+ global _playwright, _browser+ async with _browser_lock:+ if _browser is None or not _browser.is_connected():+ if _playwright is None:+ _playwright = await async_playwright().start()+ # --no-sandbox: Chromium refuses to launch as root (the user in+ # most containers/hosted Linux) unless the sandbox is disabled.+ _browser = await _playwright.chromium.launch(+ headless=True,+ args=["--no-sandbox"],+ )+ return _browser+++async def capture_preview_screenshot(+ html: str,+ device: str = "desktop",+ full_page: bool = True,+) -> bytes:+ """Render HTML in headless Chromium and return PNG bytes.++ Runs locally, so the page can load assets served from localhost+ (e.g. /local-assets/ URLs) that an external screenshot API cannot reach.+ """+ browser = await _get_browser()+ width, height = VIEWPORT_SIZES.get(device, VIEWPORT_SIZES["desktop"])+ page = await browser.new_page(+ viewport={"width": width, "height": height},+ device_scale_factor=1,+ )+ try:+ try:+ await page.set_content(+ html,+ wait_until="networkidle",+ timeout=PAGE_LOAD_TIMEOUT_MS,+ )+ except PlaywrightTimeoutError:+ # Content is already set; capture whatever rendered if the+ # network never settles (e.g. pages that poll).+ pass+ try:+ await page.evaluate("document.fonts.ready")+ except Exception:+ pass+ await page.wait_for_timeout(RENDER_SETTLE_MS)+ return await page.screenshot(full_page=full_page, type="png")+ finally:+ await page.close()diff --git a/backend/prompts/system_prompt.py b/backend/prompts/system_prompt.pyindex 66d34f2e..98d24b8a 100644--- a/backend/prompts/system_prompt.py+++ b/backend/prompts/system_prompt.py@@ -19,6 +19,7 @@ You are a coding agent that's an expert at building front-ends.- If an asset in the original screenshot is not extractable (for example, occluded by other objects or is the background), when available, use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background. NEVER USE this tool to extract the entire screenshot and embed it on the page. Our goal here is to create nicely coded pages. We should only use extracted assets for images, not for layout, etc.- Use remove_background to remove backgrounds from provided image URLs when needed (you may pass multiple image URLs).- Use retrieve_option to fetch the full HTML for a specific option (1-based option_number) when a user references another option.+- When available, always call screenshot_preview once after create_file (or after substantial edit_file changes) to see full-page desktop and mobile renderings of your current HTML and verify they match the requested design. If you spot visual problems (broken layout, overlapping elements, wrong spacing or colors), fix them with edit_file. Screenshots are slow, so take at most three screenshot_preview calls per task.# Stack-specific instructionsdiff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex a12d8594..1d89d6c6 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -24,6 +24,7 @@ aiohttp = "^3.9.5"pydantic = "^2.10"google-genai = "^1.16.1"langfuse = "^3.0.2"+playwright = "^1.60.0"[tool.poetry.group.dev.dependencies]pytest = "^7.4.3"diff --git a/backend/tests/test_agent_tool_runtime.py b/backend/tests/test_agent_tool_runtime.pyindex 85328ff2..a0518036 100644--- a/backend/tests/test_agent_tool_runtime.py+++ b/backend/tests/test_agent_tool_runtime.py@@ -1,5 +1,6 @@import base64from pathlib import Path+from typing import Any, castimport pytest@@ -214,3 +215,133 @@ async def test_extract_assets_returns_mocked_gemini_assets(]assert result.multimodal_parts[0].mime_type == "image/png"assert result.multimodal_parts[0].data == b"logo-image"+++async def test_screenshot_preview_requires_file_content() -> None:+ runtime = AgentToolRuntime(+ file_state=AgentFileState(),+ should_generate_images=False,+ openai_api_key=None,+ openai_base_url=None,+ )++ result = await runtime.execute(+ ToolCall(id="call-1", name="screenshot_preview", arguments={})+ )++ assert result.ok is False+ assert "create_file" in result.result["error"]+++async def test_screenshot_preview_returns_image_part(+ monkeypatch: pytest.MonkeyPatch,+ tmp_path: Path,+) -> None:+ temp_dir = tmp_path / "tmp-assets"+ asset_dir = tmp_path / "local-assets"+ monkeypatch.setattr("uploaded_assets.store.TEMP_ASSET_DIR", str(temp_dir))+ monkeypatch.setattr("uploaded_assets.store.LOCAL_ASSET_DIR", str(asset_dir))++ captured: list[dict[str, object]] = []++ async def fake_capture(+ html: str,+ device: str = "desktop",+ full_page: bool = True,+ ) -> bytes:+ captured.append({"html": html, "device": device, "full_page": full_page})+ return f"{device}-png-bytes".encode("ascii")++ monkeypatch.setattr(+ "agent.tools.screenshot_preview.capture_preview_screenshot", fake_capture+ )+ runtime = AgentToolRuntime(+ file_state=AgentFileState(path="index.html", content="<main>hi</main>"),+ should_generate_images=False,+ openai_api_key=None,+ openai_base_url=None,+ asset_base_url="http://127.0.0.1:7001",+ )++ result = await runtime.execute(+ ToolCall(+ id="call-1",+ name="screenshot_preview",+ arguments={"ignored": "value"},+ )+ )++ assert result.ok is True+ assert captured == [+ {"html": "<main>hi</main>", "device": "desktop", "full_page": True},+ {"html": "<main>hi</main>", "device": "mobile", "full_page": True},+ ]+ assert result.result["details"]["screenshots"] == [+ {+ "viewport": "desktop",+ "full_page": True,+ "image_part_index": 0,+ "image_display_name": "preview_desktop.png",+ "image_bytes": len(b"desktop-png-bytes"),+ },+ {+ "viewport": "mobile",+ "full_page": True,+ "image_part_index": 1,+ "image_display_name": "preview_mobile.png",+ "image_bytes": len(b"mobile-png-bytes"),+ },+ ]+ assert result.summary["status"] == "ok"+ screenshots = cast(list[dict[str, Any]], result.summary["screenshots"])+ assert screenshots[0]["image_url"].startswith(+ "http://127.0.0.1:7001/local-assets/"+ )+ assert screenshots[1]["image_url"].startswith(+ "http://127.0.0.1:7001/local-assets/"+ )+ assert screenshots[0]["image_url"] != screenshots[1]["image_url"]+ # UI URLs and image payloads must not leak into the model-facing result payload.+ assert "image_url" not in result.result["details"]["screenshots"][0]+ assert "image_data_url" not in result.result["details"]["screenshots"][0]+ assert len(list(asset_dir.iterdir())) == 2+ assert result.multimodal_parts is not None+ assert [part.display_name for part in result.multimodal_parts] == [+ "preview_desktop.png",+ "preview_mobile.png",+ ]+ assert result.multimodal_parts[0].mime_type == "image/png"+ assert result.multimodal_parts[0].data == b"desktop-png-bytes"+ assert result.multimodal_parts[1].mime_type == "image/png"+ assert result.multimodal_parts[1].data == b"mobile-png-bytes"+++async def test_screenshot_preview_reports_capture_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def failing_capture(+ html: str,+ device: str = "desktop",+ full_page: bool = True,+ ) -> bytes:+ raise Exception("boom")++ monkeypatch.setattr(+ "agent.tools.screenshot_preview.capture_preview_screenshot", failing_capture+ )+ runtime = AgentToolRuntime(+ file_state=AgentFileState(path="index.html", content="<main>hi</main>"),+ should_generate_images=False,+ openai_api_key=None,+ openai_base_url=None,+ )++ result = await runtime.execute(+ ToolCall(id="call-1", name="screenshot_preview", arguments={})+ )++ assert result.ok is False+ assert "boom" in result.result["error"]diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.pyindex 9c665507..9bf6ca90 100644--- a/backend/tests/test_agent_tools.py+++ b/backend/tests/test_agent_tools.py@@ -120,6 +120,24 @@ def test_extract_assets_tool_input_summary_uses_asset_descriptions() -> None:}+def test_canonical_tool_definitions_include_screenshot_preview() -> None:+ tools = canonical_tool_definitions(True)+ tool_names = [tool.name for tool in tools]+ assert "screenshot_preview" in tool_names+ screenshot_tool = next(+ tool for tool in tools if tool.name == "screenshot_preview"+ )+ assert screenshot_tool.parameters["properties"] == {}+++def test_canonical_tool_definitions_exclude_screenshot_preview_when_disabled() -> None:+ tool_names = [+ tool.name+ for tool in canonical_tool_definitions(True, screenshot_enabled=False)+ ]+ assert "screenshot_preview" not in tool_names++def test_canonical_tool_definitions_include_edit_image() -> None:tools = canonical_tool_definitions(True)tool_names = [tool.name for tool in tools]diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsxindex 162f5fd2..3970389a 100644--- a/frontend/src/components/agent/AgentActivity.tsx+++ b/frontend/src/components/agent/AgentActivity.tsx@@ -18,6 +18,7 @@ import {BsFiles,BsBookmarkCheck,BsBoundingBox,+ BsCamera,} from "react-icons/bs";import ReactMarkdown from "react-markdown";import { Light as SyntaxHighlighterBase } from "react-syntax-highlighter";@@ -154,6 +155,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) {if (toolName === "extract_assets") {return <BsBoundingBox className="text-orange-500" />;}+ if (toolName === "screenshot_preview") {+ return <BsCamera className="text-cyan-500" />;+ }return <BsFileEarmarkPlus className="text-gray-500" />;}@@ -245,6 +249,11 @@ function getEventTitle(event: AgentEvent): string {? `Extracted ${extractCount} assets`: "Extracted asset";}+ if (event.toolName === "screenshot_preview") {+ return event.status === "running"+ ? "Screenshotting preview"+ : "Screenshotted preview";+ }return event.status === "running" ? "Running tool" : "Tool completed";}return "Activity";@@ -287,6 +296,10 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) {hasError && successfulExtractedAssets? successfulExtractedAssets: extractedAssets;+ const screenshotPreviews =+ output && Array.isArray(output.screenshots)+ ? (output.screenshots as Array<unknown>)+ : [];return (<div className="text-sm text-gray-700 dark:text-gray-200">@@ -721,6 +734,52 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) {</div>)}+ {event.toolName === "screenshot_preview" && !hasError && (+ <div>+ {event.status === "running" && (+ <div className="text-xs text-gray-600 dark:text-gray-400 py-1.5">+ Rendering desktop and mobile previews...+ </div>+ )}+ {event.status !== "running" && (+ <div className="grid gap-3 py-2 sm:grid-cols-2">+ {(["desktop", "mobile"] as const).map((viewport) => {+ const screenshot = screenshotPreviews.find((item) => {+ const itemRecord = getRecord(item);+ return itemRecord?.viewport === viewport;+ });+ const screenshotRecord = getRecord(screenshot);+ const imageUrl =+ typeof screenshotRecord?.image_url === "string"+ ? screenshotRecord.image_url+ : null;+ return (+ <div key={viewport}>+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1 capitalize">+ {viewport}+ </div>+ {imageUrl ? (+ <div className="max-h-96 overflow-y-auto rounded border border-gray-200 dark:border-gray-700">+ <img+ src={imageUrl}+ alt={`Screenshot of the generated ${viewport} preview`}+ className="w-full"+ loading="lazy"+ />+ </div>+ ) : (+ <div className="aspect-square rounded bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">+ Missing+ </div>+ )}+ </div>+ );+ })}+ </div>+ )}+ </div>+ )}+{!event.toolName && !hasError && (<>{event.input && (