Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python
8.4 KB
Raw Blame
import base64
from typing import Any, Dict, List, Optional, cast

from asset_extraction import extract_assets_from_images
from uploaded_assets.store import persist_data_url_as_asset

from agent.state import ensure_str
from agent.tools.summaries import summarize_text
from agent.tools.types import ToolExecutionResult, ToolMultimodalPart


IMAGE_EXTENSION_BY_MIME_TYPE = {
    "image/jpeg": ".jpg",
    "image/png": ".png",
    "image/webp": ".webp",
}


def _image_data_url_to_multimodal_part(
    data_url: str,
    display_name: str,
) -> ToolMultimodalPart | None:
    if not data_url.startswith("data:image/") or "," not in data_url:
        return None

    header, encoded = data_url.split(",", 1)
    mime_type = header.removeprefix("data:").split(";", 1)[0].lower()
    if mime_type not in {"image/png", "image/jpeg", "image/webp"}:
        return None

    try:
        data = base64.b64decode(encoded, validate=True)
    except ValueError:
        return None

    return ToolMultimodalPart(
        display_name=display_name,
        mime_type=mime_type,
        data=data,
    )


async def run_extract_assets(
    args: Dict[str, Any],
    *,
    gemini_api_key: Optional[str],
    input_images: List[str],
    asset_base_url: str,
    user_id: Optional[str],
) -> ToolExecutionResult:
    if not gemini_api_key:
        return ToolExecutionResult(
            ok=False,
            result={"error": "Asset extraction requires GEMINI_API_KEY."},
            summary={"error": "Missing Gemini API key"},
        )

    if not input_images:
        return ToolExecutionResult(
            ok=False,
            result={"error": "No input images are available for asset extraction."},
            summary={"error": "No input images"},
        )

    raw_descriptions = args.get("asset_descriptions") or args.get("descriptions")
    if not isinstance(raw_descriptions, list) or not raw_descriptions:
        return ToolExecutionResult(
            ok=False,
            result={
                "error": "extract_assets requires a non-empty asset_descriptions list"
            },
            summary={"error": "Missing asset_descriptions"},
        )

    raw_description_items = cast(list[object], raw_descriptions)
    descriptions: List[str] = [
        description.strip()
        for description in raw_description_items
        if isinstance(description, str) and description.strip()
    ]
    if not descriptions:
        return ToolExecutionResult(
            ok=False,
            result={"error": "No valid asset descriptions provided"},
            summary={"error": "No valid asset_descriptions"},
        )

    try:
        extraction_result: Dict[str, Any] = await extract_assets_from_images(
            image_data_urls=input_images,
            asset_descriptions=descriptions,
            gemini_api_key=gemini_api_key,
        )
    except Exception as exc:
        # A provider failure (quota, auth, timeout, transient network) must not
        # abort the whole variant. Like the other external image tools, report it
        # as a tool error so the agent can continue or fall back (e.g. redraw with
        # generate_images) instead of propagating out of the tool runtime.
        print(f"Asset extraction failed: {exc}")
        return ToolExecutionResult(
            ok=False,
            result={"error": f"Asset extraction failed: {exc}"},
            summary={"error": "Asset extraction failed"},
        )
    assets = extraction_result.get("assets", [])
    promoted_assets: List[Dict[str, Any]] = []
    multimodal_parts: List[ToolMultimodalPart] = []
    if isinstance(assets, list):
        for index, raw_asset in enumerate(cast(list[object], assets)):
            if not isinstance(raw_asset, dict):
                continue
            asset = cast(Dict[str, Any], raw_asset)

            data_url = ensure_str(asset.get("data_url"))
            public_url: str | None = None
            content_type: str | None = None
            image_part_index: int | None = None
            image_display_name: str | None = None
            status = ensure_str(asset.get("status")) or "missing"

            if data_url:
                mime_type = data_url.split(";", 1)[0].removeprefix("data:")
                extension = IMAGE_EXTENSION_BY_MIME_TYPE.get(mime_type, ".png")
                display_name = f"asset_{index}{extension}"
                multimodal_part = _image_data_url_to_multimodal_part(
                    data_url,
                    display_name,
                )
                if multimodal_part:
                    multimodal_parts.append(multimodal_part)
                    image_part_index = len(multimodal_parts) - 1
                    image_display_name = display_name

                # Extraction is the commitment — the model asked for this crop,
                # so finalize it straight to a served asset (no temp staging).
                saved_asset = await persist_data_url_as_asset(
                    data_url,
                    asset_base_url,
                    user_id=user_id,
                )
                if saved_asset:
                    public_url = saved_asset.public_url
                    content_type = saved_asset.content_type
                    # No asset_id is surfaced on purpose: extracted crops are
                    # already finalized, and a save_assets-style id makes models
                    # redundantly call save_assets on them (which now fails — the
                    # crops were never temp-staged). The model embeds public_url.
                    status = "ok"
                else:
                    status = "error"

            promoted_assets.append(
                {
                    "description": asset.get("description"),
                    "public_url": public_url,
                    "content_type": content_type,
                    "status": status,
                    "image_part_index": image_part_index,
                    "image_display_name": image_display_name,
                    "box_2d": asset.get("box_2d"),
                    "image_index": asset.get("image_index"),
                    "label": asset.get("label"),
                }
            )

    # extraction_result carries a top-level "error" whenever Gemini couldn't box
    # every requested asset. Don't let that fail the whole tool when we still got
    # usable crops — re-derive tool status from what actually came back.
    extraction_note = ensure_str(extraction_result.get("error")) or None
    result: Dict[str, Any] = {
        **extraction_result,
        "assets": promoted_assets,
    }
    result.pop("error", None)

    extracted = [asset for asset in promoted_assets if asset.get("status") == "ok"]
    unresolved = [asset for asset in promoted_assets if asset.get("status") != "ok"]

    # Partial success is still success: keep the crops Gemini did find (and their
    # images) and tell the model which descriptions came back empty, so it can
    # fall back for just those (e.g. generate_images) instead of discarding the
    # lot. Only fail the tool when nothing could be extracted at all.
    if unresolved:
        result["unresolved_assets"] = [
            {
                "description": asset.get("description"),
                "status": asset.get("status"),
            }
            for asset in unresolved
        ]
    if not extracted:
        result["error"] = (
            extraction_note or "No assets could be extracted from the input images."
        )

    summary_assets: List[Dict[str, Any]] = []
    for asset in promoted_assets:
        public_url = ensure_str(asset.get("public_url"))
        summary_assets.append(
            {
                "description": summarize_text(
                    ensure_str(asset.get("description")), 120
                ),
                "status": ensure_str(asset.get("status")),
                "public_url": public_url,
                "content_type": ensure_str(asset.get("content_type")),
                "image_part_index": asset.get("image_part_index"),
                "image_display_name": asset.get("image_display_name"),
                "box_2d": asset.get("box_2d"),
                "image_index": asset.get("image_index"),
                "label": ensure_str(asset.get("label")),
            }
        )

    return ToolExecutionResult(
        ok=bool(extracted),
        result=result,
        summary={
            "assets": summary_assets,
            "unresolved_count": len(unresolved),
            "error": result.get("error"),
        },
        multimodal_parts=multimodal_parts,
    )