Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Improve screenshot asset extraction accuracy and robustness

Flightdeck Agent9fed693c18b3
diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.py
index d4d8a1da..57060a19 100644
--- a/backend/agent/tools/definitions.py
+++ b/backend/agent/tools/definitions.py
@@ -135,8 +135,13 @@ def _extract_assets_schema() -> Dict[str, Any]:
"items": {
"type": "string",
"description": (
- "Short text description of one visual asset to extract "
- "from the input image."
+ "Identify exactly one visual-asset occurrence. Include its "
+ "distinctive appearance (colors, shape, content, or visible "
+ "wordmark), precise location, nearby UI/context, and the "
+ "1-based screenshot number when multiple screenshots are "
+ "available. For repeated lookalikes, use a separate item for "
+ "each wanted instance and distinguish it (for example, "
+ "leftmost vs. rightmost); do not give only a generic category."
),
},
},
@@ -231,11 +236,15 @@ def canonical_tool_definitions(
CanonicalToolDefinition(
name="extract_assets",
description=(
- "Extract one or more visual assets from the input screenshot or "
- "reference images using Gemini. Pass a list of text descriptions "
- "for the assets to extract. Returns each asset (in the same order) "
- "with a permanent, embeddable public_url. These assets are already "
- "saved — do NOT call save_assets on them (save_assets is only for "
+ "Extract one or more tightly cropped visual assets from the input "
+ "screenshots or reference images using Gemini. Describe exactly "
+ "one occurrence per list item with distinctive appearance, precise "
+ "location, nearby context, and the 1-based screenshot number; "
+ "distinguish repeated lookalikes instead of naming a generic asset. "
+ "Returns each asset in request order with a permanent, embeddable "
+ "public_url and an attached crop preview; genuinely absent or "
+ "unisolatable items are unresolved. These assets are already saved "
+ "— do NOT call save_assets on them (save_assets is only for "
"user-uploaded images)."
),
parameters=_extract_assets_schema(),
diff --git a/backend/asset_extraction.py b/backend/asset_extraction.py
index a37f2311..08ea6549 100644
--- a/backend/asset_extraction.py
+++ b/backend/asset_extraction.py
@@ -1,16 +1,75 @@
+import asyncio
import base64
import io
import json
-import re
-from dataclasses import dataclass
-from typing import Any, Dict, List, cast
+import math
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Sequence, cast
+import pillow_heif
from google import genai
from google.genai import types
-from PIL import Image
+from PIL import Image, ImageOps
+from pydantic import BaseModel, Field, ValidationError
ASSET_EXTRACTION_GEMINI_MODEL = "gemini-3.6-flash"
+MAX_ASSETS_PER_GEMINI_REQUEST = 25
+DEFAULT_ASSET_MEDIA_RESOLUTION = (
+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH
+)
+SUPPORTED_GEMINI_IMAGE_MIME_TYPES = frozenset(
+ {
+ "image/png",
+ "image/jpeg",
+ "image/webp",
+ "image/heic",
+ "image/heif",
+ }
+)
+
+# Gemini accepts HEIC/HEIF, but Pillow needs its libheif-backed plugin to decode
+# those pixels before EXIF normalization and cropping. pillow-heif's public
+# function currently lacks a complete type annotation, so narrow it here.
+_register_heif_opener = cast(Callable[[], None], pillow_heif.register_heif_opener)
+_register_heif_opener()
+
+
+class AssetDetection(BaseModel):
+ """One schema-constrained answer, correlated by our stable request ID."""
+
+ request_id: str = Field(
+ description="The unchanged request_id from the corresponding input request."
+ )
+ image_index: int | None = Field(
+ description=(
+ "1-based source image number containing the requested asset, or null when "
+ "the exact asset is absent, ambiguous, or cannot be isolated."
+ )
+ )
+ box_2d: list[float] | None = Field(
+ description=(
+ "Tight [ymin, xmin, ymax, xmax] bounds normalized to 0-1000, or null "
+ "when image_index is null."
+ ),
+ min_length=4,
+ max_length=4,
+ )
+ label: str | None = Field(
+ description=(
+ "A short label that distinguishes this occurrence from lookalikes, or "
+ "null when the asset was not found."
+ )
+ )
+
+
+class AssetDetectionBatch(BaseModel):
+ detections: list[AssetDetection] = Field(
+ description="Exactly one detection for every requested request_id.",
+ max_length=MAX_ASSETS_PER_GEMINI_REQUEST,
+ )
@dataclass(frozen=True)
@@ -18,180 +77,366 @@ class SourceImage:
part: types.Part
image: Image.Image
mime_type: str
+ image_index: int
+
+
+@dataclass(frozen=True)
+class AssetRequest:
+ request_id: str
+ description: str
+
+
+def _empty_float_list() -> list[float]:
+ return []
+
+
+def _empty_string_list() -> list[str]:
+ return []
+
+
+@dataclass
+class AssetExtractionMetrics:
+ """Optional request telemetry used by the opt-in live benchmark."""
+
+ request_count: int = 0
+ prompt_token_count: int = 0
+ candidate_token_count: int = 0
+ thoughts_token_count: int = 0
+ total_token_count: int = 0
+ request_latencies_seconds: list[float] = field(
+ default_factory=_empty_float_list
+ )
+ response_ids: list[str] = field(default_factory=_empty_string_list)
+
+ def record_response(
+ self,
+ response: types.GenerateContentResponse,
+ latency_seconds: float,
+ ) -> None:
+ self.request_latencies_seconds.append(latency_seconds)
+ if response.response_id:
+ self.response_ids.append(response.response_id)
+
+ usage = response.usage_metadata
+ if usage is None:
+ return
+ self.prompt_token_count += usage.prompt_token_count or 0
+ self.candidate_token_count += usage.candidates_token_count or 0
+ self.thoughts_token_count += usage.thoughts_token_count or 0
+ self.total_token_count += usage.total_token_count or 0
+
+
+ASSET_DETECTION_SYSTEM_INSTRUCTION = """You are a precise 2D object detector for screenshot asset extraction.
+Return only one schema-valid JSON object. Never return masks, segmentation, markdown fences, prose, or invented detections. Process no more than 25 requested objects per response.
+"""
+
+
+def _normalize_image_for_detection(image: Image.Image) -> Image.Image:
+ """Apply EXIF orientation once and produce stable PNG-compatible pixels."""
+
+ oriented = ImageOps.exif_transpose(image)
+ has_alpha = "A" in oriented.getbands() or "transparency" in oriented.info
+ normalized = oriented.convert("RGBA" if has_alpha else "RGB")
+ normalized.load()
+ return normalized
-def _data_url_to_source_image(data_url: str) -> SourceImage | None:
+def _data_url_to_source_image(
+ data_url: str,
+ *,
+ image_index: int = 1,
+ media_resolution: types.PartMediaResolutionLevel = DEFAULT_ASSET_MEDIA_RESOLUTION,
+) -> SourceImage | None:
if not data_url.startswith("data:image/") or "," not in data_url:
return None
header, encoded = data_url.split(",", 1)
mime_type = header.removeprefix("data:").split(";", 1)[0].lower()
+ if mime_type not in SUPPORTED_GEMINI_IMAGE_MIME_TYPES:
+ return None
+
try:
image_bytes = base64.b64decode(encoded, validate=True)
- image = Image.open(io.BytesIO(image_bytes))
- image.load()
+ with Image.open(io.BytesIO(image_bytes)) as opened_image:
+ opened_image.seek(0)
+ normalized_image = _normalize_image_for_detection(opened_image)
except Exception:
return None
+ # Gemini must see the exact same oriented pixel matrix that we later crop.
+ # Canonical PNG also prevents a misleading/unsupported data-URL MIME header
+ # from putting model coordinates in a different orientation than Pillow.
+ normalized_output = io.BytesIO()
+ normalized_image.save(normalized_output, format="PNG")
+ normalized_bytes = normalized_output.getvalue()
+ normalized_mime_type = "image/png"
+
return SourceImage(
part=types.Part.from_bytes(
- data=image_bytes,
- mime_type=mime_type,
- media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,
+ data=normalized_bytes,
+ mime_type=normalized_mime_type,
+ media_resolution=media_resolution,
),
- image=image,
- mime_type=mime_type,
+ image=normalized_image,
+ mime_type=normalized_mime_type,
+ image_index=image_index,
)
-def _extract_json_object(text: str) -> Dict[str, Any] | None:
- stripped = text.strip()
- if stripped.startswith("```"):
- stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
- stripped = re.sub(r"\s*```$", "", stripped)
+def _stable_request_id(original_index: int) -> str:
+ return f"asset-{original_index + 1:04d}"
- try:
- parsed: Any = json.loads(stripped)
- except json.JSONDecodeError:
- match = re.search(r"\{.*\}", stripped, re.DOTALL)
- if not match:
- return None
- try:
- parsed = json.loads(match.group(0))
- except json.JSONDecodeError:
- return None
- if not isinstance(parsed, dict):
- return None
- return cast(Dict[str, Any], parsed)
-
-
-def _coerce_box(value: Any) -> list[int] | None:
- if not isinstance(value, list) or len(value) != 4:
- return None
-
- box: list[int] = []
- for coordinate in cast(list[object], value):
- if not isinstance(coordinate, (int, float, str)):
- return None
- try:
- box.append(int(round(float(coordinate))))
- except (TypeError, ValueError):
- return None
- return box
-
-
-def _crop_box_to_data_url(image: Image.Image, box_2d: list[int]) -> str | None:
- width, height = image.size
- y0, x0, y1, x1 = [max(0, min(1000, value)) for value in box_2d]
-
- left = int(x0 / 1000 * width)
- top = int(y0 / 1000 * height)
- right = int(x1 / 1000 * width)
- bottom = int(y1 / 1000 * height)
-
- if right <= left or bottom <= top:
- return None
-
- cropped = image.crop((left, top, right, bottom))
- output = io.BytesIO()
- cropped.save(output, format="PNG")
- encoded = base64.b64encode(output.getvalue()).decode("ascii")
- return f"data:image/png;base64,{encoded}"
+def _chunk_requests(
+ requests: Sequence[AssetRequest],
+) -> list[list[AssetRequest]]:
+ return [
+ list(requests[start : start + MAX_ASSETS_PER_GEMINI_REQUEST])
+ for start in range(0, len(requests), MAX_ASSETS_PER_GEMINI_REQUEST)
+ ]
-async def _locate_asset_box(
- client: genai.Client,
- source_images: List[SourceImage],
- description: str,
-) -> Dict[str, Any] | None:
- image_list_text = "\n".join(
- f"- image_index {index}" for index in range(1, len(source_images) + 1)
+def _build_detection_prompt(
+ source_images: Sequence[SourceImage],
+ requests: Sequence[AssetRequest],
+) -> str:
+ source_mapping = "\n".join(
+ f"- attached image {attachment_index} = source image {source.image_index} "
+ f"({source.image.width}x{source.image.height} pixels after EXIF normalization)"
+ for attachment_index, source in enumerate(source_images, start=1)
+ )
+ request_json = json.dumps(
+ [
+ {
+ "request_id": request.request_id,
+ "description": request.description,
+ }
+ for request in requests
+ ],
+ ensure_ascii=False,
+ indent=2,
)
- prompt = f"""Find the bounding box for this visual asset:
-{description}
+ return f"""Locate each requested visual asset in the attached source images.
-Images available:
-{image_list_text}
+SOURCE IMAGE MAPPING (image_index must use the 1-based source image number):
+{source_mapping}
-Return only JSON with this exact shape:
-{{"image_index": 1, "box_2d": [y0, x0, y1, x1], "label": "short label"}}
+REQUESTS:
+{request_json}
-Rules:
-- image_index is 1-based and identifies which input image contains the asset.
-- box_2d uses normalized coordinates from 0 to 1000.
-- box_2d order must be [y0, x0, y1, x1].
-- Crop tightly around the requested asset.
-- If the asset is not visible, return {{"image_index": null, "box_2d": null, "label": "not found"}}.
-- Return JSON only. No markdown.
+BOUNDING-BOX RULES:
+- Return exactly one detections record per request, retaining each request_id unchanged. Requests are independent and output order is not used for matching.
+- box_2d is [ymin, xmin, ymax, xmax], relative to the selected source image and normalized to the documented 0-1000 range.
+- Return the smallest axis-aligned box that contains the whole visible requested asset. Include every visible edge/pixel of the asset; do not clip it.
+- Exclude surrounding UI: cards, containers, page backgrounds, padding, borders, nearby text, captions, controls, and unrelated shadows. Include one of those only when the description explicitly makes it part of the target.
+- For a photo or illustration, bound the visible media itself, not its card or caption. For a logo/wordmark, include the complete logo only. For a small icon, include the glyph, not its button/container/badge unless requested.
+- Repeated lookalikes are separate instances. Use the description's appearance, location, nearby context, and source image number to choose the one exact occurrence; do not reuse or merge boxes. Give labels distinguishing duplicate instances.
+- If the described occurrence is absent, not uniquely distinguishable, substantially occluded, or cannot be isolated from unrelated UI, return that request_id with image_index=null, box_2d=null, and label=null. Never guess a box, substitute a lookalike, or return the whole screenshot.
+- Return JSON only through the supplied schema. Never return masks, polygons, code fences, or explanatory text.
"""
+
+def _parse_detection_response(
+ response: types.GenerateContentResponse,
+) -> AssetDetectionBatch:
+ parsed = response.parsed
+ try:
+ if isinstance(parsed, AssetDetectionBatch):
+ return parsed
+ if parsed is not None:
+ return AssetDetectionBatch.model_validate(parsed)
+ if response.text:
+ return AssetDetectionBatch.model_validate_json(response.text)
+ except (ValidationError, TypeError, ValueError):
+ pass
+ return AssetDetectionBatch(detections=[])
+
+
+async def _locate_asset_batch(
+ client: genai.Client,
+ source_images: Sequence[SourceImage],
+ requests: Sequence[AssetRequest],
+ metrics: AssetExtractionMetrics | None = None,
+) -> AssetDetectionBatch:
+ prompt = _build_detection_prompt(source_images, requests)
+ if metrics is not None:
+ metrics.request_count += 1
+ started_at = time.perf_counter()
response = await client.aio.models.generate_content(
model=ASSET_EXTRACTION_GEMINI_MODEL,
contents=[
types.Content(
role="user",
- parts=[source.part for source in source_images] + [types.Part(text=prompt)],
+ # Keep the task prompt after all image parts, as recommended for
+ # image understanding. The prompt maps attachment order back to
+ # the caller's original 1-based source image numbers.
+ parts=[source.part for source in source_images]
+ + [types.Part(text=prompt)],
)
],
config=types.GenerateContentConfig(
- temperature=0,
+ system_instruction=ASSET_DETECTION_SYSTEM_INSTRUCTION,
+ temperature=0.5,
response_mime_type="application/json",
+ response_schema=AssetDetectionBatch,
thinking_config=types.ThinkingConfig(
- thinking_level=cast(Any, "minimal")
+ thinking_level=types.ThinkingLevel.MINIMAL
),
),
)
+ if metrics is not None:
+ metrics.record_response(response, time.perf_counter() - started_at)
+ return _parse_detection_response(response)
- return _extract_json_object(response.text or "")
+
+def _normalize_box(value: object) -> list[float] | None:
+ if not isinstance(value, list) or len(value) != 4:
+ return None
+
+ coordinates: list[float] = []
+ for coordinate in cast(list[object], value):
+ if isinstance(coordinate, bool) or not isinstance(coordinate, (int, float)):
+ return None
+ number = float(coordinate)
+ if not math.isfinite(number):
+ return None
+ coordinates.append(number)
+
+ raw_ymin, raw_xmin, raw_ymax, raw_xmax = coordinates
+ ymin, ymax = sorted((raw_ymin, raw_ymax))
+ xmin, xmax = sorted((raw_xmin, raw_xmax))
+ ymin = max(0.0, min(1000.0, ymin))
+ xmin = max(0.0, min(1000.0, xmin))
+ ymax = max(0.0, min(1000.0, ymax))
+ xmax = max(0.0, min(1000.0, xmax))
+
+ if ymax <= ymin or xmax <= xmin:
+ return None
+ return [ymin, xmin, ymax, xmax]
+
+
+def _crop_box_to_data_url(image: Image.Image, box_2d: list[float]) -> str | None:
+ normalized_box = _normalize_box(box_2d)
+ if normalized_box is None:
+ return None
+
+ width, height = image.size
+ ymin, xmin, ymax, xmax = normalized_box
+ # Outward rounding preserves every edge pixel instead of truncating the
+ # model's bottom/right bounds and accidentally shaving a one-pixel border.
+ left = max(0, min(width, math.floor(xmin / 1000 * width)))
+ top = max(0, min(height, math.floor(ymin / 1000 * height)))
+ right = max(0, min(width, math.ceil(xmax / 1000 * width)))
+ bottom = max(0, min(height, math.ceil(ymax / 1000 * height)))
+
+ if right <= left or bottom <= top:
+ return None
+
+ cropped = image.crop((left, top, right, bottom))
+ if cropped.width <= 0 or cropped.height <= 0:
+ return None
+
+ output = io.BytesIO()
+ cropped.save(output, format="PNG")
+ encoded = base64.b64encode(output.getvalue()).decode("ascii")
+ return f"data:image/png;base64,{encoded}"
async def extract_assets_from_images(
image_data_urls: List[str],
asset_descriptions: List[str],
gemini_api_key: str,
+ *,
+ media_resolution: types.PartMediaResolutionLevel = DEFAULT_ASSET_MEDIA_RESOLUTION,
+ metrics: AssetExtractionMetrics | None = None,
) -> Dict[str, Any]:
- source_images = [
- source
- for image in image_data_urls
- if (source := _data_url_to_source_image(image)) is not None
- ]
+ source_images: list[SourceImage] = []
+ for image_index, data_url in enumerate(image_data_urls, start=1):
+ source = _data_url_to_source_image(
+ data_url,
+ image_index=image_index,
+ media_resolution=media_resolution,
+ )
+ if source is not None:
+ source_images.append(source)
+
if not source_images:
return {
"assets": [],
"error": "No valid input images were available for asset extraction.",
}
- client = genai.Client(api_key=gemini_api_key)
- assets: List[Dict[str, Any]] = []
+ requests = [
+ AssetRequest(
+ request_id=_stable_request_id(original_index),
+ description=description,
+ )
+ for original_index, description in enumerate(asset_descriptions)
+ ]
+ if not requests:
+ return {"assets": []}
+
+ # Per-part media resolution is a Gemini 3 v1alpha feature. Batches are
+ # independent, so requests above the Cookbook's 25-object cap can run in
+ # parallel while still being mapped deterministically by stable IDs.
+ client = genai.Client(
+ api_key=gemini_api_key,
+ http_options=types.HttpOptions(api_version="v1alpha"),
+ )
+ request_chunks = _chunk_requests(requests)
+ batch_results = await asyncio.gather(
+ *(
+ _locate_asset_batch(client, source_images, chunk, metrics)
+ for chunk in request_chunks
+ )
+ )
- for description in asset_descriptions:
- detection = await _locate_asset_box(client, source_images, description)
- image_index = detection.get("image_index") if detection else None
- box = _coerce_box(detection.get("box_2d")) if detection else None
+ detections_by_request_id: dict[str, AssetDetection] = {}
+ for request_chunk, batch in zip(request_chunks, batch_results):
+ expected_ids = {request.request_id for request in request_chunk}
+ for detection in batch.detections:
+ if (
+ detection.request_id in expected_ids
+ and detection.request_id not in detections_by_request_id
+ ):
+ detections_by_request_id[detection.request_id] = detection
+
+ source_by_image_index = {
+ source.image_index: source for source in source_images
+ }
+ assets: List[Dict[str, Any]] = []
+ for request in requests:
+ detection = detections_by_request_id.get(request.request_id)
+ image_index = detection.image_index if detection is not None else None
+ if isinstance(image_index, bool):
+ image_index = None
+ box = _normalize_box(detection.box_2d) if detection is not None else None
+ label = detection.label if detection is not None else None
data_url = None
status = "missing"
- if isinstance(image_index, int) and box:
- source_index = image_index - 1
- if 0 <= source_index < len(source_images):
- data_url = _crop_box_to_data_url(source_images[source_index].image, box)
+ if isinstance(image_index, int) and box is not None:
+ source = source_by_image_index.get(image_index)
+ if source is not None:
+ data_url = _crop_box_to_data_url(source.image, box)
status = "ok" if data_url else "error"
assets.append(
{
- "description": description,
+ "description": request.description,
"data_url": data_url,
"status": status,
"box_2d": box,
"image_index": image_index,
- "label": detection.get("label") if detection else None,
+ "label": label,
}
)
result: Dict[str, Any] = {"assets": assets}
if any(asset["status"] != "ok" for asset in assets):
- result["error"] = "Gemini did not return usable bounding boxes for every requested asset."
+ result["error"] = (
+ "Gemini did not return usable bounding boxes for every requested asset."
+ )
return result
diff --git a/backend/evals/asset_extraction_benchmark.py b/backend/evals/asset_extraction_benchmark.py
new file mode 100644
index 00000000..c0f6ab3e
--- /dev/null
+++ b/backend/evals/asset_extraction_benchmark.py
@@ -0,0 +1,865 @@
+"""Opt-in live benchmark for the Gemini asset-extraction pipeline.
+
+Run from ``backend`` so no API traffic happens accidentally::
+
+ poetry run python -m evals.asset_extraction_benchmark --live
+
+The report compares the pre-batching settings (one request per asset,
+ULTRA_HIGH, temperature 0, free-form JSON) with the new batched structured
+pipeline at HIGH and ULTRA_HIGH. It writes JSON, an HTML summary, crops, and
+box overlays under ``evals_data/asset_extraction_benchmark``.
+"""
+
+import argparse
+import asyncio
+import base64
+import html
+import io
+import json
+import os
+import re
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, cast
+
+from dotenv import load_dotenv
+from google import genai
+from google.genai import types
+from PIL import Image, ImageDraw, ImageFont
+
+from asset_extraction import (
+ ASSET_EXTRACTION_GEMINI_MODEL,
+ AssetExtractionMetrics,
+ SourceImage,
+ _crop_box_to_data_url,
+ _data_url_to_source_image,
+ _normalize_box,
+ extract_assets_from_images,
+)
+
+
+DEFAULT_OUTPUT_ROOT = Path("evals_data/asset_extraction_benchmark")
+DEFAULT_IOU_THRESHOLD = 0.5
+
+
+@dataclass(frozen=True)
+class BenchmarkTarget:
+ target_id: str
+ category: str
+ description: str
+ image_index: int | None
+ expected_box_2d: list[float] | None
+
+
+@dataclass(frozen=True)
+class BenchmarkFixture:
+ image_data_urls: list[str]
+ image_paths: list[Path]
+ image_sizes: list[tuple[int, int]]
+ targets: list[BenchmarkTarget]
+
+
+def _font(size: int, *, bold: bool = False) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:
+ filename = "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf"
+ path = Path("/usr/share/fonts/truetype/dejavu") / filename
+ try:
+ return ImageFont.truetype(str(path), size=size)
+ except OSError:
+ return ImageFont.load_default()
+
+
+def _to_data_url(image: Image.Image) -> str:
+ output = io.BytesIO()
+ image.save(output, format="PNG")
+ encoded = base64.b64encode(output.getvalue()).decode("ascii")
+ return f"data:image/png;base64,{encoded}"
+
+
+def _normalized_box(
+ pixel_box: tuple[int, int, int, int], image_size: tuple[int, int]
+) -> list[float]:
+ left, top, right, bottom = pixel_box
+ width, height = image_size
+ return [
+ top / height * 1000,
+ left / width * 1000,
+ bottom / height * 1000,
+ right / width * 1000,
+ ]
+
+
+def _draw_star(
+ draw: ImageDraw.ImageDraw,
+ center: tuple[int, int],
+ radius: int,
+ fill: str,
+) -> tuple[int, int, int, int]:
+ cx, cy = center
+ points = [
+ (cx, cy - radius),
+ (cx + radius // 3, cy - radius // 3),
+ (cx + radius, cy),
+ (cx + radius // 3, cy + radius // 3),
+ (cx, cy + radius),
+ (cx - radius // 3, cy + radius // 3),
+ (cx - radius, cy),
+ (cx - radius // 3, cy - radius // 3),
+ ]
+ draw.polygon(points, fill=fill)
+ return (cx - radius, cy - radius, cx + radius + 1, cy + radius + 1)
+
+
+def _build_first_screenshot() -> tuple[Image.Image, dict[str, tuple[int, int, int, int]]]:
+ size = (1200, 800)
+ image = Image.new("RGB", size, "#f4f6fb")
+ draw = ImageDraw.Draw(image)
+ draw.rectangle((0, 0, 1200, 124), fill="white")
+ draw.line((0, 123, 1200, 123), fill="#dfe3ed", width=2)
+
+ # Logo/wordmark: a distinct visual target within a much larger navbar.
+ logo_mark = (48, 34, 108, 94)
+ draw.rounded_rectangle(logo_mark, radius=15, fill="#6d4aff")
+ draw.polygon([(63, 64), (78, 47), (94, 64), (78, 81)], fill="white")
+ logo_font = _font(42, bold=True)
+ logo_text_xy = (122, 38)
+ draw.text(logo_text_xy, "LUMA", font=logo_font, fill="#161a2b")
+ text_box = draw.textbbox(logo_text_xy, "LUMA", font=logo_font)
+ logo_box = (
+ logo_mark[0],
+ min(logo_mark[1], text_box[1]),
+ text_box[2],
+ max(logo_mark[3], text_box[3]),
+ )
+
+ # Small magnifier glyph inside a button; expected bounds exclude the button.
+ draw.ellipse((1070, 30, 1142, 102), fill="#f1f3f8", outline="#d8dce7", width=2)
+ draw.ellipse((1091, 49, 1118, 76), outline="#252a3d", width=5)
+ draw.line((1114, 72, 1130, 88), fill="#252a3d", width=5)
+ small_icon_box = (1088, 46, 1134, 92)
+
+ # Illustration inside a surrounding card, with unrelated title/copy beneath.
+ draw.rounded_rectangle((42, 145, 758, 720), radius=28, fill="white")
+ illustration_box = (76, 178, 724, 560)
+ left, top, right, bottom = illustration_box
+ for y in range(top, bottom):
+ ratio = (y - top) / (bottom - top)
+ color = (
+ int(80 + 120 * ratio),
+ int(120 + 80 * ratio),
+ int(235 - 45 * ratio),
+ )
+ draw.line((left, y, right, y), fill=color)
+ draw.ellipse((535, 220, 625, 310), fill="#ffd66b")
+ draw.polygon(
+ [(76, 500), (245, 320), (392, 500), (525, 355), (724, 520), (724, 560), (76, 560)],
+ fill="#273b78",
+ )
+ draw.polygon(
+ [(76, 520), (290, 405), (430, 520), (620, 415), (724, 505), (724, 560), (76, 560)],
+ fill="#5a78bb",
+ )
+ draw.text((76, 594), "Explore beyond the ordinary", font=_font(28, bold=True), fill="#171b2c")
+ draw.text((76, 638), "Curated stories and field notes", font=_font(20), fill="#687086")
+
+ # Two intentionally identical assets, differentiated only by location/context.
+ for card_box, label in [((800, 185, 950, 355), "Primary"), ((980, 185, 1130, 355), "Backup")]:
+ draw.rounded_rectangle(card_box, radius=22, fill="white", outline="#e4e7ef", width=2)
+ draw.text((card_box[0] + 34, 310), label, font=_font(18), fill="#61687b")
+ left_duplicate_box = _draw_star(draw, (875, 255), 34, "#11a8a1")
+ right_duplicate_box = _draw_star(draw, (1055, 255), 34, "#11a8a1")
+
+ return image, {
+ "logo": logo_box,
+ "illustration": illustration_box,
+ "small_icon": small_icon_box,
+ "duplicate_left": left_duplicate_box,
+ "duplicate_right": right_duplicate_box,
+ }
+
+
+def _build_second_screenshot() -> tuple[Image.Image, dict[str, tuple[int, int, int, int]]]:
+ size = (1000, 700)
+ image = Image.new("RGB", size, "#eef1f5")
+ draw = ImageDraw.Draw(image)
+ draw.rectangle((0, 0, 1000, 96), fill="#17223a")
+ draw.text((38, 27), "TRAVEL NOTES", font=_font(28, bold=True), fill="white")
+ draw.rounded_rectangle((72, 120, 928, 650), radius=30, fill="white")
+
+ photo_box = (108, 154, 892, 520)
+ left, top, right, bottom = photo_box
+ for y in range(top, bottom):
+ ratio = (y - top) / (bottom - top)
+ color = (
+ int(245 - 75 * ratio),
+ int(190 - 55 * ratio),
+ int(150 + 65 * ratio),
+ )
+ draw.line((left, y, right, y), fill=color)
+ draw.ellipse((685, 202, 765, 282), fill="#fff0a6")
+ draw.polygon([(108, 470), (290, 270), (470, 470)], fill="#34465c")
+ draw.polygon([(330, 500), (585, 240), (835, 500)], fill="#536b7f")
+ draw.polygon([(108, 475), (300, 380), (510, 500), (720, 365), (892, 485), (892, 520), (108, 520)], fill="#1c6670")
+ draw.text((108, 557), "Dawn over the northern ridge", font=_font(28, bold=True), fill="#17223a")
+ draw.text((108, 600), "Field journal · Day 14", font=_font(19), fill="#6b7383")
+
+ return image, {"second_source_photo": photo_box}
+
+
+def build_fixture(output_dir: Path) -> BenchmarkFixture:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ first, first_boxes = _build_first_screenshot()
+ second, second_boxes = _build_second_screenshot()
+ images = [first, second]
+ paths: list[Path] = []
+ for index, image in enumerate(images, start=1):
+ path = output_dir / f"fixture_source_{index}.png"
+ image.save(path, format="PNG")
+ paths.append(path)
+
+ targets = [
+ BenchmarkTarget(
+ target_id="logo",
+ category="logo",
+ description=(
+ "Complete purple diamond mark plus LUMA wordmark at the far left "
+ "of the white top navigation in screenshot 1; exclude the navbar."
+ ),
+ image_index=1,
+ expected_box_2d=_normalized_box(first_boxes["logo"], first.size),
+ ),
+ BenchmarkTarget(
+ target_id="illustration",
+ category="photo_or_illustration",
+ description=(
+ "Rectangular blue mountain-and-sun illustration in the large left "
+ "card of screenshot 1; exclude its white card and text below."
+ ),
+ image_index=1,
+ expected_box_2d=_normalized_box(first_boxes["illustration"], first.size),
+ ),
+ BenchmarkTarget(
+ target_id="small_icon",
+ category="small_icon",
+ description=(
+ "Small dark magnifying-glass glyph inside the pale circular control "
+ "at the top-right of screenshot 1; extract the glyph only."
+ ),
+ image_index=1,
+ expected_box_2d=_normalized_box(first_boxes["small_icon"], first.size),
+ ),
+ BenchmarkTarget(
+ target_id="duplicate_left",
+ category="duplicate_instance",
+ description=(
+ "Left teal four-point sparkle icon in the card labeled Primary on "
+ "the right side of screenshot 1; icon only, not its card or label."
+ ),
+ image_index=1,
+ expected_box_2d=_normalized_box(first_boxes["duplicate_left"], first.size),
+ ),
+ BenchmarkTarget(
+ target_id="duplicate_right",
+ category="duplicate_instance",
+ description=(
+ "Right teal four-point sparkle icon in the card labeled Backup on "
+ "the right side of screenshot 1; icon only, not its card or label."
+ ),
+ image_index=1,
+ expected_box_2d=_normalized_box(first_boxes["duplicate_right"], first.size),
+ ),
+ BenchmarkTarget(
+ target_id="second_source_photo",
+ category="multiple_source_images",
+ description=(
+ "Wide sunrise mountain photo in the central white article card of "
+ "screenshot 2; exclude the card, header, and caption below."
+ ),
+ image_index=2,
+ expected_box_2d=_normalized_box(
+ second_boxes["second_source_photo"], second.size
+ ),
+ ),
+ BenchmarkTarget(
+ target_id="absent",
+ category="absent_target",
+ description=(
+ "Orange rocket icon next to a Launch button in screenshot 2; this "
+ "exact asset may be absent, so do not substitute another icon."
+ ),
+ image_index=None,
+ expected_box_2d=None,
+ ),
+ ]
+
+ return BenchmarkFixture(
+ image_data_urls=[_to_data_url(image) for image in images],
+ image_paths=paths,
+ image_sizes=[image.size for image in images],
+ targets=targets,
+ )
+
+
+def box_iou(first: list[float], second: list[float]) -> float:
+ first_ymin, first_xmin, first_ymax, first_xmax = first
+ second_ymin, second_xmin, second_ymax, second_xmax = second
+ intersection_width = max(0.0, min(first_xmax, second_xmax) - max(first_xmin, second_xmin))
+ intersection_height = max(0.0, min(first_ymax, second_ymax) - max(first_ymin, second_ymin))
+ intersection = intersection_width * intersection_height
+ first_area = max(0.0, first_xmax - first_xmin) * max(0.0, first_ymax - first_ymin)
+ second_area = max(0.0, second_xmax - second_xmin) * max(0.0, second_ymax - second_ymin)
+ union = first_area + second_area - intersection
+ return intersection / union if union > 0 else 0.0
+
+
+def _legacy_json_object(text: str) -> dict[str, Any] | None:
+ """Reproduce the pre-change free-form/regex parser only for comparison."""
+
+ stripped = text.strip()
+ if stripped.startswith("```"):
+ stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
+ stripped = re.sub(r"\s*```$", "", stripped)
+ try:
+ parsed: Any = json.loads(stripped)
+ except json.JSONDecodeError:
+ match = re.search(r"\{.*\}", stripped, re.DOTALL)
+ if match is None:
+ return None
+ try:
+ parsed = json.loads(match.group(0))
+ except json.JSONDecodeError:
+ return None
+ return cast(dict[str, Any], parsed) if isinstance(parsed, dict) else None
+
+
+def _legacy_prompt(source_images: list[SourceImage], description: str) -> str:
+ image_list_text = "\n".join(
+ f"- image_index {source.image_index}" for source in source_images
+ )
+ return f"""Find the bounding box for this visual asset:
+
+{description}
+
+Images available:
+{image_list_text}
+
+Return only JSON with this exact shape:
+{{"image_index": 1, "box_2d": [y0, x0, y1, x1], "label": "short label"}}
+
+Rules:
+- image_index is 1-based and identifies which input image contains the asset.
+- box_2d uses normalized coordinates from 0 to 1000.
+- box_2d order must be [y0, x0, y1, x1].
+- Crop tightly around the requested asset.
+- If the asset is not visible, return {{"image_index": null, "box_2d": null, "label": "not found"}}.
+- Return JSON only. No markdown.
+"""
+
+
+async def _run_legacy_profile(
+ fixture: BenchmarkFixture,
+ api_key: str,
+) -> tuple[dict[str, Any], AssetExtractionMetrics, float]:
+ source_images = [
+ source
+ for image_index, data_url in enumerate(fixture.image_data_urls, start=1)
+ if (
+ source := _data_url_to_source_image(
+ data_url,
+ image_index=image_index,
+ media_resolution=(
+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH
+ ),
+ )
+ )
+ is not None
+ ]
+ client = genai.Client(
+ api_key=api_key,
+ http_options=types.HttpOptions(api_version="v1alpha"),
+ )
+ metrics = AssetExtractionMetrics()
+ assets: list[dict[str, Any]] = []
+ wall_started = time.perf_counter()
+
+ for target in fixture.targets:
+ prompt = _legacy_prompt(source_images, target.description)
+ metrics.request_count += 1
+ request_started = time.perf_counter()
+ response = await client.aio.models.generate_content(
+ model=ASSET_EXTRACTION_GEMINI_MODEL,
+ contents=[
+ types.Content(
+ role="user",
+ parts=[source.part for source in source_images]
+ + [types.Part(text=prompt)],
+ )
+ ],
+ config=types.GenerateContentConfig(
+ temperature=0,
+ response_mime_type="application/json",
+ thinking_config=types.ThinkingConfig(
+ thinking_level=types.ThinkingLevel.MINIMAL
+ ),
+ ),
+ )
+ metrics.record_response(response, time.perf_counter() - request_started)
+ detection = _legacy_json_object(response.text or "")
+ raw_image_index = detection.get("image_index") if detection else None
+ image_index = (
+ raw_image_index
+ if isinstance(raw_image_index, int) and not isinstance(raw_image_index, bool)
+ else None
+ )
+ box = _normalize_box(detection.get("box_2d")) if detection else None
+ label = detection.get("label") if detection else None
+ data_url = None
+ status = "missing"
+ if image_index is not None and box is not None:
+ source = next(
+ (
+ item
+ for item in source_images
+ if item.image_index == image_index
+ ),
+ None,
+ )
+ if source is not None:
+ data_url = _crop_box_to_data_url(source.image, box)
+ status = "ok" if data_url else "error"
+ assets.append(
+ {
+ "description": target.description,
+ "data_url": data_url,
+ "status": status,
+ "box_2d": box,
+ "image_index": image_index,
+ "label": label,
+ }
+ )
+
+ return {"assets": assets}, metrics, time.perf_counter() - wall_started
+
+
+async def _run_new_profile(
+ fixture: BenchmarkFixture,
+ api_key: str,
+ media_resolution: types.PartMediaResolutionLevel,
+) -> tuple[dict[str, Any], AssetExtractionMetrics, float]:
+ metrics = AssetExtractionMetrics()
+ wall_started = time.perf_counter()
+ result = await extract_assets_from_images(
+ image_data_urls=fixture.image_data_urls,
+ asset_descriptions=[target.description for target in fixture.targets],
+ gemini_api_key=api_key,
+ media_resolution=media_resolution,
+ metrics=metrics,
+ )
+ return result, metrics, time.perf_counter() - wall_started
+
+
+def _decode_crop(data_url: str) -> Image.Image:
+ _, encoded = data_url.split(",", 1)
+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as image:
+ return image.copy()
+
+
+def _evaluate_profile(
+ *,
+ name: str,
+ settings: dict[str, Any],
+ result: dict[str, Any],
+ metrics: AssetExtractionMetrics,
+ latency_seconds: float,
+ fixture: BenchmarkFixture,
+ output_dir: Path,
+ iou_threshold: float,
+) -> dict[str, Any]:
+ raw_assets = result.get("assets")
+ assets = cast(list[dict[str, Any]], raw_assets) if isinstance(raw_assets, list) else []
+ target_results: list[dict[str, Any]] = []
+ present_ious: list[float] = []
+ success_count = 0
+
+ crops_dir = output_dir / "crops"
+ crops_dir.mkdir(exist_ok=True)
+ for index, target in enumerate(fixture.targets):
+ asset = assets[index] if index < len(assets) else {}
+ status = str(asset.get("status") or "missing")
+ predicted_index_raw = asset.get("image_index")
+ predicted_index = (
+ predicted_index_raw if isinstance(predicted_index_raw, int) else None
+ )
+ predicted_box = _normalize_box(asset.get("box_2d"))
+ iou: float | None = None
+ source_match = predicted_index == target.image_index
+
+ if target.expected_box_2d is None:
+ success = status == "missing" and predicted_box is None
+ else:
+ iou = (
+ box_iou(target.expected_box_2d, predicted_box)
+ if predicted_box is not None and source_match
+ else 0.0
+ )
+ present_ious.append(iou)
+ success = status == "ok" and source_match and iou >= iou_threshold
+ success_count += int(success)
+
+ crop_file = None
+ data_url = asset.get("data_url")
+ if isinstance(data_url, str) and data_url:
+ crop_file = f"crops/{name}_{target.target_id}.png"
+ _decode_crop(data_url).save(output_dir / crop_file, format="PNG")
+
+ target_results.append(
+ {
+ "target_id": target.target_id,
+ "category": target.category,
+ "description": target.description,
+ "expected_image_index": target.image_index,
+ "predicted_image_index": predicted_index,
+ "expected_box_2d": target.expected_box_2d,
+ "predicted_box_2d": predicted_box,
+ "status": status,
+ "source_match": source_match,
+ "box_iou": round(iou, 4) if iou is not None else None,
+ "success": success,
+ "crop_file": crop_file,
+ }
+ )
+
+ profile: dict[str, Any] = {
+ "name": name,
+ "settings": settings,
+ "latency_seconds": round(latency_seconds, 3),
+ "request_count": metrics.request_count,
+ "token_usage": {
+ "prompt": metrics.prompt_token_count,
+ "candidate": metrics.candidate_token_count,
+ "thoughts": metrics.thoughts_token_count,
+ "total": metrics.total_token_count,
+ },
+ "success_count": success_count,
+ "target_count": len(fixture.targets),
+ "success_rate": round(success_count / len(fixture.targets), 4),
+ "mean_present_box_iou": round(
+ sum(present_ious) / len(present_ious), 4
+ )
+ if present_ious
+ else None,
+ "targets": target_results,
+ }
+ _write_overlays(profile, fixture, output_dir)
+ return profile
+
+
+def _normalized_to_pixels(
+ box: list[float], image_size: tuple[int, int]
+) -> tuple[int, int, int, int]:
+ ymin, xmin, ymax, xmax = box
+ width, height = image_size
+ return (
+ round(xmin / 1000 * width),
+ round(ymin / 1000 * height),
+ round(xmax / 1000 * width),
+ round(ymax / 1000 * height),
+ )
+
+
+def _write_overlays(
+ profile: dict[str, Any], fixture: BenchmarkFixture, output_dir: Path
+) -> None:
+ targets = cast(list[dict[str, Any]], profile["targets"])
+ for image_index, (source_path, image_size) in enumerate(
+ zip(fixture.image_paths, fixture.image_sizes), start=1
+ ):
+ with Image.open(source_path) as source:
+ overlay = source.convert("RGB")
+ draw = ImageDraw.Draw(overlay)
+ font = _font(15, bold=True)
+ for target in targets:
+ target_id = str(target["target_id"])
+ if target["expected_image_index"] == image_index:
+ expected = target["expected_box_2d"]
+ if isinstance(expected, list):
+ box = _normalized_to_pixels(cast(list[float], expected), image_size)
+ draw.rectangle(box, outline="#16a36a", width=4)
+ draw.text((box[0] + 3, box[1] + 3), f"E {target_id}", font=font, fill="#087d4d")
+ if target["predicted_image_index"] == image_index:
+ predicted = target["predicted_box_2d"]
+ if isinstance(predicted, list):
+ box = _normalized_to_pixels(cast(list[float], predicted), image_size)
+ draw.rectangle(box, outline="#e34b4b", width=3)
+ draw.text((box[0] + 3, max(0, box[1] - 19)), f"P {target_id}", font=font, fill="#b92727")
+ filename = f"{profile['name']}_source_{image_index}_overlay.png"
+ overlay.save(output_dir / filename, format="PNG")
+
+
+def _metric(value: Any, digits: int = 3) -> str:
+ if isinstance(value, float):
+ return f"{value:.{digits}f}"
+ return str(value)
+
+
+def _write_html_report(manifest: dict[str, Any], output_dir: Path) -> Path:
+ profiles = cast(list[dict[str, Any]], manifest["profiles"])
+ metric_cards = "".join(
+ f"""
+ <article class="metric-card">
+ <div class="profile-name">{html.escape(str(profile['name']))}</div>
+ <div class="score">{profile['success_count']}/{profile['target_count']}</div>
+ <div class="score-label">successful targets</div>
+ <dl>
+ <div><dt>Mean box IoU</dt><dd>{_metric(profile['mean_present_box_iou'])}</dd></div>
+ <div><dt>Wall latency</dt><dd>{_metric(profile['latency_seconds'])}s</dd></div>
+ <div><dt>Requests</dt><dd>{profile['request_count']}</dd></div>
+ <div><dt>Total tokens</dt><dd>{profile['token_usage']['total']:,}</dd></div>
+ <div><dt>Prompt tokens</dt><dd>{profile['token_usage']['prompt']:,}</dd></div>
+ </dl>
+ </article>
+ """
+ for profile in profiles
+ )
+
+ sections: list[str] = []
+ for profile in profiles:
+ rows = []
+ for target in cast(list[dict[str, Any]], profile["targets"]):
+ iou = "—" if target["box_iou"] is None else _metric(target["box_iou"], 3)
+ crop = (
+ f'<img class="crop" src="{html.escape(target["crop_file"])}" alt="crop">'
+ if target["crop_file"]
+ else '<span class="missing">none</span>'
+ )
+ outcome = "PASS" if target["success"] else "FAIL"
+ rows.append(
+ f"""
+ <tr>
+ <td><strong>{html.escape(target['target_id'])}</strong><br><span>{html.escape(target['category'])}</span></td>
+ <td>{target['expected_image_index'] or 'absent'} → {target['predicted_image_index'] or 'missing'}</td>
+ <td>{iou}</td>
+ <td><span class="pill {'pass' if target['success'] else 'fail'}">{outcome}</span></td>
+ <td>{crop}</td>
+ </tr>
+ """
+ )
+ overlays = "".join(
+ f"<img src=\"{profile['name']}_source_{index}_overlay.png\" alt=\"{profile['name']} source {index} overlay\">"
+ for index in range(1, 3)
+ )
+ sections.append(
+ f"""
+ <section>
+ <div class="section-heading">
+ <div><p class="eyebrow">Profile</p><h2>{html.escape(profile['name'])}</h2></div>
+ <code>{html.escape(json.dumps(profile['settings'], sort_keys=True))}</code>
+ </div>
+ <div class="overlays">{overlays}</div>
+ <table>
+ <thead><tr><th>Target</th><th>Source</th><th>IoU</th><th>Result</th><th>Crop</th></tr></thead>
+ <tbody>{''.join(rows)}</tbody>
+ </table>
+ </section>
+ """
+ )
+
+ document = f"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Gemini asset extraction benchmark</title>
+<style>
+ :root {{ color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, sans-serif; color: #172033; background: #eef1f7; }}
+ * {{ box-sizing: border-box; }}
+ body {{ margin: 0; }}
+ main {{ width: min(1320px, calc(100% - 48px)); margin: 36px auto 72px; }}
+ header {{ background: linear-gradient(135deg, #171b32, #34246f); color: white; border-radius: 24px; padding: 34px 38px; box-shadow: 0 18px 45px #24265025; }}
+ .eyebrow {{ margin: 0 0 7px; text-transform: uppercase; letter-spacing: .14em; font-size: 12px; font-weight: 800; color: #8f7cff; }}
+ header .eyebrow {{ color: #bdb2ff; }}
+ h1 {{ margin: 0; font-size: 38px; letter-spacing: -.035em; }}
+ header p:last-child {{ margin: 12px 0 0; color: #d6d8e8; max-width: 850px; line-height: 1.55; }}
+ .metrics {{ display: grid; grid-template-columns: repeat(3, 1fr); gap: 18px; margin: 22px 0; }}
+ .metric-card, section {{ background: white; border: 1px solid #dfe3ec; border-radius: 20px; box-shadow: 0 10px 30px #2f3c5910; }}
+ .metric-card {{ padding: 24px; }}
+ .profile-name {{ font: 700 13px ui-monospace, monospace; color: #6756c8; }}
+ .score {{ font-size: 42px; font-weight: 800; letter-spacing: -.04em; margin-top: 12px; }}
+ .score-label {{ color: #71798b; margin-top: -5px; }}
+ dl {{ margin: 20px 0 0; }}
+ dl div {{ display: flex; justify-content: space-between; border-top: 1px solid #eef0f5; padding: 9px 0; }}
+ dt {{ color: #737b8d; }} dd {{ margin: 0; font-weight: 700; }}
+ section {{ padding: 28px; margin-top: 22px; }}
+ .section-heading {{ display: flex; justify-content: space-between; align-items: end; gap: 20px; margin-bottom: 20px; }}
+ h2 {{ margin: 0; font-size: 28px; }}
+ code {{ max-width: 68%; white-space: normal; color: #596175; background: #f4f5f8; padding: 10px 12px; border-radius: 10px; }}
+ .overlays {{ display: grid; grid-template-columns: 1.2fr 1fr; gap: 14px; margin-bottom: 22px; align-items: start; }}
+ .overlays img {{ width: 100%; border-radius: 13px; border: 1px solid #dfe3eb; }}
+ table {{ width: 100%; border-collapse: collapse; }}
+ th, td {{ text-align: left; border-top: 1px solid #e9ebf0; padding: 12px 10px; vertical-align: middle; }}
+ th {{ color: #737b8d; font-size: 12px; text-transform: uppercase; letter-spacing: .08em; }}
+ td span {{ color: #7b8292; font-size: 12px; }}
+ .pill {{ display: inline-block; padding: 5px 8px; border-radius: 7px; font-weight: 800; font-size: 11px; }}
+ .pill.pass {{ color: #087d4d; background: #dff7eb; }} .pill.fail {{ color: #ac2929; background: #fee6e6; }}
+ .crop {{ width: 90px; max-height: 62px; object-fit: contain; background: #eef0f4; border-radius: 8px; border: 1px solid #dde1e9; }}
+ .missing {{ color: #959cab; }}
+ @media (max-width: 900px) {{ .metrics {{ grid-template-columns: 1fr; }} .overlays {{ grid-template-columns: 1fr; }} .section-heading {{ align-items: start; flex-direction: column; }} code {{ max-width: 100%; }} }}
+</style>
+</head>
+<body><main>
+<header>
+ <p class="eyebrow">Live Gemini 3.6 Flash fixture · {html.escape(manifest['generated_at'])}</p>
+ <h1>Asset extraction: old vs. batched + schema</h1>
+ <p>Seven ordered targets across two synthetic screenshots: logo, illustration, small glyph, repeated lookalikes, second-source mapping, and a genuinely absent item. Green boxes are ground truth; red boxes are model predictions. Success requires the right source and IoU ≥ {manifest['iou_threshold']} (or a missing result for the absent target).</p>
+</header>
+<div class="metrics">{metric_cards}</div>
+{''.join(sections)}
+</main></body></html>"""
+ report_path = output_dir / "report.html"
+ report_path.write_text(document)
+ return report_path
+
+
+def _profile_settings(name: str) -> dict[str, Any]:
+ if name == "old_ultra_high":
+ return {
+ "batch_size": 1,
+ "response_schema": False,
+ "temperature": 0,
+ "thinking": "minimal",
+ "media_resolution": "ULTRA_HIGH (2240 image tokens/image)",
+ }
+ resolution = "HIGH" if name == "new_high" else "ULTRA_HIGH"
+ image_tokens = 1120 if resolution == "HIGH" else 2240
+ return {
+ "batch_size": 25,
+ "response_schema": True,
+ "temperature": 0.5,
+ "thinking": "minimal",
+ "media_resolution": f"{resolution} ({image_tokens} image tokens/image)",
+ }
+
+
+async def run_benchmark(
+ *,
+ api_key: str,
+ output_dir: Path,
+ iou_threshold: float = DEFAULT_IOU_THRESHOLD,
+) -> Path:
+ fixture = build_fixture(output_dir)
+ profile_runs: list[
+ tuple[
+ str,
+ tuple[dict[str, Any], AssetExtractionMetrics, float],
+ ]
+ ] = []
+
+ print("running old_ultra_high (one request per target)...", flush=True)
+ profile_runs.append(
+ ("old_ultra_high", await _run_legacy_profile(fixture, api_key))
+ )
+ print("running new_high (one batched structured request)...", flush=True)
+ profile_runs.append(
+ (
+ "new_high",
+ await _run_new_profile(
+ fixture,
+ api_key,
+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_HIGH,
+ ),
+ )
+ )
+ print("running new_ultra_high (resolution sensitivity)...", flush=True)
+ profile_runs.append(
+ (
+ "new_ultra_high",
+ await _run_new_profile(
+ fixture,
+ api_key,
+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,
+ ),
+ )
+ )
+
+ profiles = [
+ _evaluate_profile(
+ name=name,
+ settings=_profile_settings(name),
+ result=run_result[0],
+ metrics=run_result[1],
+ latency_seconds=run_result[2],
+ fixture=fixture,
+ output_dir=output_dir,
+ iou_threshold=iou_threshold,
+ )
+ for name, run_result in profile_runs
+ ]
+ manifest: dict[str, Any] = {
+ "generated_at": datetime.now(timezone.utc).isoformat(),
+ "model": ASSET_EXTRACTION_GEMINI_MODEL,
+ "iou_threshold": iou_threshold,
+ "fixture": {
+ "source_count": len(fixture.image_data_urls),
+ "target_count": len(fixture.targets),
+ "categories": sorted({target.category for target in fixture.targets}),
+ },
+ "profiles": profiles,
+ }
+ (output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))
+ report_path = _write_html_report(manifest, output_dir)
+
+ print("\nprofile\tsuccess\tmean_iou\tlatency_s\trequests\ttotal_tokens")
+ for profile in profiles:
+ print(
+ f"{profile['name']}\t{profile['success_count']}/{profile['target_count']}\t"
+ f"{profile['mean_present_box_iou']}\t{profile['latency_seconds']}\t"
+ f"{profile['request_count']}\t{profile['token_usage']['total']}"
+ )
+ print(f"manifest={output_dir / 'manifest.json'}")
+ print(f"report={report_path}")
+ return report_path
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Run the opt-in live Gemini asset-extraction fixture benchmark."
+ )
+ parser.add_argument(
+ "--live",
+ action="store_true",
+ help="Required acknowledgement that this command makes paid Gemini API calls.",
+ )
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument(
+ "--iou-threshold",
+ type=float,
+ default=DEFAULT_IOU_THRESHOLD,
+ )
+ return parser.parse_args()
+
+
+async def main() -> None:
+ load_dotenv(Path(".env"))
+ args = parse_args()
+ if not args.live:
+ raise SystemExit(
+ "Live API calls are disabled by default. Re-run with --live to opt in."
+ )
+ if not 0 <= args.iou_threshold <= 1:
+ raise SystemExit("--iou-threshold must be between 0 and 1")
+ api_key = os.environ.get("GEMINI_API_KEY")
+ if not api_key:
+ raise SystemExit("Missing GEMINI_API_KEY")
+ output_dir = args.output_dir or (
+ DEFAULT_OUTPUT_ROOT / time.strftime("%Y%m%d_%H%M%S")
+ )
+ await run_benchmark(
+ api_key=api_key,
+ output_dir=output_dir,
+ iou_threshold=args.iou_threshold,
+ )
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/backend/poetry.lock b/backend/poetry.lock
index bc3a3f78..1a3cb431 100644
--- a/backend/poetry.lock
+++ b/backend/poetry.lock
@@ -1665,6 +1665,77 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa
typing = ["typing-extensions ; python_version < \"3.10\""]
xmp = ["defusedxml"]
+[[package]]
+name = "pillow-heif"
+version = "0.18.0"
+description = "Python interface for libheif library"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+ {file = "pillow_heif-0.18.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:e1ad1d97f42fc39de8639b3f45d4d053e00158fc64f1674a14d8912cf81791e3"},
+ {file = "pillow_heif-0.18.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:c45b8d19d8bb1fc61f1f648d042da16d9085506055665a64b56ce8d8ed83c42b"},
+ {file = "pillow_heif-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d524458837bdc5410f66de8e68e864bd179d19a1c205daf7f8c9a07194cc5615"},
+ {file = "pillow_heif-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f032593b3cfc96970efc91860ef6eaa62b1e661418d7f9ec186dff9ac7c9844"},
+ {file = "pillow_heif-0.18.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06649ea6bfac8ca5e7ac898c78c4aad2fd0bc1ce278fa86c503170010902193b"},
+ {file = "pillow_heif-0.18.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8612f4c2e8a3388647c3ce0b7810398cf941aceecd4b2c7790331a53117baf10"},
+ {file = "pillow_heif-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:a7cc374452f5b00cf44171a7bfc08c016b0c0a9f226a99369ffbeb13fd45fa7b"},
+ {file = "pillow_heif-0.18.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:a5d8bfcf8b23b67b8937bcc25fd464f1ca383d3d1d65220463be81ccf6c8185b"},
+ {file = "pillow_heif-0.18.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:2a4c41e76f2da4e046f170cb3716b7aedc466a194509bc0bf1a7c735d5278b8c"},
+ {file = "pillow_heif-0.18.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68781589ee03bf4bd670e55444c25cb0784451b0beacfb0f79d7f56ae497a767"},
+ {file = "pillow_heif-0.18.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8782f9f82c534f4a37ce3c609505f992f340a495da1247951108218a201d0e9"},
+ {file = "pillow_heif-0.18.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2902460d60843e2c379320f1b752a6d4879e3ab0479674ee323d7dee487cccc8"},
+ {file = "pillow_heif-0.18.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c8bebd570446a7b4f7db5ca8eb333dd4591fda13524bc49eee34b3f5cf40741b"},
+ {file = "pillow_heif-0.18.0-cp311-cp311-win_amd64.whl", hash = "sha256:4476bbd7bb7cc1d94c35f0c85786dbe528661bc937422db03fdc865b9ee91d30"},
+ {file = "pillow_heif-0.18.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:c795e7ccceea33e01e49ce536139f94cabb1bf017393666f76c05a9daebae2da"},
+ {file = "pillow_heif-0.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4dd5b3ec09be45c1ef63be31773df90e18ee08e5e950018b0a349924b54a24ac"},
+ {file = "pillow_heif-0.18.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb2eade59c2654c2643a3b637de37c19e75a77c66a3e9a5e0ae26210e4f48aee"},
+ {file = "pillow_heif-0.18.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35b59d599bfdb8454739db6b92f0841ecadbe887babb5ed5abd5299587843eef"},
+ {file = "pillow_heif-0.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:26a0b8b8e899e7bcc876ee61fcadb0f0b849bd6a0d5c20f0e969c77a43b40568"},
+ {file = "pillow_heif-0.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0276a3e0c667677ed0c67f4512cdf2f674065018049307ba4de5cb4648b0a33e"},
+ {file = "pillow_heif-0.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:5916fa31f2015626dd2372d14e24521ea6caed11b25be14faa9b9c67731087ce"},
+ {file = "pillow_heif-0.18.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:ca554f086bc146f1a798adcd77fdecd81564cc0cd74802ee61e3869ab87282f7"},
+ {file = "pillow_heif-0.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:2821d30d22bbb94c2a0fae25eb566421bf22c909958e031d3f0973b482b88515"},
+ {file = "pillow_heif-0.18.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8376309e178f39a2891183cb9662f1c2c87b8614ff13871f077f89edf65ecf48"},
+ {file = "pillow_heif-0.18.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3549e26a65e04e7e986888993b03aae0f848576c2404b5edf12d7db76ef2e72b"},
+ {file = "pillow_heif-0.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0138a08ed90e54c230878c0b8cb92447ad591b7b2e86bfca145029322ba384c7"},
+ {file = "pillow_heif-0.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f552419c8bd754603f1dfbc7f8cdd666118fdd3d063d67974c5bd5a8d7fed9de"},
+ {file = "pillow_heif-0.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:be148b8463ac5d25fdf94d70c69a53712890cd3974ead906c98e7bf35fc96ba6"},
+ {file = "pillow_heif-0.18.0-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:bca173920f16ea8d1c40a970f002be15ac34a5fa99d39403a85472e265db2357"},
+ {file = "pillow_heif-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e8e9860259688700f13baed015632c4ddaf813d26cc856e37ebf0a3f171661"},
+ {file = "pillow_heif-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b9080d96d51158774e3022fc9af19b650863cbb23fac991458cb354b1aa63d"},
+ {file = "pillow_heif-0.18.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5b2ee478e373c0502dc431b22840dd0c551c4ce0e1007ab13f038a868ed375c1"},
+ {file = "pillow_heif-0.18.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:35d2d7199dc34f28aef39cec984c043b1ee30d5c46048566584dc61cf4108c09"},
+ {file = "pillow_heif-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:d399ac5fb499c8feb9770503db25073dfeaccd01238bcb6aaf01354cd83db123"},
+ {file = "pillow_heif-0.18.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:3af89fa7a73143bc49fc18f92b1a6c0fa68ecb56fb56224fb369c2f56729fbb6"},
+ {file = "pillow_heif-0.18.0-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:8b0b9a66e604aef2d0a19a7cb2247c5f9b3352827bb1b00816053ce4982ec8ab"},
+ {file = "pillow_heif-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02af8950d190e3bea28ed2d0ca40798eeae88eaf6e099ee44ec654667f979d97"},
+ {file = "pillow_heif-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:165430447de0f7da259eb07d9487571784912a64c75cd0c52d0d506c114ec7ce"},
+ {file = "pillow_heif-0.18.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7666cbab98246cf9355625e66bf1bb885fdcf8ff4a917f4db04231e80ea692ee"},
+ {file = "pillow_heif-0.18.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:f403aadee232509ee2fdedbfda2dd0bae75098e70a8ddcc010061f92ab962517"},
+ {file = "pillow_heif-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:7ed6cc5ea21f04b15b7604e20592e0ee760ee10fb2da2209b85c94bf0b6f1034"},
+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-macosx_12_0_x86_64.whl", hash = "sha256:744b8a00a817e7139a7e2fd296092689116700dfd63e34941abdc8ae85b3a982"},
+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-macosx_14_0_arm64.whl", hash = "sha256:89b1d63be7e8036ab45f0cd58e27e54113cfd7e852e91606b5cec4fa788a503f"},
+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a8ab9803d79e84e7751cc0930d381112efbf71461ca123a5c2b7abf1050c72"},
+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7649431ea30a9c342888d814f07d2aed9cab06ef5fe5bf065d514eceb2c8d24e"},
+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8ee07e334f99bdf399d207bb19653496c65fcbf006f2cee964047f6d6d57acd9"},
+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-macosx_12_0_x86_64.whl", hash = "sha256:4148a7f17d184c815f428949d6e60582136ef0802a7462c842ee9fe15ca9be16"},
+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-macosx_14_0_arm64.whl", hash = "sha256:5908e8079f62ec8ace9e7c554691a82ece088d0945d980a877e981f208e85193"},
+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28358d5e4e3aeb4af2a60a20187099ba03ab8619bcec8212900657371778da96"},
+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8988a9ce18425aff189913905ce28e61220061c3f222e08213eb473b88a41a20"},
+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7ef23775de70124a02ad9be69af58126ec63a2e2987495355e75cabb265c01cb"},
+ {file = "pillow_heif-0.18.0.tar.gz", hash = "sha256:70318dad9faa76121c6592ac0ab59881ff0dac6ab791a922e70d82c7706cce88"},
+]
+
+[package.dependencies]
+pillow = ">=10.1.0"
+
+[package.extras]
+dev = ["coverage", "defusedxml", "numpy", "opencv-python (==4.10.0.84)", "packaging", "pre-commit", "pylint", "pympler", "pytest"]
+docs = ["sphinx (>=4.4)", "sphinx-issues (>=3.0.1)", "sphinx-rtd-theme (>=1.0)"]
+tests = ["defusedxml", "numpy", "packaging", "pympler", "pytest"]
+tests-min = ["defusedxml", "packaging", "pytest"]
+
[[package]]
name = "platformdirs"
version = "4.3.6"
@@ -2739,4 +2810,4 @@ type = ["pytest-mypy"]
[metadata]
lock-version = "2.1"
python-versions = "^3.10"
-content-hash = "d908fae380038bf6a45010c9a74ec2fd99ea12e2aa2191889127585a73105b39"
+content-hash = "e1084d6aa27fbfdc96410c6eaa15bc9f2e7835798aa784762bbe12d61f01897b"
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 843d9183..a162d9d4 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -25,6 +25,7 @@ pydantic = "^2.10"
google-genai = "^1.16.1"
langfuse = "^3.0.2"
playwright = "^1.61.0"
+pillow-heif = "^0.18.0"
[tool.poetry.group.dev.dependencies]
pytest = "^7.4.3"
diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.py
index 2be3f3e7..40e603a2 100644
--- a/backend/tests/test_agent_tools.py
+++ b/backend/tests/test_agent_tools.py
@@ -60,10 +60,30 @@ def test_canonical_tool_definitions_include_extract_assets() -> None:
assert "extract_assets" in tool_names
extract_assets_tool = next(tool for tool in tools if tool.name == "extract_assets")
assert extract_assets_tool.parameters["required"] == ["asset_descriptions"]
- assert (
- extract_assets_tool.parameters["properties"]["asset_descriptions"]["type"]
- == "array"
- )
+ asset_descriptions = extract_assets_tool.parameters["properties"][
+ "asset_descriptions"
+ ]
+ assert asset_descriptions["type"] == "array"
+
+ # Good caller-side descriptors are essential for separating lookalikes and
+ # mapping assets across multiple screenshots instead of making Gemini guess.
+ item_description = asset_descriptions["items"]["description"].lower()
+ for required_detail in (
+ "distinctive appearance",
+ "precise location",
+ "nearby ui/context",
+ "1-based screenshot number",
+ "repeated lookalikes",
+ "leftmost vs. rightmost",
+ ):
+ assert required_detail in item_description
+
+ tool_description = extract_assets_tool.description.lower()
+ assert "request order" in tool_description
+ assert "permanent, embeddable public_url" in tool_description
+ assert "attached crop preview" in tool_description
+ assert "absent or unisolatable" in tool_description
+ assert "do not call save_assets" in tool_description
def test_canonical_tool_definitions_exclude_extract_assets_when_disabled() -> None:
diff --git a/backend/tests/test_asset_extraction.py b/backend/tests/test_asset_extraction.py
index bf13959b..bd4ae3c6 100644
--- a/backend/tests/test_asset_extraction.py
+++ b/backend/tests/test_asset_extraction.py
@@ -1,24 +1,52 @@
import base64
import io
+import json
+import re
+from collections.abc import Callable
+from typing import Any, cast
import pytest
from google.genai import types
from PIL import Image
-from asset_extraction import extract_assets_from_images
+from asset_extraction import (
+ ASSET_EXTRACTION_GEMINI_MODEL,
+ MAX_ASSETS_PER_GEMINI_REQUEST,
+ AssetDetectionBatch,
+ AssetExtractionMetrics,
+ extract_assets_from_images,
+)
-def _png_data_url(width: int = 10, height: int = 10) -> str:
+def _image_data_url(
+ image: Image.Image,
+ *,
+ image_format: str = "PNG",
+ mime_type: str = "image/png",
+ save_kwargs: dict[str, Any] | None = None,
+) -> str:
+ output = io.BytesIO()
+ image.save(output, format=image_format, **(save_kwargs or {}))
+ encoded = base64.b64encode(output.getvalue()).decode("ascii")
+ return f"data:{mime_type};base64,{encoded}"
+
+
+def _quadrant_png_data_url(width: int = 10, height: int = 10) -> str:
image = Image.new("RGB", (width, height), "white")
- # Put a black square in the top-left quadrant so the crop is inspectable.
for x in range(0, width // 2):
for y in range(0, height // 2):
image.putpixel((x, y), (0, 0, 0))
+ return _image_data_url(image)
- output = io.BytesIO()
- image.save(output, format="PNG")
- encoded = base64.b64encode(output.getvalue()).decode("ascii")
- return f"data:image/png;base64,{encoded}"
+
+def _split_color_png_data_url(
+ left: str = "red", right: str = "blue", width: int = 20, height: int = 10
+) -> str:
+ image = Image.new("RGB", (width, height), right)
+ for x in range(width // 2):
+ for y in range(height):
+ image.putpixel((x, y), Image.new("RGB", (1, 1), left).getpixel((0, 0)))
+ return _image_data_url(image)
def _decode_png_data_url(data_url: str) -> Image.Image:
@@ -28,33 +56,45 @@ def _decode_png_data_url(data_url: str) -> Image.Image:
return image
-async def test_extract_assets_from_images_uses_gemini_boxes_and_crops_original(
+def _response(
+ detections: list[dict[str, Any]],
+ *,
+ prompt_tokens: int | None = None,
+ candidate_tokens: int | None = None,
+ total_tokens: int | None = None,
+ response_id: str | None = None,
+) -> types.GenerateContentResponse:
+ usage = None
+ if any(
+ value is not None
+ for value in (prompt_tokens, candidate_tokens, total_tokens)
+ ):
+ usage = types.GenerateContentResponseUsageMetadata(
+ prompt_token_count=prompt_tokens,
+ candidates_token_count=candidate_tokens,
+ total_token_count=total_tokens,
+ )
+ # The SDK populates `parsed` after transport parsing. Passing a dict to the
+ # response constructor itself is coerced through its broad BaseModel union.
+ response = types.GenerateContentResponse(
+ usage_metadata=usage,
+ response_id=response_id,
+ )
+ response.parsed = {"detections": detections}
+ return response
+
+
+def _install_fake_client(
monkeypatch: pytest.MonkeyPatch,
-) -> None:
- calls: list[dict[str, object]] = []
+ responder: Callable[[dict[str, Any]], types.GenerateContentResponse],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+ calls: list[dict[str, Any]] = []
+ clients: list[dict[str, Any]] = []
class FakeModels:
- async def generate_content(self, **kwargs):
+ async def generate_content(self, **kwargs: Any) -> types.GenerateContentResponse:
calls.append(kwargs)
- return types.GenerateContentResponse(
- candidates=[
- types.Candidate(
- content=types.Content(
- role="model",
- parts=[
- types.Part(
- text=(
- '{"image_index": 1, '
- '"box_2d": [0, 0, 500, 500], '
- '"label": "logo"}'
- )
- )
- ],
- )
- )
- ]
- )
+ return responder(kwargs)
class FakeAio:
models = FakeModels()
@@ -62,22 +102,451 @@ async def test_extract_assets_from_images_uses_gemini_boxes_and_crops_original(
class FakeClient:
aio = FakeAio()
- def __init__(self, api_key: str) -> None:
- assert api_key == "gemini-key"
+ def __init__(self, **kwargs: Any) -> None:
+ clients.append(kwargs)
monkeypatch.setattr("asset_extraction.genai.Client", FakeClient)
+ return calls, clients
+
+
+def _parts_for_call(call: dict[str, Any]) -> list[types.Part]:
+ contents = cast(list[types.Content], call["contents"])
+ return cast(list[types.Part], contents[0].parts)
+
+
+def _prompt_for_call(call: dict[str, Any]) -> str:
+ return _parts_for_call(call)[-1].text or ""
+
+
+async def test_batches_assets_with_schema_config_and_maps_out_of_order_response(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:
+ return _response(
+ [
+ {
+ "request_id": "asset-0002",
+ "image_index": 1,
+ "box_2d": [500, 500, 1000, 1000],
+ "label": "lower-right avatar",
+ },
+ {
+ "request_id": "asset-0001",
+ "image_index": 1,
+ "box_2d": [0, 0, 500, 500],
+ "label": "upper-left logo",
+ },
+ ],
+ prompt_tokens=100,
+ candidate_tokens=20,
+ total_tokens=120,
+ response_id="provider-response-1",
+ )
+
+ calls, clients = _install_fake_client(monkeypatch, responder)
+ metrics = AssetExtractionMetrics()
+ descriptions = [
+ "black square logo in the upper-left of screenshot 1",
+ "white avatar in the lower-right of screenshot 1",
+ ]
result = await extract_assets_from_images(
- image_data_urls=[_png_data_url()],
- asset_descriptions=["logo", "avatar"],
+ image_data_urls=[_quadrant_png_data_url()],
+ asset_descriptions=descriptions,
gemini_api_key="gemini-key",
+ metrics=metrics,
+ )
+
+ # Both descriptions share one multimodal request rather than paying image
+ # tokens and latency once per asset.
+ assert len(calls) == 1
+ assert clients[0]["api_key"] == "gemini-key"
+ assert clients[0]["http_options"].api_version == "v1alpha"
+ assert calls[0]["model"] == ASSET_EXTRACTION_GEMINI_MODEL
+
+ config = cast(types.GenerateContentConfig, calls[0]["config"])
+ assert config.temperature == 0.5
+ assert config.response_mime_type == "application/json"
+ assert config.response_schema is AssetDetectionBatch
+ # Gemini v1alpha rejects the SDK's serialized `additional_properties`
+ # keyword even though standard JSON Schema supports it.
+ assert "additionalProperties" not in json.dumps(
+ AssetDetectionBatch.model_json_schema()
)
+ assert config.thinking_config is not None
+ assert config.thinking_config.thinking_level == types.ThinkingLevel.MINIMAL
+ assert "Never return masks" in str(config.system_instruction)
+
+ parts = _parts_for_call(calls[0])
+ assert len(parts) == 2 # one normalized image + one prompt
+ assert parts[0].inline_data is not None
+ assert parts[0].inline_data.mime_type == "image/png"
+ assert parts[0].media_resolution is not None
+ assert (
+ parts[0].media_resolution.level
+ == types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH
+ )
+
+ prompt = parts[-1].text or ""
+ assert '"request_id": "asset-0001"' in prompt
+ assert '"request_id": "asset-0002"' in prompt
+ assert "[ymin, xmin, ymax, xmax]" in prompt
+ assert "smallest axis-aligned box" in prompt
+ assert "Exclude surrounding UI" in prompt
+ assert "Repeated lookalikes are separate instances" in prompt
+ assert "image_index=null, box_2d=null" in prompt
+ assert "Never return masks" in prompt
+ assert "Never guess a box" in prompt
- assert len(calls) == 2
- assert calls[0]["model"] == "gemini-3.6-flash"
- assets = result["assets"]
- assert assets[0]["description"] == "logo"
+ assets = cast(list[dict[str, Any]], result["assets"])
+ # Response order is intentionally reversed; result order remains caller order.
+ assert [asset["description"] for asset in assets] == descriptions
+ assert [asset["label"] for asset in assets] == [
+ "upper-left logo",
+ "lower-right avatar",
+ ]
assert assets[0]["status"] == "ok"
assert assets[0]["box_2d"] == [0, 0, 500, 500]
- cropped = _decode_png_data_url(assets[0]["data_url"])
- assert cropped.size == (5, 5)
+ assert _decode_png_data_url(cast(str, assets[0]["data_url"])).size == (5, 5)
+ assert metrics.request_count == 1
+ assert metrics.prompt_token_count == 100
+ assert metrics.candidate_token_count == 20
+ assert metrics.total_token_count == 120
+ assert metrics.response_ids == ["provider-response-1"]
+
+
+async def test_chunks_at_cookbook_25_object_cap_with_stable_global_ids(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ids_by_call: list[list[str]] = []
+
+ def responder(call: dict[str, Any]) -> types.GenerateContentResponse:
+ request_ids = re.findall(r'"request_id": "(asset-\d{4})"', _prompt_for_call(call))
+ ids_by_call.append(request_ids)
+ return _response(
+ [
+ {
+ "request_id": request_id,
+ "image_index": None,
+ "box_2d": None,
+ "label": None,
+ }
+ for request_id in reversed(request_ids)
+ ]
+ )
+
+ calls, _clients = _install_fake_client(monkeypatch, responder)
+ descriptions = [f"asset number {index}" for index in range(51)]
+
+ result = await extract_assets_from_images(
+ image_data_urls=[_quadrant_png_data_url()],
+ asset_descriptions=descriptions,
+ gemini_api_key="gemini-key",
+ )
+
+ assert len(calls) == 3
+ assert sorted(len(request_ids) for request_ids in ids_by_call) == [1, 25, 25]
+ assert all(
+ len(request_ids) <= MAX_ASSETS_PER_GEMINI_REQUEST
+ for request_ids in ids_by_call
+ )
+ assert {request_id for ids in ids_by_call for request_id in ids} == {
+ f"asset-{index:04d}" for index in range(1, 52)
+ }
+ assets = cast(list[dict[str, Any]], result["assets"])
+ assert [asset["description"] for asset in assets] == descriptions
+ assert all(asset["status"] == "missing" for asset in assets)
+
+
+async def test_maps_duplicate_instances_missing_items_and_original_source_numbers(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:
+ return _response(
+ [
+ {
+ "request_id": "asset-0004",
+ "image_index": None,
+ "box_2d": None,
+ "label": None,
+ },
+ {
+ "request_id": "asset-0002",
+ "image_index": 1,
+ "box_2d": [0, 500, 1000, 1000],
+ "label": "right duplicate",
+ },
+ {
+ "request_id": "asset-0003",
+ "image_index": 3,
+ "box_2d": [0, 0, 1000, 1000],
+ "label": "green illustration",
+ },
+ {
+ "request_id": "asset-0001",
+ "image_index": 1,
+ "box_2d": [0, 0, 1000, 500],
+ "label": "left duplicate",
+ },
+ ]
+ )
+
+ calls, _clients = _install_fake_client(monkeypatch, responder)
+ invalid_gif = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=="
+ descriptions = [
+ "left red duplicate tile in screenshot 1",
+ "right blue duplicate tile in screenshot 1",
+ "green illustration centered in screenshot 3",
+ "orange rocket icon that is absent from every screenshot",
+ ]
+
+ result = await extract_assets_from_images(
+ image_data_urls=[
+ _split_color_png_data_url(),
+ invalid_gif,
+ _image_data_url(Image.new("RGB", (8, 6), "green")),
+ ],
+ asset_descriptions=descriptions,
+ gemini_api_key="gemini-key",
+ )
+
+ prompt = _prompt_for_call(calls[0])
+ assert "attached image 1 = source image 1" in prompt
+ # Invalid source 2 is skipped without renumbering the caller's screenshot 3.
+ assert "attached image 2 = source image 3" in prompt
+
+ assets = cast(list[dict[str, Any]], result["assets"])
+ assert [asset["description"] for asset in assets] == descriptions
+ assert [asset["image_index"] for asset in assets] == [1, 1, 3, None]
+ assert [asset["status"] for asset in assets] == ["ok", "ok", "ok", "missing"]
+
+ left_crop = _decode_png_data_url(cast(str, assets[0]["data_url"]))
+ right_crop = _decode_png_data_url(cast(str, assets[1]["data_url"]))
+ third_crop = _decode_png_data_url(cast(str, assets[2]["data_url"]))
+ assert left_crop.getpixel((0, 0)) == (255, 0, 0)
+ assert right_crop.getpixel((0, 0)) == (0, 0, 255)
+ assert third_crop.getpixel((0, 0)) == (0, 128, 0)
+ assert assets[3]["data_url"] is None
+
+
+async def test_exif_orientation_is_identical_for_gemini_and_crop_source(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ sent_image_sizes: list[tuple[int, int]] = []
+
+ def responder(call: dict[str, Any]) -> types.GenerateContentResponse:
+ image_part = _parts_for_call(call)[0]
+ assert image_part.inline_data is not None
+ sent_image = Image.open(io.BytesIO(image_part.inline_data.data or b""))
+ sent_image.load()
+ sent_image_sizes.append(sent_image.size)
+ return _response(
+ [
+ {
+ "request_id": "asset-0001",
+ "image_index": 1,
+ "box_2d": [0, 0, 1000, 1000],
+ "label": "rotated photo",
+ }
+ ]
+ )
+
+ _calls, _clients = _install_fake_client(monkeypatch, responder)
+ landscape = Image.new("RGB", (4, 2), "purple")
+ exif = Image.Exif()
+ exif[274] = 6 # rotate 90 degrees clockwise for display
+ oriented_jpeg = _image_data_url(
+ landscape,
+ image_format="JPEG",
+ mime_type="image/jpeg",
+ save_kwargs={"exif": exif, "quality": 100},
+ )
+
+ result = await extract_assets_from_images(
+ image_data_urls=[oriented_jpeg],
+ asset_descriptions=["whole purple photo in screenshot 1"],
+ gemini_api_key="gemini-key",
+ )
+
+ assert sent_image_sizes == [(2, 4)]
+ asset = cast(list[dict[str, Any]], result["assets"])[0]
+ crop = _decode_png_data_url(cast(str, asset["data_url"]))
+ assert crop.size == (2, 4)
+
+
+async def test_normalizes_documented_heif_input_to_shared_png_pixels(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ sent_mime_types: list[str] = []
+
+ def responder(call: dict[str, Any]) -> types.GenerateContentResponse:
+ image_part = _parts_for_call(call)[0]
+ assert image_part.inline_data is not None
+ sent_mime_types.append(image_part.inline_data.mime_type or "")
+ return _response(
+ [
+ {
+ "request_id": "asset-0001",
+ "image_index": 1,
+ "box_2d": [0, 0, 1000, 1000],
+ "label": "orange HEIF image",
+ }
+ ]
+ )
+
+ _calls, _clients = _install_fake_client(monkeypatch, responder)
+ heif_data_url = _image_data_url(
+ Image.new("RGB", (3, 2), "orange"),
+ image_format="HEIF",
+ mime_type="image/heif",
+ )
+
+ result = await extract_assets_from_images(
+ image_data_urls=[heif_data_url],
+ asset_descriptions=["orange image in screenshot 1"],
+ gemini_api_key="gemini-key",
+ )
+
+ assert sent_mime_types == ["image/png"]
+ asset = cast(list[dict[str, Any]], result["assets"])[0]
+ crop = _decode_png_data_url(cast(str, asset["data_url"]))
+ assert crop.size == (3, 2)
+
+
+async def test_semantically_normalizes_or_rejects_boxes_before_crop(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:
+ return _response(
+ [
+ {
+ "request_id": "asset-0001",
+ "image_index": 1,
+ "box_2d": [1200, 1100, -100, -50],
+ "label": "reversed and clamped",
+ },
+ {
+ "request_id": "asset-0002",
+ "image_index": 1,
+ "box_2d": [0, 0, float("nan"), 500],
+ "label": "non-finite",
+ },
+ {
+ "request_id": "asset-0003",
+ "image_index": 1,
+ "box_2d": [-100, -100, -50, -50],
+ "label": "empty after clamp",
+ },
+ {
+ "request_id": "asset-0004",
+ "image_index": 1,
+ "box_2d": [0.1, 0.1, 99.9, 99.9],
+ "label": "fractional one pixel",
+ },
+ {
+ "request_id": "asset-0005",
+ "image_index": 99,
+ "box_2d": [0, 0, 500, 500],
+ "label": "invalid source",
+ },
+ ]
+ )
+
+ _calls, _clients = _install_fake_client(monkeypatch, responder)
+ result = await extract_assets_from_images(
+ image_data_urls=[_quadrant_png_data_url()],
+ asset_descriptions=["reversed", "nan", "empty", "fractional", "bad source"],
+ gemini_api_key="gemini-key",
+ )
+
+ assets = cast(list[dict[str, Any]], result["assets"])
+ assert assets[0]["box_2d"] == [0, 0, 1000, 1000]
+ assert assets[0]["status"] == "ok"
+ assert _decode_png_data_url(cast(str, assets[0]["data_url"])).size == (10, 10)
+
+ assert assets[1]["box_2d"] is None
+ assert assets[1]["status"] == "missing"
+ assert assets[2]["box_2d"] is None
+ assert assets[2]["status"] == "missing"
+
+ # Outward floor/ceil retains a real edge pixel for a small valid box.
+ assert assets[3]["status"] == "ok"
+ assert _decode_png_data_url(cast(str, assets[3]["data_url"])).size == (1, 1)
+
+ assert assets[4]["box_2d"] == [0, 0, 500, 500]
+ assert assets[4]["status"] == "missing"
+ assert assets[4]["data_url"] is None
+ assert "error" in result
+
+
+async def test_does_not_fallback_to_regex_parsing_for_fenced_free_form_json(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:
+ return types.GenerateContentResponse(
+ candidates=[
+ types.Candidate(
+ content=types.Content(
+ role="model",
+ parts=[
+ types.Part(
+ text=(
+ "```json\n"
+ '{"detections":[{"request_id":"asset-0001",'
+ '"image_index":1,"box_2d":[0,0,500,500],'
+ '"label":"logo"}]}\n'
+ "```"
+ )
+ )
+ ],
+ )
+ )
+ ]
+ )
+
+ _calls, _clients = _install_fake_client(monkeypatch, responder)
+ result = await extract_assets_from_images(
+ image_data_urls=[_quadrant_png_data_url()],
+ asset_descriptions=["logo"],
+ gemini_api_key="gemini-key",
+ )
+
+ asset = cast(list[dict[str, Any]], result["assets"])[0]
+ assert asset["status"] == "missing"
+ assert asset["box_2d"] is None
+
+
+async def test_rejects_unsupported_or_undecodable_inputs_before_client_creation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ client_created = False
+
+ class UnexpectedClient:
+ def __init__(self, **_kwargs: Any) -> None:
+ nonlocal client_created
+ client_created = True
+
+ monkeypatch.setattr("asset_extraction.genai.Client", UnexpectedClient)
+
+ result = await extract_assets_from_images(
+ image_data_urls=[
+ "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==",
+ "data:image/png;base64,not-valid-base64",
+ ],
+ asset_descriptions=["logo"],
+ gemini_api_key="gemini-key",
+ )
+
+ assert client_created is False
+ assert result == {
+ "assets": [],
+ "error": "No valid input images were available for asset extraction.",
+ }
diff --git a/backend/tests/test_asset_extraction_benchmark.py b/backend/tests/test_asset_extraction_benchmark.py
new file mode 100644
index 00000000..58dbd5ee
--- /dev/null
+++ b/backend/tests/test_asset_extraction_benchmark.py
@@ -0,0 +1,53 @@
+from pathlib import Path
+
+import pytest
+from PIL import Image
+
+from evals.asset_extraction_benchmark import box_iou, build_fixture
+
+
+def test_live_benchmark_fixture_covers_required_asset_cases(tmp_path: Path) -> None:
+ fixture = build_fixture(tmp_path)
+
+ assert len(fixture.image_data_urls) == 2
+ assert [path.name for path in fixture.image_paths] == [
+ "fixture_source_1.png",
+ "fixture_source_2.png",
+ ]
+ assert all(path.exists() for path in fixture.image_paths)
+ with Image.open(fixture.image_paths[0]) as first:
+ assert first.size == (1200, 800)
+ with Image.open(fixture.image_paths[1]) as second:
+ assert second.size == (1000, 700)
+
+ categories = {target.category for target in fixture.targets}
+ assert categories == {
+ "logo",
+ "photo_or_illustration",
+ "small_icon",
+ "duplicate_instance",
+ "multiple_source_images",
+ "absent_target",
+ }
+ assert sum(
+ target.category == "duplicate_instance" for target in fixture.targets
+ ) == 2
+ second_source = next(
+ target
+ for target in fixture.targets
+ if target.category == "multiple_source_images"
+ )
+ assert second_source.image_index == 2
+ absent = next(
+ target for target in fixture.targets if target.category == "absent_target"
+ )
+ assert absent.image_index is None
+ assert absent.expected_box_2d is None
+
+
+def test_box_iou_reports_exact_partial_and_disjoint_boxes() -> None:
+ assert box_iou([0, 0, 500, 500], [0, 0, 500, 500]) == 1
+ assert box_iou([0, 0, 500, 500], [500, 500, 1000, 1000]) == 0
+ assert box_iou([0, 0, 500, 500], [250, 250, 750, 750]) == pytest.approx(
+ 1 / 7
+ )