Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Improve screenshot asset extraction accuracy and robustness
Flightdeck Agent9fed693c18b3
diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.pyindex d4d8a1da..57060a19 100644--- a/backend/agent/tools/definitions.py+++ b/backend/agent/tools/definitions.py@@ -135,8 +135,13 @@ def _extract_assets_schema() -> Dict[str, Any]:"items": {"type": "string","description": (- "Short text description of one visual asset to extract "- "from the input image."+ "Identify exactly one visual-asset occurrence. Include its "+ "distinctive appearance (colors, shape, content, or visible "+ "wordmark), precise location, nearby UI/context, and the "+ "1-based screenshot number when multiple screenshots are "+ "available. For repeated lookalikes, use a separate item for "+ "each wanted instance and distinguish it (for example, "+ "leftmost vs. rightmost); do not give only a generic category."),},},@@ -231,11 +236,15 @@ def canonical_tool_definitions(CanonicalToolDefinition(name="extract_assets",description=(- "Extract one or more visual assets from the input screenshot or "- "reference images using Gemini. Pass a list of text descriptions "- "for the assets to extract. Returns each asset (in the same order) "- "with a permanent, embeddable public_url. These assets are already "- "saved — do NOT call save_assets on them (save_assets is only for "+ "Extract one or more tightly cropped visual assets from the input "+ "screenshots or reference images using Gemini. Describe exactly "+ "one occurrence per list item with distinctive appearance, precise "+ "location, nearby context, and the 1-based screenshot number; "+ "distinguish repeated lookalikes instead of naming a generic asset. "+ "Returns each asset in request order with a permanent, embeddable "+ "public_url and an attached crop preview; genuinely absent or "+ "unisolatable items are unresolved. These assets are already saved "+ "— do NOT call save_assets on them (save_assets is only for ""user-uploaded images)."),parameters=_extract_assets_schema(),diff --git a/backend/asset_extraction.py b/backend/asset_extraction.pyindex a37f2311..08ea6549 100644--- a/backend/asset_extraction.py+++ b/backend/asset_extraction.py@@ -1,16 +1,75 @@+import asyncioimport base64import ioimport json-import re-from dataclasses import dataclass-from typing import Any, Dict, List, cast+import math+import time+from collections.abc import Callable+from dataclasses import dataclass, field+from typing import Any, Dict, List, Sequence, cast+import pillow_heiffrom google import genaifrom google.genai import types-from PIL import Image+from PIL import Image, ImageOps+from pydantic import BaseModel, Field, ValidationErrorASSET_EXTRACTION_GEMINI_MODEL = "gemini-3.6-flash"+MAX_ASSETS_PER_GEMINI_REQUEST = 25+DEFAULT_ASSET_MEDIA_RESOLUTION = (+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH+)+SUPPORTED_GEMINI_IMAGE_MIME_TYPES = frozenset(+ {+ "image/png",+ "image/jpeg",+ "image/webp",+ "image/heic",+ "image/heif",+ }+)++# Gemini accepts HEIC/HEIF, but Pillow needs its libheif-backed plugin to decode+# those pixels before EXIF normalization and cropping. pillow-heif's public+# function currently lacks a complete type annotation, so narrow it here.+_register_heif_opener = cast(Callable[[], None], pillow_heif.register_heif_opener)+_register_heif_opener()+++class AssetDetection(BaseModel):+ """One schema-constrained answer, correlated by our stable request ID."""++ request_id: str = Field(+ description="The unchanged request_id from the corresponding input request."+ )+ image_index: int | None = Field(+ description=(+ "1-based source image number containing the requested asset, or null when "+ "the exact asset is absent, ambiguous, or cannot be isolated."+ )+ )+ box_2d: list[float] | None = Field(+ description=(+ "Tight [ymin, xmin, ymax, xmax] bounds normalized to 0-1000, or null "+ "when image_index is null."+ ),+ min_length=4,+ max_length=4,+ )+ label: str | None = Field(+ description=(+ "A short label that distinguishes this occurrence from lookalikes, or "+ "null when the asset was not found."+ )+ )+++class AssetDetectionBatch(BaseModel):+ detections: list[AssetDetection] = Field(+ description="Exactly one detection for every requested request_id.",+ max_length=MAX_ASSETS_PER_GEMINI_REQUEST,+ )@dataclass(frozen=True)@@ -18,180 +77,366 @@ class SourceImage:part: types.Partimage: Image.Imagemime_type: str+ image_index: int+++@dataclass(frozen=True)+class AssetRequest:+ request_id: str+ description: str+++def _empty_float_list() -> list[float]:+ return []+++def _empty_string_list() -> list[str]:+ return []+++@dataclass+class AssetExtractionMetrics:+ """Optional request telemetry used by the opt-in live benchmark."""++ request_count: int = 0+ prompt_token_count: int = 0+ candidate_token_count: int = 0+ thoughts_token_count: int = 0+ total_token_count: int = 0+ request_latencies_seconds: list[float] = field(+ default_factory=_empty_float_list+ )+ response_ids: list[str] = field(default_factory=_empty_string_list)++ def record_response(+ self,+ response: types.GenerateContentResponse,+ latency_seconds: float,+ ) -> None:+ self.request_latencies_seconds.append(latency_seconds)+ if response.response_id:+ self.response_ids.append(response.response_id)++ usage = response.usage_metadata+ if usage is None:+ return+ self.prompt_token_count += usage.prompt_token_count or 0+ self.candidate_token_count += usage.candidates_token_count or 0+ self.thoughts_token_count += usage.thoughts_token_count or 0+ self.total_token_count += usage.total_token_count or 0+++ASSET_DETECTION_SYSTEM_INSTRUCTION = """You are a precise 2D object detector for screenshot asset extraction.+Return only one schema-valid JSON object. Never return masks, segmentation, markdown fences, prose, or invented detections. Process no more than 25 requested objects per response.+"""+++def _normalize_image_for_detection(image: Image.Image) -> Image.Image:+ """Apply EXIF orientation once and produce stable PNG-compatible pixels."""++ oriented = ImageOps.exif_transpose(image)+ has_alpha = "A" in oriented.getbands() or "transparency" in oriented.info+ normalized = oriented.convert("RGBA" if has_alpha else "RGB")+ normalized.load()+ return normalized-def _data_url_to_source_image(data_url: str) -> SourceImage | None:+def _data_url_to_source_image(+ data_url: str,+ *,+ image_index: int = 1,+ media_resolution: types.PartMediaResolutionLevel = DEFAULT_ASSET_MEDIA_RESOLUTION,+) -> SourceImage | None:if not data_url.startswith("data:image/") or "," not in data_url:return Noneheader, encoded = data_url.split(",", 1)mime_type = header.removeprefix("data:").split(";", 1)[0].lower()+ if mime_type not in SUPPORTED_GEMINI_IMAGE_MIME_TYPES:+ return None+try:image_bytes = base64.b64decode(encoded, validate=True)- image = Image.open(io.BytesIO(image_bytes))- image.load()+ with Image.open(io.BytesIO(image_bytes)) as opened_image:+ opened_image.seek(0)+ normalized_image = _normalize_image_for_detection(opened_image)except Exception:return None+ # Gemini must see the exact same oriented pixel matrix that we later crop.+ # Canonical PNG also prevents a misleading/unsupported data-URL MIME header+ # from putting model coordinates in a different orientation than Pillow.+ normalized_output = io.BytesIO()+ normalized_image.save(normalized_output, format="PNG")+ normalized_bytes = normalized_output.getvalue()+ normalized_mime_type = "image/png"+return SourceImage(part=types.Part.from_bytes(- data=image_bytes,- mime_type=mime_type,- media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,+ data=normalized_bytes,+ mime_type=normalized_mime_type,+ media_resolution=media_resolution,),- image=image,- mime_type=mime_type,+ image=normalized_image,+ mime_type=normalized_mime_type,+ image_index=image_index,)-def _extract_json_object(text: str) -> Dict[str, Any] | None:- stripped = text.strip()- if stripped.startswith("```"):- stripped = re.sub(r"^```(?:json)?\s*", "", stripped)- stripped = re.sub(r"\s*```$", "", stripped)+def _stable_request_id(original_index: int) -> str:+ return f"asset-{original_index + 1:04d}"- try:- parsed: Any = json.loads(stripped)- except json.JSONDecodeError:- match = re.search(r"\{.*\}", stripped, re.DOTALL)- if not match:- return None- try:- parsed = json.loads(match.group(0))- except json.JSONDecodeError:- return None- if not isinstance(parsed, dict):- return None- return cast(Dict[str, Any], parsed)---def _coerce_box(value: Any) -> list[int] | None:- if not isinstance(value, list) or len(value) != 4:- return None-- box: list[int] = []- for coordinate in cast(list[object], value):- if not isinstance(coordinate, (int, float, str)):- return None- try:- box.append(int(round(float(coordinate))))- except (TypeError, ValueError):- return None- return box---def _crop_box_to_data_url(image: Image.Image, box_2d: list[int]) -> str | None:- width, height = image.size- y0, x0, y1, x1 = [max(0, min(1000, value)) for value in box_2d]-- left = int(x0 / 1000 * width)- top = int(y0 / 1000 * height)- right = int(x1 / 1000 * width)- bottom = int(y1 / 1000 * height)-- if right <= left or bottom <= top:- return None-- cropped = image.crop((left, top, right, bottom))- output = io.BytesIO()- cropped.save(output, format="PNG")- encoded = base64.b64encode(output.getvalue()).decode("ascii")- return f"data:image/png;base64,{encoded}"+def _chunk_requests(+ requests: Sequence[AssetRequest],+) -> list[list[AssetRequest]]:+ return [+ list(requests[start : start + MAX_ASSETS_PER_GEMINI_REQUEST])+ for start in range(0, len(requests), MAX_ASSETS_PER_GEMINI_REQUEST)+ ]-async def _locate_asset_box(- client: genai.Client,- source_images: List[SourceImage],- description: str,-) -> Dict[str, Any] | None:- image_list_text = "\n".join(- f"- image_index {index}" for index in range(1, len(source_images) + 1)+def _build_detection_prompt(+ source_images: Sequence[SourceImage],+ requests: Sequence[AssetRequest],+) -> str:+ source_mapping = "\n".join(+ f"- attached image {attachment_index} = source image {source.image_index} "+ f"({source.image.width}x{source.image.height} pixels after EXIF normalization)"+ for attachment_index, source in enumerate(source_images, start=1)+ )+ request_json = json.dumps(+ [+ {+ "request_id": request.request_id,+ "description": request.description,+ }+ for request in requests+ ],+ ensure_ascii=False,+ indent=2,)- prompt = f"""Find the bounding box for this visual asset:-{description}+ return f"""Locate each requested visual asset in the attached source images.-Images available:-{image_list_text}+SOURCE IMAGE MAPPING (image_index must use the 1-based source image number):+{source_mapping}-Return only JSON with this exact shape:-{{"image_index": 1, "box_2d": [y0, x0, y1, x1], "label": "short label"}}+REQUESTS:+{request_json}-Rules:-- image_index is 1-based and identifies which input image contains the asset.-- box_2d uses normalized coordinates from 0 to 1000.-- box_2d order must be [y0, x0, y1, x1].-- Crop tightly around the requested asset.-- If the asset is not visible, return {{"image_index": null, "box_2d": null, "label": "not found"}}.-- Return JSON only. No markdown.+BOUNDING-BOX RULES:+- Return exactly one detections record per request, retaining each request_id unchanged. Requests are independent and output order is not used for matching.+- box_2d is [ymin, xmin, ymax, xmax], relative to the selected source image and normalized to the documented 0-1000 range.+- Return the smallest axis-aligned box that contains the whole visible requested asset. Include every visible edge/pixel of the asset; do not clip it.+- Exclude surrounding UI: cards, containers, page backgrounds, padding, borders, nearby text, captions, controls, and unrelated shadows. Include one of those only when the description explicitly makes it part of the target.+- For a photo or illustration, bound the visible media itself, not its card or caption. For a logo/wordmark, include the complete logo only. For a small icon, include the glyph, not its button/container/badge unless requested.+- Repeated lookalikes are separate instances. Use the description's appearance, location, nearby context, and source image number to choose the one exact occurrence; do not reuse or merge boxes. Give labels distinguishing duplicate instances.+- If the described occurrence is absent, not uniquely distinguishable, substantially occluded, or cannot be isolated from unrelated UI, return that request_id with image_index=null, box_2d=null, and label=null. Never guess a box, substitute a lookalike, or return the whole screenshot.+- Return JSON only through the supplied schema. Never return masks, polygons, code fences, or explanatory text."""++def _parse_detection_response(+ response: types.GenerateContentResponse,+) -> AssetDetectionBatch:+ parsed = response.parsed+ try:+ if isinstance(parsed, AssetDetectionBatch):+ return parsed+ if parsed is not None:+ return AssetDetectionBatch.model_validate(parsed)+ if response.text:+ return AssetDetectionBatch.model_validate_json(response.text)+ except (ValidationError, TypeError, ValueError):+ pass+ return AssetDetectionBatch(detections=[])+++async def _locate_asset_batch(+ client: genai.Client,+ source_images: Sequence[SourceImage],+ requests: Sequence[AssetRequest],+ metrics: AssetExtractionMetrics | None = None,+) -> AssetDetectionBatch:+ prompt = _build_detection_prompt(source_images, requests)+ if metrics is not None:+ metrics.request_count += 1+ started_at = time.perf_counter()response = await client.aio.models.generate_content(model=ASSET_EXTRACTION_GEMINI_MODEL,contents=[types.Content(role="user",- parts=[source.part for source in source_images] + [types.Part(text=prompt)],+ # Keep the task prompt after all image parts, as recommended for+ # image understanding. The prompt maps attachment order back to+ # the caller's original 1-based source image numbers.+ parts=[source.part for source in source_images]+ + [types.Part(text=prompt)],)],config=types.GenerateContentConfig(- temperature=0,+ system_instruction=ASSET_DETECTION_SYSTEM_INSTRUCTION,+ temperature=0.5,response_mime_type="application/json",+ response_schema=AssetDetectionBatch,thinking_config=types.ThinkingConfig(- thinking_level=cast(Any, "minimal")+ thinking_level=types.ThinkingLevel.MINIMAL),),)+ if metrics is not None:+ metrics.record_response(response, time.perf_counter() - started_at)+ return _parse_detection_response(response)- return _extract_json_object(response.text or "")++def _normalize_box(value: object) -> list[float] | None:+ if not isinstance(value, list) or len(value) != 4:+ return None++ coordinates: list[float] = []+ for coordinate in cast(list[object], value):+ if isinstance(coordinate, bool) or not isinstance(coordinate, (int, float)):+ return None+ number = float(coordinate)+ if not math.isfinite(number):+ return None+ coordinates.append(number)++ raw_ymin, raw_xmin, raw_ymax, raw_xmax = coordinates+ ymin, ymax = sorted((raw_ymin, raw_ymax))+ xmin, xmax = sorted((raw_xmin, raw_xmax))+ ymin = max(0.0, min(1000.0, ymin))+ xmin = max(0.0, min(1000.0, xmin))+ ymax = max(0.0, min(1000.0, ymax))+ xmax = max(0.0, min(1000.0, xmax))++ if ymax <= ymin or xmax <= xmin:+ return None+ return [ymin, xmin, ymax, xmax]+++def _crop_box_to_data_url(image: Image.Image, box_2d: list[float]) -> str | None:+ normalized_box = _normalize_box(box_2d)+ if normalized_box is None:+ return None++ width, height = image.size+ ymin, xmin, ymax, xmax = normalized_box+ # Outward rounding preserves every edge pixel instead of truncating the+ # model's bottom/right bounds and accidentally shaving a one-pixel border.+ left = max(0, min(width, math.floor(xmin / 1000 * width)))+ top = max(0, min(height, math.floor(ymin / 1000 * height)))+ right = max(0, min(width, math.ceil(xmax / 1000 * width)))+ bottom = max(0, min(height, math.ceil(ymax / 1000 * height)))++ if right <= left or bottom <= top:+ return None++ cropped = image.crop((left, top, right, bottom))+ if cropped.width <= 0 or cropped.height <= 0:+ return None++ output = io.BytesIO()+ cropped.save(output, format="PNG")+ encoded = base64.b64encode(output.getvalue()).decode("ascii")+ return f"data:image/png;base64,{encoded}"async def extract_assets_from_images(image_data_urls: List[str],asset_descriptions: List[str],gemini_api_key: str,+ *,+ media_resolution: types.PartMediaResolutionLevel = DEFAULT_ASSET_MEDIA_RESOLUTION,+ metrics: AssetExtractionMetrics | None = None,) -> Dict[str, Any]:- source_images = [- source- for image in image_data_urls- if (source := _data_url_to_source_image(image)) is not None- ]+ source_images: list[SourceImage] = []+ for image_index, data_url in enumerate(image_data_urls, start=1):+ source = _data_url_to_source_image(+ data_url,+ image_index=image_index,+ media_resolution=media_resolution,+ )+ if source is not None:+ source_images.append(source)+if not source_images:return {"assets": [],"error": "No valid input images were available for asset extraction.",}- client = genai.Client(api_key=gemini_api_key)- assets: List[Dict[str, Any]] = []+ requests = [+ AssetRequest(+ request_id=_stable_request_id(original_index),+ description=description,+ )+ for original_index, description in enumerate(asset_descriptions)+ ]+ if not requests:+ return {"assets": []}++ # Per-part media resolution is a Gemini 3 v1alpha feature. Batches are+ # independent, so requests above the Cookbook's 25-object cap can run in+ # parallel while still being mapped deterministically by stable IDs.+ client = genai.Client(+ api_key=gemini_api_key,+ http_options=types.HttpOptions(api_version="v1alpha"),+ )+ request_chunks = _chunk_requests(requests)+ batch_results = await asyncio.gather(+ *(+ _locate_asset_batch(client, source_images, chunk, metrics)+ for chunk in request_chunks+ )+ )- for description in asset_descriptions:- detection = await _locate_asset_box(client, source_images, description)- image_index = detection.get("image_index") if detection else None- box = _coerce_box(detection.get("box_2d")) if detection else None+ detections_by_request_id: dict[str, AssetDetection] = {}+ for request_chunk, batch in zip(request_chunks, batch_results):+ expected_ids = {request.request_id for request in request_chunk}+ for detection in batch.detections:+ if (+ detection.request_id in expected_ids+ and detection.request_id not in detections_by_request_id+ ):+ detections_by_request_id[detection.request_id] = detection++ source_by_image_index = {+ source.image_index: source for source in source_images+ }+ assets: List[Dict[str, Any]] = []+ for request in requests:+ detection = detections_by_request_id.get(request.request_id)+ image_index = detection.image_index if detection is not None else None+ if isinstance(image_index, bool):+ image_index = None+ box = _normalize_box(detection.box_2d) if detection is not None else None+ label = detection.label if detection is not None else Nonedata_url = Nonestatus = "missing"- if isinstance(image_index, int) and box:- source_index = image_index - 1- if 0 <= source_index < len(source_images):- data_url = _crop_box_to_data_url(source_images[source_index].image, box)+ if isinstance(image_index, int) and box is not None:+ source = source_by_image_index.get(image_index)+ if source is not None:+ data_url = _crop_box_to_data_url(source.image, box)status = "ok" if data_url else "error"assets.append({- "description": description,+ "description": request.description,"data_url": data_url,"status": status,"box_2d": box,"image_index": image_index,- "label": detection.get("label") if detection else None,+ "label": label,})result: Dict[str, Any] = {"assets": assets}if any(asset["status"] != "ok" for asset in assets):- result["error"] = "Gemini did not return usable bounding boxes for every requested asset."+ result["error"] = (+ "Gemini did not return usable bounding boxes for every requested asset."+ )return resultdiff --git a/backend/evals/asset_extraction_benchmark.py b/backend/evals/asset_extraction_benchmark.pynew file mode 100644index 00000000..c0f6ab3e--- /dev/null+++ b/backend/evals/asset_extraction_benchmark.py@@ -0,0 +1,865 @@+"""Opt-in live benchmark for the Gemini asset-extraction pipeline.++Run from ``backend`` so no API traffic happens accidentally::++ poetry run python -m evals.asset_extraction_benchmark --live++The report compares the pre-batching settings (one request per asset,+ULTRA_HIGH, temperature 0, free-form JSON) with the new batched structured+pipeline at HIGH and ULTRA_HIGH. It writes JSON, an HTML summary, crops, and+box overlays under ``evals_data/asset_extraction_benchmark``.+"""++import argparse+import asyncio+import base64+import html+import io+import json+import os+import re+import time+from dataclasses import dataclass+from datetime import datetime, timezone+from pathlib import Path+from typing import Any, cast++from dotenv import load_dotenv+from google import genai+from google.genai import types+from PIL import Image, ImageDraw, ImageFont++from asset_extraction import (+ ASSET_EXTRACTION_GEMINI_MODEL,+ AssetExtractionMetrics,+ SourceImage,+ _crop_box_to_data_url,+ _data_url_to_source_image,+ _normalize_box,+ extract_assets_from_images,+)+++DEFAULT_OUTPUT_ROOT = Path("evals_data/asset_extraction_benchmark")+DEFAULT_IOU_THRESHOLD = 0.5+++@dataclass(frozen=True)+class BenchmarkTarget:+ target_id: str+ category: str+ description: str+ image_index: int | None+ expected_box_2d: list[float] | None+++@dataclass(frozen=True)+class BenchmarkFixture:+ image_data_urls: list[str]+ image_paths: list[Path]+ image_sizes: list[tuple[int, int]]+ targets: list[BenchmarkTarget]+++def _font(size: int, *, bold: bool = False) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:+ filename = "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf"+ path = Path("/usr/share/fonts/truetype/dejavu") / filename+ try:+ return ImageFont.truetype(str(path), size=size)+ except OSError:+ return ImageFont.load_default()+++def _to_data_url(image: Image.Image) -> str:+ output = io.BytesIO()+ image.save(output, format="PNG")+ encoded = base64.b64encode(output.getvalue()).decode("ascii")+ return f"data:image/png;base64,{encoded}"+++def _normalized_box(+ pixel_box: tuple[int, int, int, int], image_size: tuple[int, int]+) -> list[float]:+ left, top, right, bottom = pixel_box+ width, height = image_size+ return [+ top / height * 1000,+ left / width * 1000,+ bottom / height * 1000,+ right / width * 1000,+ ]+++def _draw_star(+ draw: ImageDraw.ImageDraw,+ center: tuple[int, int],+ radius: int,+ fill: str,+) -> tuple[int, int, int, int]:+ cx, cy = center+ points = [+ (cx, cy - radius),+ (cx + radius // 3, cy - radius // 3),+ (cx + radius, cy),+ (cx + radius // 3, cy + radius // 3),+ (cx, cy + radius),+ (cx - radius // 3, cy + radius // 3),+ (cx - radius, cy),+ (cx - radius // 3, cy - radius // 3),+ ]+ draw.polygon(points, fill=fill)+ return (cx - radius, cy - radius, cx + radius + 1, cy + radius + 1)+++def _build_first_screenshot() -> tuple[Image.Image, dict[str, tuple[int, int, int, int]]]:+ size = (1200, 800)+ image = Image.new("RGB", size, "#f4f6fb")+ draw = ImageDraw.Draw(image)+ draw.rectangle((0, 0, 1200, 124), fill="white")+ draw.line((0, 123, 1200, 123), fill="#dfe3ed", width=2)++ # Logo/wordmark: a distinct visual target within a much larger navbar.+ logo_mark = (48, 34, 108, 94)+ draw.rounded_rectangle(logo_mark, radius=15, fill="#6d4aff")+ draw.polygon([(63, 64), (78, 47), (94, 64), (78, 81)], fill="white")+ logo_font = _font(42, bold=True)+ logo_text_xy = (122, 38)+ draw.text(logo_text_xy, "LUMA", font=logo_font, fill="#161a2b")+ text_box = draw.textbbox(logo_text_xy, "LUMA", font=logo_font)+ logo_box = (+ logo_mark[0],+ min(logo_mark[1], text_box[1]),+ text_box[2],+ max(logo_mark[3], text_box[3]),+ )++ # Small magnifier glyph inside a button; expected bounds exclude the button.+ draw.ellipse((1070, 30, 1142, 102), fill="#f1f3f8", outline="#d8dce7", width=2)+ draw.ellipse((1091, 49, 1118, 76), outline="#252a3d", width=5)+ draw.line((1114, 72, 1130, 88), fill="#252a3d", width=5)+ small_icon_box = (1088, 46, 1134, 92)++ # Illustration inside a surrounding card, with unrelated title/copy beneath.+ draw.rounded_rectangle((42, 145, 758, 720), radius=28, fill="white")+ illustration_box = (76, 178, 724, 560)+ left, top, right, bottom = illustration_box+ for y in range(top, bottom):+ ratio = (y - top) / (bottom - top)+ color = (+ int(80 + 120 * ratio),+ int(120 + 80 * ratio),+ int(235 - 45 * ratio),+ )+ draw.line((left, y, right, y), fill=color)+ draw.ellipse((535, 220, 625, 310), fill="#ffd66b")+ draw.polygon(+ [(76, 500), (245, 320), (392, 500), (525, 355), (724, 520), (724, 560), (76, 560)],+ fill="#273b78",+ )+ draw.polygon(+ [(76, 520), (290, 405), (430, 520), (620, 415), (724, 505), (724, 560), (76, 560)],+ fill="#5a78bb",+ )+ draw.text((76, 594), "Explore beyond the ordinary", font=_font(28, bold=True), fill="#171b2c")+ draw.text((76, 638), "Curated stories and field notes", font=_font(20), fill="#687086")++ # Two intentionally identical assets, differentiated only by location/context.+ for card_box, label in [((800, 185, 950, 355), "Primary"), ((980, 185, 1130, 355), "Backup")]:+ draw.rounded_rectangle(card_box, radius=22, fill="white", outline="#e4e7ef", width=2)+ draw.text((card_box[0] + 34, 310), label, font=_font(18), fill="#61687b")+ left_duplicate_box = _draw_star(draw, (875, 255), 34, "#11a8a1")+ right_duplicate_box = _draw_star(draw, (1055, 255), 34, "#11a8a1")++ return image, {+ "logo": logo_box,+ "illustration": illustration_box,+ "small_icon": small_icon_box,+ "duplicate_left": left_duplicate_box,+ "duplicate_right": right_duplicate_box,+ }+++def _build_second_screenshot() -> tuple[Image.Image, dict[str, tuple[int, int, int, int]]]:+ size = (1000, 700)+ image = Image.new("RGB", size, "#eef1f5")+ draw = ImageDraw.Draw(image)+ draw.rectangle((0, 0, 1000, 96), fill="#17223a")+ draw.text((38, 27), "TRAVEL NOTES", font=_font(28, bold=True), fill="white")+ draw.rounded_rectangle((72, 120, 928, 650), radius=30, fill="white")++ photo_box = (108, 154, 892, 520)+ left, top, right, bottom = photo_box+ for y in range(top, bottom):+ ratio = (y - top) / (bottom - top)+ color = (+ int(245 - 75 * ratio),+ int(190 - 55 * ratio),+ int(150 + 65 * ratio),+ )+ draw.line((left, y, right, y), fill=color)+ draw.ellipse((685, 202, 765, 282), fill="#fff0a6")+ draw.polygon([(108, 470), (290, 270), (470, 470)], fill="#34465c")+ draw.polygon([(330, 500), (585, 240), (835, 500)], fill="#536b7f")+ draw.polygon([(108, 475), (300, 380), (510, 500), (720, 365), (892, 485), (892, 520), (108, 520)], fill="#1c6670")+ draw.text((108, 557), "Dawn over the northern ridge", font=_font(28, bold=True), fill="#17223a")+ draw.text((108, 600), "Field journal · Day 14", font=_font(19), fill="#6b7383")++ return image, {"second_source_photo": photo_box}+++def build_fixture(output_dir: Path) -> BenchmarkFixture:+ output_dir.mkdir(parents=True, exist_ok=True)+ first, first_boxes = _build_first_screenshot()+ second, second_boxes = _build_second_screenshot()+ images = [first, second]+ paths: list[Path] = []+ for index, image in enumerate(images, start=1):+ path = output_dir / f"fixture_source_{index}.png"+ image.save(path, format="PNG")+ paths.append(path)++ targets = [+ BenchmarkTarget(+ target_id="logo",+ category="logo",+ description=(+ "Complete purple diamond mark plus LUMA wordmark at the far left "+ "of the white top navigation in screenshot 1; exclude the navbar."+ ),+ image_index=1,+ expected_box_2d=_normalized_box(first_boxes["logo"], first.size),+ ),+ BenchmarkTarget(+ target_id="illustration",+ category="photo_or_illustration",+ description=(+ "Rectangular blue mountain-and-sun illustration in the large left "+ "card of screenshot 1; exclude its white card and text below."+ ),+ image_index=1,+ expected_box_2d=_normalized_box(first_boxes["illustration"], first.size),+ ),+ BenchmarkTarget(+ target_id="small_icon",+ category="small_icon",+ description=(+ "Small dark magnifying-glass glyph inside the pale circular control "+ "at the top-right of screenshot 1; extract the glyph only."+ ),+ image_index=1,+ expected_box_2d=_normalized_box(first_boxes["small_icon"], first.size),+ ),+ BenchmarkTarget(+ target_id="duplicate_left",+ category="duplicate_instance",+ description=(+ "Left teal four-point sparkle icon in the card labeled Primary on "+ "the right side of screenshot 1; icon only, not its card or label."+ ),+ image_index=1,+ expected_box_2d=_normalized_box(first_boxes["duplicate_left"], first.size),+ ),+ BenchmarkTarget(+ target_id="duplicate_right",+ category="duplicate_instance",+ description=(+ "Right teal four-point sparkle icon in the card labeled Backup on "+ "the right side of screenshot 1; icon only, not its card or label."+ ),+ image_index=1,+ expected_box_2d=_normalized_box(first_boxes["duplicate_right"], first.size),+ ),+ BenchmarkTarget(+ target_id="second_source_photo",+ category="multiple_source_images",+ description=(+ "Wide sunrise mountain photo in the central white article card of "+ "screenshot 2; exclude the card, header, and caption below."+ ),+ image_index=2,+ expected_box_2d=_normalized_box(+ second_boxes["second_source_photo"], second.size+ ),+ ),+ BenchmarkTarget(+ target_id="absent",+ category="absent_target",+ description=(+ "Orange rocket icon next to a Launch button in screenshot 2; this "+ "exact asset may be absent, so do not substitute another icon."+ ),+ image_index=None,+ expected_box_2d=None,+ ),+ ]++ return BenchmarkFixture(+ image_data_urls=[_to_data_url(image) for image in images],+ image_paths=paths,+ image_sizes=[image.size for image in images],+ targets=targets,+ )+++def box_iou(first: list[float], second: list[float]) -> float:+ first_ymin, first_xmin, first_ymax, first_xmax = first+ second_ymin, second_xmin, second_ymax, second_xmax = second+ intersection_width = max(0.0, min(first_xmax, second_xmax) - max(first_xmin, second_xmin))+ intersection_height = max(0.0, min(first_ymax, second_ymax) - max(first_ymin, second_ymin))+ intersection = intersection_width * intersection_height+ first_area = max(0.0, first_xmax - first_xmin) * max(0.0, first_ymax - first_ymin)+ second_area = max(0.0, second_xmax - second_xmin) * max(0.0, second_ymax - second_ymin)+ union = first_area + second_area - intersection+ return intersection / union if union > 0 else 0.0+++def _legacy_json_object(text: str) -> dict[str, Any] | None:+ """Reproduce the pre-change free-form/regex parser only for comparison."""++ stripped = text.strip()+ if stripped.startswith("```"):+ stripped = re.sub(r"^```(?:json)?\s*", "", stripped)+ stripped = re.sub(r"\s*```$", "", stripped)+ try:+ parsed: Any = json.loads(stripped)+ except json.JSONDecodeError:+ match = re.search(r"\{.*\}", stripped, re.DOTALL)+ if match is None:+ return None+ try:+ parsed = json.loads(match.group(0))+ except json.JSONDecodeError:+ return None+ return cast(dict[str, Any], parsed) if isinstance(parsed, dict) else None+++def _legacy_prompt(source_images: list[SourceImage], description: str) -> str:+ image_list_text = "\n".join(+ f"- image_index {source.image_index}" for source in source_images+ )+ return f"""Find the bounding box for this visual asset:++{description}++Images available:+{image_list_text}++Return only JSON with this exact shape:+{{"image_index": 1, "box_2d": [y0, x0, y1, x1], "label": "short label"}}++Rules:+- image_index is 1-based and identifies which input image contains the asset.+- box_2d uses normalized coordinates from 0 to 1000.+- box_2d order must be [y0, x0, y1, x1].+- Crop tightly around the requested asset.+- If the asset is not visible, return {{"image_index": null, "box_2d": null, "label": "not found"}}.+- Return JSON only. No markdown.+"""+++async def _run_legacy_profile(+ fixture: BenchmarkFixture,+ api_key: str,+) -> tuple[dict[str, Any], AssetExtractionMetrics, float]:+ source_images = [+ source+ for image_index, data_url in enumerate(fixture.image_data_urls, start=1)+ if (+ source := _data_url_to_source_image(+ data_url,+ image_index=image_index,+ media_resolution=(+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH+ ),+ )+ )+ is not None+ ]+ client = genai.Client(+ api_key=api_key,+ http_options=types.HttpOptions(api_version="v1alpha"),+ )+ metrics = AssetExtractionMetrics()+ assets: list[dict[str, Any]] = []+ wall_started = time.perf_counter()++ for target in fixture.targets:+ prompt = _legacy_prompt(source_images, target.description)+ metrics.request_count += 1+ request_started = time.perf_counter()+ response = await client.aio.models.generate_content(+ model=ASSET_EXTRACTION_GEMINI_MODEL,+ contents=[+ types.Content(+ role="user",+ parts=[source.part for source in source_images]+ + [types.Part(text=prompt)],+ )+ ],+ config=types.GenerateContentConfig(+ temperature=0,+ response_mime_type="application/json",+ thinking_config=types.ThinkingConfig(+ thinking_level=types.ThinkingLevel.MINIMAL+ ),+ ),+ )+ metrics.record_response(response, time.perf_counter() - request_started)+ detection = _legacy_json_object(response.text or "")+ raw_image_index = detection.get("image_index") if detection else None+ image_index = (+ raw_image_index+ if isinstance(raw_image_index, int) and not isinstance(raw_image_index, bool)+ else None+ )+ box = _normalize_box(detection.get("box_2d")) if detection else None+ label = detection.get("label") if detection else None+ data_url = None+ status = "missing"+ if image_index is not None and box is not None:+ source = next(+ (+ item+ for item in source_images+ if item.image_index == image_index+ ),+ None,+ )+ if source is not None:+ data_url = _crop_box_to_data_url(source.image, box)+ status = "ok" if data_url else "error"+ assets.append(+ {+ "description": target.description,+ "data_url": data_url,+ "status": status,+ "box_2d": box,+ "image_index": image_index,+ "label": label,+ }+ )++ return {"assets": assets}, metrics, time.perf_counter() - wall_started+++async def _run_new_profile(+ fixture: BenchmarkFixture,+ api_key: str,+ media_resolution: types.PartMediaResolutionLevel,+) -> tuple[dict[str, Any], AssetExtractionMetrics, float]:+ metrics = AssetExtractionMetrics()+ wall_started = time.perf_counter()+ result = await extract_assets_from_images(+ image_data_urls=fixture.image_data_urls,+ asset_descriptions=[target.description for target in fixture.targets],+ gemini_api_key=api_key,+ media_resolution=media_resolution,+ metrics=metrics,+ )+ return result, metrics, time.perf_counter() - wall_started+++def _decode_crop(data_url: str) -> Image.Image:+ _, encoded = data_url.split(",", 1)+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as image:+ return image.copy()+++def _evaluate_profile(+ *,+ name: str,+ settings: dict[str, Any],+ result: dict[str, Any],+ metrics: AssetExtractionMetrics,+ latency_seconds: float,+ fixture: BenchmarkFixture,+ output_dir: Path,+ iou_threshold: float,+) -> dict[str, Any]:+ raw_assets = result.get("assets")+ assets = cast(list[dict[str, Any]], raw_assets) if isinstance(raw_assets, list) else []+ target_results: list[dict[str, Any]] = []+ present_ious: list[float] = []+ success_count = 0++ crops_dir = output_dir / "crops"+ crops_dir.mkdir(exist_ok=True)+ for index, target in enumerate(fixture.targets):+ asset = assets[index] if index < len(assets) else {}+ status = str(asset.get("status") or "missing")+ predicted_index_raw = asset.get("image_index")+ predicted_index = (+ predicted_index_raw if isinstance(predicted_index_raw, int) else None+ )+ predicted_box = _normalize_box(asset.get("box_2d"))+ iou: float | None = None+ source_match = predicted_index == target.image_index++ if target.expected_box_2d is None:+ success = status == "missing" and predicted_box is None+ else:+ iou = (+ box_iou(target.expected_box_2d, predicted_box)+ if predicted_box is not None and source_match+ else 0.0+ )+ present_ious.append(iou)+ success = status == "ok" and source_match and iou >= iou_threshold+ success_count += int(success)++ crop_file = None+ data_url = asset.get("data_url")+ if isinstance(data_url, str) and data_url:+ crop_file = f"crops/{name}_{target.target_id}.png"+ _decode_crop(data_url).save(output_dir / crop_file, format="PNG")++ target_results.append(+ {+ "target_id": target.target_id,+ "category": target.category,+ "description": target.description,+ "expected_image_index": target.image_index,+ "predicted_image_index": predicted_index,+ "expected_box_2d": target.expected_box_2d,+ "predicted_box_2d": predicted_box,+ "status": status,+ "source_match": source_match,+ "box_iou": round(iou, 4) if iou is not None else None,+ "success": success,+ "crop_file": crop_file,+ }+ )++ profile: dict[str, Any] = {+ "name": name,+ "settings": settings,+ "latency_seconds": round(latency_seconds, 3),+ "request_count": metrics.request_count,+ "token_usage": {+ "prompt": metrics.prompt_token_count,+ "candidate": metrics.candidate_token_count,+ "thoughts": metrics.thoughts_token_count,+ "total": metrics.total_token_count,+ },+ "success_count": success_count,+ "target_count": len(fixture.targets),+ "success_rate": round(success_count / len(fixture.targets), 4),+ "mean_present_box_iou": round(+ sum(present_ious) / len(present_ious), 4+ )+ if present_ious+ else None,+ "targets": target_results,+ }+ _write_overlays(profile, fixture, output_dir)+ return profile+++def _normalized_to_pixels(+ box: list[float], image_size: tuple[int, int]+) -> tuple[int, int, int, int]:+ ymin, xmin, ymax, xmax = box+ width, height = image_size+ return (+ round(xmin / 1000 * width),+ round(ymin / 1000 * height),+ round(xmax / 1000 * width),+ round(ymax / 1000 * height),+ )+++def _write_overlays(+ profile: dict[str, Any], fixture: BenchmarkFixture, output_dir: Path+) -> None:+ targets = cast(list[dict[str, Any]], profile["targets"])+ for image_index, (source_path, image_size) in enumerate(+ zip(fixture.image_paths, fixture.image_sizes), start=1+ ):+ with Image.open(source_path) as source:+ overlay = source.convert("RGB")+ draw = ImageDraw.Draw(overlay)+ font = _font(15, bold=True)+ for target in targets:+ target_id = str(target["target_id"])+ if target["expected_image_index"] == image_index:+ expected = target["expected_box_2d"]+ if isinstance(expected, list):+ box = _normalized_to_pixels(cast(list[float], expected), image_size)+ draw.rectangle(box, outline="#16a36a", width=4)+ draw.text((box[0] + 3, box[1] + 3), f"E {target_id}", font=font, fill="#087d4d")+ if target["predicted_image_index"] == image_index:+ predicted = target["predicted_box_2d"]+ if isinstance(predicted, list):+ box = _normalized_to_pixels(cast(list[float], predicted), image_size)+ draw.rectangle(box, outline="#e34b4b", width=3)+ draw.text((box[0] + 3, max(0, box[1] - 19)), f"P {target_id}", font=font, fill="#b92727")+ filename = f"{profile['name']}_source_{image_index}_overlay.png"+ overlay.save(output_dir / filename, format="PNG")+++def _metric(value: Any, digits: int = 3) -> str:+ if isinstance(value, float):+ return f"{value:.{digits}f}"+ return str(value)+++def _write_html_report(manifest: dict[str, Any], output_dir: Path) -> Path:+ profiles = cast(list[dict[str, Any]], manifest["profiles"])+ metric_cards = "".join(+ f"""+ <article class="metric-card">+ <div class="profile-name">{html.escape(str(profile['name']))}</div>+ <div class="score">{profile['success_count']}/{profile['target_count']}</div>+ <div class="score-label">successful targets</div>+ <dl>+ <div><dt>Mean box IoU</dt><dd>{_metric(profile['mean_present_box_iou'])}</dd></div>+ <div><dt>Wall latency</dt><dd>{_metric(profile['latency_seconds'])}s</dd></div>+ <div><dt>Requests</dt><dd>{profile['request_count']}</dd></div>+ <div><dt>Total tokens</dt><dd>{profile['token_usage']['total']:,}</dd></div>+ <div><dt>Prompt tokens</dt><dd>{profile['token_usage']['prompt']:,}</dd></div>+ </dl>+ </article>+ """+ for profile in profiles+ )++ sections: list[str] = []+ for profile in profiles:+ rows = []+ for target in cast(list[dict[str, Any]], profile["targets"]):+ iou = "—" if target["box_iou"] is None else _metric(target["box_iou"], 3)+ crop = (+ f'<img class="crop" src="{html.escape(target["crop_file"])}" alt="crop">'+ if target["crop_file"]+ else '<span class="missing">none</span>'+ )+ outcome = "PASS" if target["success"] else "FAIL"+ rows.append(+ f"""+ <tr>+ <td><strong>{html.escape(target['target_id'])}</strong><br><span>{html.escape(target['category'])}</span></td>+ <td>{target['expected_image_index'] or 'absent'} → {target['predicted_image_index'] or 'missing'}</td>+ <td>{iou}</td>+ <td><span class="pill {'pass' if target['success'] else 'fail'}">{outcome}</span></td>+ <td>{crop}</td>+ </tr>+ """+ )+ overlays = "".join(+ f"<img src=\"{profile['name']}_source_{index}_overlay.png\" alt=\"{profile['name']} source {index} overlay\">"+ for index in range(1, 3)+ )+ sections.append(+ f"""+ <section>+ <div class="section-heading">+ <div><p class="eyebrow">Profile</p><h2>{html.escape(profile['name'])}</h2></div>+ <code>{html.escape(json.dumps(profile['settings'], sort_keys=True))}</code>+ </div>+ <div class="overlays">{overlays}</div>+ <table>+ <thead><tr><th>Target</th><th>Source</th><th>IoU</th><th>Result</th><th>Crop</th></tr></thead>+ <tbody>{''.join(rows)}</tbody>+ </table>+ </section>+ """+ )++ document = f"""<!doctype html>+<html lang="en">+<head>+<meta charset="utf-8">+<meta name="viewport" content="width=device-width, initial-scale=1">+<title>Gemini asset extraction benchmark</title>+<style>+ :root {{ color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, sans-serif; color: #172033; background: #eef1f7; }}+ * {{ box-sizing: border-box; }}+ body {{ margin: 0; }}+ main {{ width: min(1320px, calc(100% - 48px)); margin: 36px auto 72px; }}+ header {{ background: linear-gradient(135deg, #171b32, #34246f); color: white; border-radius: 24px; padding: 34px 38px; box-shadow: 0 18px 45px #24265025; }}+ .eyebrow {{ margin: 0 0 7px; text-transform: uppercase; letter-spacing: .14em; font-size: 12px; font-weight: 800; color: #8f7cff; }}+ header .eyebrow {{ color: #bdb2ff; }}+ h1 {{ margin: 0; font-size: 38px; letter-spacing: -.035em; }}+ header p:last-child {{ margin: 12px 0 0; color: #d6d8e8; max-width: 850px; line-height: 1.55; }}+ .metrics {{ display: grid; grid-template-columns: repeat(3, 1fr); gap: 18px; margin: 22px 0; }}+ .metric-card, section {{ background: white; border: 1px solid #dfe3ec; border-radius: 20px; box-shadow: 0 10px 30px #2f3c5910; }}+ .metric-card {{ padding: 24px; }}+ .profile-name {{ font: 700 13px ui-monospace, monospace; color: #6756c8; }}+ .score {{ font-size: 42px; font-weight: 800; letter-spacing: -.04em; margin-top: 12px; }}+ .score-label {{ color: #71798b; margin-top: -5px; }}+ dl {{ margin: 20px 0 0; }}+ dl div {{ display: flex; justify-content: space-between; border-top: 1px solid #eef0f5; padding: 9px 0; }}+ dt {{ color: #737b8d; }} dd {{ margin: 0; font-weight: 700; }}+ section {{ padding: 28px; margin-top: 22px; }}+ .section-heading {{ display: flex; justify-content: space-between; align-items: end; gap: 20px; margin-bottom: 20px; }}+ h2 {{ margin: 0; font-size: 28px; }}+ code {{ max-width: 68%; white-space: normal; color: #596175; background: #f4f5f8; padding: 10px 12px; border-radius: 10px; }}+ .overlays {{ display: grid; grid-template-columns: 1.2fr 1fr; gap: 14px; margin-bottom: 22px; align-items: start; }}+ .overlays img {{ width: 100%; border-radius: 13px; border: 1px solid #dfe3eb; }}+ table {{ width: 100%; border-collapse: collapse; }}+ th, td {{ text-align: left; border-top: 1px solid #e9ebf0; padding: 12px 10px; vertical-align: middle; }}+ th {{ color: #737b8d; font-size: 12px; text-transform: uppercase; letter-spacing: .08em; }}+ td span {{ color: #7b8292; font-size: 12px; }}+ .pill {{ display: inline-block; padding: 5px 8px; border-radius: 7px; font-weight: 800; font-size: 11px; }}+ .pill.pass {{ color: #087d4d; background: #dff7eb; }} .pill.fail {{ color: #ac2929; background: #fee6e6; }}+ .crop {{ width: 90px; max-height: 62px; object-fit: contain; background: #eef0f4; border-radius: 8px; border: 1px solid #dde1e9; }}+ .missing {{ color: #959cab; }}+ @media (max-width: 900px) {{ .metrics {{ grid-template-columns: 1fr; }} .overlays {{ grid-template-columns: 1fr; }} .section-heading {{ align-items: start; flex-direction: column; }} code {{ max-width: 100%; }} }}+</style>+</head>+<body><main>+<header>+ <p class="eyebrow">Live Gemini 3.6 Flash fixture · {html.escape(manifest['generated_at'])}</p>+ <h1>Asset extraction: old vs. batched + schema</h1>+ <p>Seven ordered targets across two synthetic screenshots: logo, illustration, small glyph, repeated lookalikes, second-source mapping, and a genuinely absent item. Green boxes are ground truth; red boxes are model predictions. Success requires the right source and IoU ≥ {manifest['iou_threshold']} (or a missing result for the absent target).</p>+</header>+<div class="metrics">{metric_cards}</div>+{''.join(sections)}+</main></body></html>"""+ report_path = output_dir / "report.html"+ report_path.write_text(document)+ return report_path+++def _profile_settings(name: str) -> dict[str, Any]:+ if name == "old_ultra_high":+ return {+ "batch_size": 1,+ "response_schema": False,+ "temperature": 0,+ "thinking": "minimal",+ "media_resolution": "ULTRA_HIGH (2240 image tokens/image)",+ }+ resolution = "HIGH" if name == "new_high" else "ULTRA_HIGH"+ image_tokens = 1120 if resolution == "HIGH" else 2240+ return {+ "batch_size": 25,+ "response_schema": True,+ "temperature": 0.5,+ "thinking": "minimal",+ "media_resolution": f"{resolution} ({image_tokens} image tokens/image)",+ }+++async def run_benchmark(+ *,+ api_key: str,+ output_dir: Path,+ iou_threshold: float = DEFAULT_IOU_THRESHOLD,+) -> Path:+ fixture = build_fixture(output_dir)+ profile_runs: list[+ tuple[+ str,+ tuple[dict[str, Any], AssetExtractionMetrics, float],+ ]+ ] = []++ print("running old_ultra_high (one request per target)...", flush=True)+ profile_runs.append(+ ("old_ultra_high", await _run_legacy_profile(fixture, api_key))+ )+ print("running new_high (one batched structured request)...", flush=True)+ profile_runs.append(+ (+ "new_high",+ await _run_new_profile(+ fixture,+ api_key,+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_HIGH,+ ),+ )+ )+ print("running new_ultra_high (resolution sensitivity)...", flush=True)+ profile_runs.append(+ (+ "new_ultra_high",+ await _run_new_profile(+ fixture,+ api_key,+ types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,+ ),+ )+ )++ profiles = [+ _evaluate_profile(+ name=name,+ settings=_profile_settings(name),+ result=run_result[0],+ metrics=run_result[1],+ latency_seconds=run_result[2],+ fixture=fixture,+ output_dir=output_dir,+ iou_threshold=iou_threshold,+ )+ for name, run_result in profile_runs+ ]+ manifest: dict[str, Any] = {+ "generated_at": datetime.now(timezone.utc).isoformat(),+ "model": ASSET_EXTRACTION_GEMINI_MODEL,+ "iou_threshold": iou_threshold,+ "fixture": {+ "source_count": len(fixture.image_data_urls),+ "target_count": len(fixture.targets),+ "categories": sorted({target.category for target in fixture.targets}),+ },+ "profiles": profiles,+ }+ (output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))+ report_path = _write_html_report(manifest, output_dir)++ print("\nprofile\tsuccess\tmean_iou\tlatency_s\trequests\ttotal_tokens")+ for profile in profiles:+ print(+ f"{profile['name']}\t{profile['success_count']}/{profile['target_count']}\t"+ f"{profile['mean_present_box_iou']}\t{profile['latency_seconds']}\t"+ f"{profile['request_count']}\t{profile['token_usage']['total']}"+ )+ print(f"manifest={output_dir / 'manifest.json'}")+ print(f"report={report_path}")+ return report_path+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser(+ description="Run the opt-in live Gemini asset-extraction fixture benchmark."+ )+ parser.add_argument(+ "--live",+ action="store_true",+ help="Required acknowledgement that this command makes paid Gemini API calls.",+ )+ parser.add_argument("--output-dir", type=Path)+ parser.add_argument(+ "--iou-threshold",+ type=float,+ default=DEFAULT_IOU_THRESHOLD,+ )+ return parser.parse_args()+++async def main() -> None:+ load_dotenv(Path(".env"))+ args = parse_args()+ if not args.live:+ raise SystemExit(+ "Live API calls are disabled by default. Re-run with --live to opt in."+ )+ if not 0 <= args.iou_threshold <= 1:+ raise SystemExit("--iou-threshold must be between 0 and 1")+ api_key = os.environ.get("GEMINI_API_KEY")+ if not api_key:+ raise SystemExit("Missing GEMINI_API_KEY")+ output_dir = args.output_dir or (+ DEFAULT_OUTPUT_ROOT / time.strftime("%Y%m%d_%H%M%S")+ )+ await run_benchmark(+ api_key=api_key,+ output_dir=output_dir,+ iou_threshold=args.iou_threshold,+ )+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/backend/poetry.lock b/backend/poetry.lockindex bc3a3f78..1a3cb431 100644--- a/backend/poetry.lock+++ b/backend/poetry.lock@@ -1665,6 +1665,77 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "patyping = ["typing-extensions ; python_version < \"3.10\""]xmp = ["defusedxml"]+[[package]]+name = "pillow-heif"+version = "0.18.0"+description = "Python interface for libheif library"+optional = false+python-versions = ">=3.8"+groups = ["main"]+files = [+ {file = "pillow_heif-0.18.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:e1ad1d97f42fc39de8639b3f45d4d053e00158fc64f1674a14d8912cf81791e3"},+ {file = "pillow_heif-0.18.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:c45b8d19d8bb1fc61f1f648d042da16d9085506055665a64b56ce8d8ed83c42b"},+ {file = "pillow_heif-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d524458837bdc5410f66de8e68e864bd179d19a1c205daf7f8c9a07194cc5615"},+ {file = "pillow_heif-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f032593b3cfc96970efc91860ef6eaa62b1e661418d7f9ec186dff9ac7c9844"},+ {file = "pillow_heif-0.18.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06649ea6bfac8ca5e7ac898c78c4aad2fd0bc1ce278fa86c503170010902193b"},+ {file = "pillow_heif-0.18.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8612f4c2e8a3388647c3ce0b7810398cf941aceecd4b2c7790331a53117baf10"},+ {file = "pillow_heif-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:a7cc374452f5b00cf44171a7bfc08c016b0c0a9f226a99369ffbeb13fd45fa7b"},+ {file = "pillow_heif-0.18.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:a5d8bfcf8b23b67b8937bcc25fd464f1ca383d3d1d65220463be81ccf6c8185b"},+ {file = "pillow_heif-0.18.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:2a4c41e76f2da4e046f170cb3716b7aedc466a194509bc0bf1a7c735d5278b8c"},+ {file = "pillow_heif-0.18.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68781589ee03bf4bd670e55444c25cb0784451b0beacfb0f79d7f56ae497a767"},+ {file = "pillow_heif-0.18.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8782f9f82c534f4a37ce3c609505f992f340a495da1247951108218a201d0e9"},+ {file = "pillow_heif-0.18.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2902460d60843e2c379320f1b752a6d4879e3ab0479674ee323d7dee487cccc8"},+ {file = "pillow_heif-0.18.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c8bebd570446a7b4f7db5ca8eb333dd4591fda13524bc49eee34b3f5cf40741b"},+ {file = "pillow_heif-0.18.0-cp311-cp311-win_amd64.whl", hash = "sha256:4476bbd7bb7cc1d94c35f0c85786dbe528661bc937422db03fdc865b9ee91d30"},+ {file = "pillow_heif-0.18.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:c795e7ccceea33e01e49ce536139f94cabb1bf017393666f76c05a9daebae2da"},+ {file = "pillow_heif-0.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4dd5b3ec09be45c1ef63be31773df90e18ee08e5e950018b0a349924b54a24ac"},+ {file = "pillow_heif-0.18.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb2eade59c2654c2643a3b637de37c19e75a77c66a3e9a5e0ae26210e4f48aee"},+ {file = "pillow_heif-0.18.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35b59d599bfdb8454739db6b92f0841ecadbe887babb5ed5abd5299587843eef"},+ {file = "pillow_heif-0.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:26a0b8b8e899e7bcc876ee61fcadb0f0b849bd6a0d5c20f0e969c77a43b40568"},+ {file = "pillow_heif-0.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0276a3e0c667677ed0c67f4512cdf2f674065018049307ba4de5cb4648b0a33e"},+ {file = "pillow_heif-0.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:5916fa31f2015626dd2372d14e24521ea6caed11b25be14faa9b9c67731087ce"},+ {file = "pillow_heif-0.18.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:ca554f086bc146f1a798adcd77fdecd81564cc0cd74802ee61e3869ab87282f7"},+ {file = "pillow_heif-0.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:2821d30d22bbb94c2a0fae25eb566421bf22c909958e031d3f0973b482b88515"},+ {file = "pillow_heif-0.18.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8376309e178f39a2891183cb9662f1c2c87b8614ff13871f077f89edf65ecf48"},+ {file = "pillow_heif-0.18.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3549e26a65e04e7e986888993b03aae0f848576c2404b5edf12d7db76ef2e72b"},+ {file = "pillow_heif-0.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0138a08ed90e54c230878c0b8cb92447ad591b7b2e86bfca145029322ba384c7"},+ {file = "pillow_heif-0.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f552419c8bd754603f1dfbc7f8cdd666118fdd3d063d67974c5bd5a8d7fed9de"},+ {file = "pillow_heif-0.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:be148b8463ac5d25fdf94d70c69a53712890cd3974ead906c98e7bf35fc96ba6"},+ {file = "pillow_heif-0.18.0-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:bca173920f16ea8d1c40a970f002be15ac34a5fa99d39403a85472e265db2357"},+ {file = "pillow_heif-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e8e9860259688700f13baed015632c4ddaf813d26cc856e37ebf0a3f171661"},+ {file = "pillow_heif-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b9080d96d51158774e3022fc9af19b650863cbb23fac991458cb354b1aa63d"},+ {file = "pillow_heif-0.18.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5b2ee478e373c0502dc431b22840dd0c551c4ce0e1007ab13f038a868ed375c1"},+ {file = "pillow_heif-0.18.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:35d2d7199dc34f28aef39cec984c043b1ee30d5c46048566584dc61cf4108c09"},+ {file = "pillow_heif-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:d399ac5fb499c8feb9770503db25073dfeaccd01238bcb6aaf01354cd83db123"},+ {file = "pillow_heif-0.18.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:3af89fa7a73143bc49fc18f92b1a6c0fa68ecb56fb56224fb369c2f56729fbb6"},+ {file = "pillow_heif-0.18.0-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:8b0b9a66e604aef2d0a19a7cb2247c5f9b3352827bb1b00816053ce4982ec8ab"},+ {file = "pillow_heif-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02af8950d190e3bea28ed2d0ca40798eeae88eaf6e099ee44ec654667f979d97"},+ {file = "pillow_heif-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:165430447de0f7da259eb07d9487571784912a64c75cd0c52d0d506c114ec7ce"},+ {file = "pillow_heif-0.18.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7666cbab98246cf9355625e66bf1bb885fdcf8ff4a917f4db04231e80ea692ee"},+ {file = "pillow_heif-0.18.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:f403aadee232509ee2fdedbfda2dd0bae75098e70a8ddcc010061f92ab962517"},+ {file = "pillow_heif-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:7ed6cc5ea21f04b15b7604e20592e0ee760ee10fb2da2209b85c94bf0b6f1034"},+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-macosx_12_0_x86_64.whl", hash = "sha256:744b8a00a817e7139a7e2fd296092689116700dfd63e34941abdc8ae85b3a982"},+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-macosx_14_0_arm64.whl", hash = "sha256:89b1d63be7e8036ab45f0cd58e27e54113cfd7e852e91606b5cec4fa788a503f"},+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a8ab9803d79e84e7751cc0930d381112efbf71461ca123a5c2b7abf1050c72"},+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7649431ea30a9c342888d814f07d2aed9cab06ef5fe5bf065d514eceb2c8d24e"},+ {file = "pillow_heif-0.18.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8ee07e334f99bdf399d207bb19653496c65fcbf006f2cee964047f6d6d57acd9"},+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-macosx_12_0_x86_64.whl", hash = "sha256:4148a7f17d184c815f428949d6e60582136ef0802a7462c842ee9fe15ca9be16"},+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-macosx_14_0_arm64.whl", hash = "sha256:5908e8079f62ec8ace9e7c554691a82ece088d0945d980a877e981f208e85193"},+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28358d5e4e3aeb4af2a60a20187099ba03ab8619bcec8212900657371778da96"},+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8988a9ce18425aff189913905ce28e61220061c3f222e08213eb473b88a41a20"},+ {file = "pillow_heif-0.18.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7ef23775de70124a02ad9be69af58126ec63a2e2987495355e75cabb265c01cb"},+ {file = "pillow_heif-0.18.0.tar.gz", hash = "sha256:70318dad9faa76121c6592ac0ab59881ff0dac6ab791a922e70d82c7706cce88"},+]++[package.dependencies]+pillow = ">=10.1.0"++[package.extras]+dev = ["coverage", "defusedxml", "numpy", "opencv-python (==4.10.0.84)", "packaging", "pre-commit", "pylint", "pympler", "pytest"]+docs = ["sphinx (>=4.4)", "sphinx-issues (>=3.0.1)", "sphinx-rtd-theme (>=1.0)"]+tests = ["defusedxml", "numpy", "packaging", "pympler", "pytest"]+tests-min = ["defusedxml", "packaging", "pytest"]+[[package]]name = "platformdirs"version = "4.3.6"@@ -2739,4 +2810,4 @@ type = ["pytest-mypy"][metadata]lock-version = "2.1"python-versions = "^3.10"-content-hash = "d908fae380038bf6a45010c9a74ec2fd99ea12e2aa2191889127585a73105b39"+content-hash = "e1084d6aa27fbfdc96410c6eaa15bc9f2e7835798aa784762bbe12d61f01897b"diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 843d9183..a162d9d4 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -25,6 +25,7 @@ pydantic = "^2.10"google-genai = "^1.16.1"langfuse = "^3.0.2"playwright = "^1.61.0"+pillow-heif = "^0.18.0"[tool.poetry.group.dev.dependencies]pytest = "^7.4.3"diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.pyindex 2be3f3e7..40e603a2 100644--- a/backend/tests/test_agent_tools.py+++ b/backend/tests/test_agent_tools.py@@ -60,10 +60,30 @@ def test_canonical_tool_definitions_include_extract_assets() -> None:assert "extract_assets" in tool_namesextract_assets_tool = next(tool for tool in tools if tool.name == "extract_assets")assert extract_assets_tool.parameters["required"] == ["asset_descriptions"]- assert (- extract_assets_tool.parameters["properties"]["asset_descriptions"]["type"]- == "array"- )+ asset_descriptions = extract_assets_tool.parameters["properties"][+ "asset_descriptions"+ ]+ assert asset_descriptions["type"] == "array"++ # Good caller-side descriptors are essential for separating lookalikes and+ # mapping assets across multiple screenshots instead of making Gemini guess.+ item_description = asset_descriptions["items"]["description"].lower()+ for required_detail in (+ "distinctive appearance",+ "precise location",+ "nearby ui/context",+ "1-based screenshot number",+ "repeated lookalikes",+ "leftmost vs. rightmost",+ ):+ assert required_detail in item_description++ tool_description = extract_assets_tool.description.lower()+ assert "request order" in tool_description+ assert "permanent, embeddable public_url" in tool_description+ assert "attached crop preview" in tool_description+ assert "absent or unisolatable" in tool_description+ assert "do not call save_assets" in tool_descriptiondef test_canonical_tool_definitions_exclude_extract_assets_when_disabled() -> None:diff --git a/backend/tests/test_asset_extraction.py b/backend/tests/test_asset_extraction.pyindex bf13959b..bd4ae3c6 100644--- a/backend/tests/test_asset_extraction.py+++ b/backend/tests/test_asset_extraction.py@@ -1,24 +1,52 @@import base64import io+import json+import re+from collections.abc import Callable+from typing import Any, castimport pytestfrom google.genai import typesfrom PIL import Image-from asset_extraction import extract_assets_from_images+from asset_extraction import (+ ASSET_EXTRACTION_GEMINI_MODEL,+ MAX_ASSETS_PER_GEMINI_REQUEST,+ AssetDetectionBatch,+ AssetExtractionMetrics,+ extract_assets_from_images,+)-def _png_data_url(width: int = 10, height: int = 10) -> str:+def _image_data_url(+ image: Image.Image,+ *,+ image_format: str = "PNG",+ mime_type: str = "image/png",+ save_kwargs: dict[str, Any] | None = None,+) -> str:+ output = io.BytesIO()+ image.save(output, format=image_format, **(save_kwargs or {}))+ encoded = base64.b64encode(output.getvalue()).decode("ascii")+ return f"data:{mime_type};base64,{encoded}"+++def _quadrant_png_data_url(width: int = 10, height: int = 10) -> str:image = Image.new("RGB", (width, height), "white")- # Put a black square in the top-left quadrant so the crop is inspectable.for x in range(0, width // 2):for y in range(0, height // 2):image.putpixel((x, y), (0, 0, 0))+ return _image_data_url(image)- output = io.BytesIO()- image.save(output, format="PNG")- encoded = base64.b64encode(output.getvalue()).decode("ascii")- return f"data:image/png;base64,{encoded}"++def _split_color_png_data_url(+ left: str = "red", right: str = "blue", width: int = 20, height: int = 10+) -> str:+ image = Image.new("RGB", (width, height), right)+ for x in range(width // 2):+ for y in range(height):+ image.putpixel((x, y), Image.new("RGB", (1, 1), left).getpixel((0, 0)))+ return _image_data_url(image)def _decode_png_data_url(data_url: str) -> Image.Image:@@ -28,33 +56,45 @@ def _decode_png_data_url(data_url: str) -> Image.Image:return image-async def test_extract_assets_from_images_uses_gemini_boxes_and_crops_original(+def _response(+ detections: list[dict[str, Any]],+ *,+ prompt_tokens: int | None = None,+ candidate_tokens: int | None = None,+ total_tokens: int | None = None,+ response_id: str | None = None,+) -> types.GenerateContentResponse:+ usage = None+ if any(+ value is not None+ for value in (prompt_tokens, candidate_tokens, total_tokens)+ ):+ usage = types.GenerateContentResponseUsageMetadata(+ prompt_token_count=prompt_tokens,+ candidates_token_count=candidate_tokens,+ total_token_count=total_tokens,+ )+ # The SDK populates `parsed` after transport parsing. Passing a dict to the+ # response constructor itself is coerced through its broad BaseModel union.+ response = types.GenerateContentResponse(+ usage_metadata=usage,+ response_id=response_id,+ )+ response.parsed = {"detections": detections}+ return response+++def _install_fake_client(monkeypatch: pytest.MonkeyPatch,-) -> None:- calls: list[dict[str, object]] = []+ responder: Callable[[dict[str, Any]], types.GenerateContentResponse],+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:+ calls: list[dict[str, Any]] = []+ clients: list[dict[str, Any]] = []class FakeModels:- async def generate_content(self, **kwargs):+ async def generate_content(self, **kwargs: Any) -> types.GenerateContentResponse:calls.append(kwargs)- return types.GenerateContentResponse(- candidates=[- types.Candidate(- content=types.Content(- role="model",- parts=[- types.Part(- text=(- '{"image_index": 1, '- '"box_2d": [0, 0, 500, 500], '- '"label": "logo"}'- )- )- ],- )- )- ]- )+ return responder(kwargs)class FakeAio:models = FakeModels()@@ -62,22 +102,451 @@ async def test_extract_assets_from_images_uses_gemini_boxes_and_crops_original(class FakeClient:aio = FakeAio()- def __init__(self, api_key: str) -> None:- assert api_key == "gemini-key"+ def __init__(self, **kwargs: Any) -> None:+ clients.append(kwargs)monkeypatch.setattr("asset_extraction.genai.Client", FakeClient)+ return calls, clients+++def _parts_for_call(call: dict[str, Any]) -> list[types.Part]:+ contents = cast(list[types.Content], call["contents"])+ return cast(list[types.Part], contents[0].parts)+++def _prompt_for_call(call: dict[str, Any]) -> str:+ return _parts_for_call(call)[-1].text or ""+++async def test_batches_assets_with_schema_config_and_maps_out_of_order_response(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:+ return _response(+ [+ {+ "request_id": "asset-0002",+ "image_index": 1,+ "box_2d": [500, 500, 1000, 1000],+ "label": "lower-right avatar",+ },+ {+ "request_id": "asset-0001",+ "image_index": 1,+ "box_2d": [0, 0, 500, 500],+ "label": "upper-left logo",+ },+ ],+ prompt_tokens=100,+ candidate_tokens=20,+ total_tokens=120,+ response_id="provider-response-1",+ )++ calls, clients = _install_fake_client(monkeypatch, responder)+ metrics = AssetExtractionMetrics()+ descriptions = [+ "black square logo in the upper-left of screenshot 1",+ "white avatar in the lower-right of screenshot 1",+ ]result = await extract_assets_from_images(- image_data_urls=[_png_data_url()],- asset_descriptions=["logo", "avatar"],+ image_data_urls=[_quadrant_png_data_url()],+ asset_descriptions=descriptions,gemini_api_key="gemini-key",+ metrics=metrics,+ )++ # Both descriptions share one multimodal request rather than paying image+ # tokens and latency once per asset.+ assert len(calls) == 1+ assert clients[0]["api_key"] == "gemini-key"+ assert clients[0]["http_options"].api_version == "v1alpha"+ assert calls[0]["model"] == ASSET_EXTRACTION_GEMINI_MODEL++ config = cast(types.GenerateContentConfig, calls[0]["config"])+ assert config.temperature == 0.5+ assert config.response_mime_type == "application/json"+ assert config.response_schema is AssetDetectionBatch+ # Gemini v1alpha rejects the SDK's serialized `additional_properties`+ # keyword even though standard JSON Schema supports it.+ assert "additionalProperties" not in json.dumps(+ AssetDetectionBatch.model_json_schema())+ assert config.thinking_config is not None+ assert config.thinking_config.thinking_level == types.ThinkingLevel.MINIMAL+ assert "Never return masks" in str(config.system_instruction)++ parts = _parts_for_call(calls[0])+ assert len(parts) == 2 # one normalized image + one prompt+ assert parts[0].inline_data is not None+ assert parts[0].inline_data.mime_type == "image/png"+ assert parts[0].media_resolution is not None+ assert (+ parts[0].media_resolution.level+ == types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH+ )++ prompt = parts[-1].text or ""+ assert '"request_id": "asset-0001"' in prompt+ assert '"request_id": "asset-0002"' in prompt+ assert "[ymin, xmin, ymax, xmax]" in prompt+ assert "smallest axis-aligned box" in prompt+ assert "Exclude surrounding UI" in prompt+ assert "Repeated lookalikes are separate instances" in prompt+ assert "image_index=null, box_2d=null" in prompt+ assert "Never return masks" in prompt+ assert "Never guess a box" in prompt- assert len(calls) == 2- assert calls[0]["model"] == "gemini-3.6-flash"- assets = result["assets"]- assert assets[0]["description"] == "logo"+ assets = cast(list[dict[str, Any]], result["assets"])+ # Response order is intentionally reversed; result order remains caller order.+ assert [asset["description"] for asset in assets] == descriptions+ assert [asset["label"] for asset in assets] == [+ "upper-left logo",+ "lower-right avatar",+ ]assert assets[0]["status"] == "ok"assert assets[0]["box_2d"] == [0, 0, 500, 500]- cropped = _decode_png_data_url(assets[0]["data_url"])- assert cropped.size == (5, 5)+ assert _decode_png_data_url(cast(str, assets[0]["data_url"])).size == (5, 5)+ assert metrics.request_count == 1+ assert metrics.prompt_token_count == 100+ assert metrics.candidate_token_count == 20+ assert metrics.total_token_count == 120+ assert metrics.response_ids == ["provider-response-1"]+++async def test_chunks_at_cookbook_25_object_cap_with_stable_global_ids(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ids_by_call: list[list[str]] = []++ def responder(call: dict[str, Any]) -> types.GenerateContentResponse:+ request_ids = re.findall(r'"request_id": "(asset-\d{4})"', _prompt_for_call(call))+ ids_by_call.append(request_ids)+ return _response(+ [+ {+ "request_id": request_id,+ "image_index": None,+ "box_2d": None,+ "label": None,+ }+ for request_id in reversed(request_ids)+ ]+ )++ calls, _clients = _install_fake_client(monkeypatch, responder)+ descriptions = [f"asset number {index}" for index in range(51)]++ result = await extract_assets_from_images(+ image_data_urls=[_quadrant_png_data_url()],+ asset_descriptions=descriptions,+ gemini_api_key="gemini-key",+ )++ assert len(calls) == 3+ assert sorted(len(request_ids) for request_ids in ids_by_call) == [1, 25, 25]+ assert all(+ len(request_ids) <= MAX_ASSETS_PER_GEMINI_REQUEST+ for request_ids in ids_by_call+ )+ assert {request_id for ids in ids_by_call for request_id in ids} == {+ f"asset-{index:04d}" for index in range(1, 52)+ }+ assets = cast(list[dict[str, Any]], result["assets"])+ assert [asset["description"] for asset in assets] == descriptions+ assert all(asset["status"] == "missing" for asset in assets)+++async def test_maps_duplicate_instances_missing_items_and_original_source_numbers(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:+ return _response(+ [+ {+ "request_id": "asset-0004",+ "image_index": None,+ "box_2d": None,+ "label": None,+ },+ {+ "request_id": "asset-0002",+ "image_index": 1,+ "box_2d": [0, 500, 1000, 1000],+ "label": "right duplicate",+ },+ {+ "request_id": "asset-0003",+ "image_index": 3,+ "box_2d": [0, 0, 1000, 1000],+ "label": "green illustration",+ },+ {+ "request_id": "asset-0001",+ "image_index": 1,+ "box_2d": [0, 0, 1000, 500],+ "label": "left duplicate",+ },+ ]+ )++ calls, _clients = _install_fake_client(monkeypatch, responder)+ invalid_gif = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=="+ descriptions = [+ "left red duplicate tile in screenshot 1",+ "right blue duplicate tile in screenshot 1",+ "green illustration centered in screenshot 3",+ "orange rocket icon that is absent from every screenshot",+ ]++ result = await extract_assets_from_images(+ image_data_urls=[+ _split_color_png_data_url(),+ invalid_gif,+ _image_data_url(Image.new("RGB", (8, 6), "green")),+ ],+ asset_descriptions=descriptions,+ gemini_api_key="gemini-key",+ )++ prompt = _prompt_for_call(calls[0])+ assert "attached image 1 = source image 1" in prompt+ # Invalid source 2 is skipped without renumbering the caller's screenshot 3.+ assert "attached image 2 = source image 3" in prompt++ assets = cast(list[dict[str, Any]], result["assets"])+ assert [asset["description"] for asset in assets] == descriptions+ assert [asset["image_index"] for asset in assets] == [1, 1, 3, None]+ assert [asset["status"] for asset in assets] == ["ok", "ok", "ok", "missing"]++ left_crop = _decode_png_data_url(cast(str, assets[0]["data_url"]))+ right_crop = _decode_png_data_url(cast(str, assets[1]["data_url"]))+ third_crop = _decode_png_data_url(cast(str, assets[2]["data_url"]))+ assert left_crop.getpixel((0, 0)) == (255, 0, 0)+ assert right_crop.getpixel((0, 0)) == (0, 0, 255)+ assert third_crop.getpixel((0, 0)) == (0, 128, 0)+ assert assets[3]["data_url"] is None+++async def test_exif_orientation_is_identical_for_gemini_and_crop_source(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ sent_image_sizes: list[tuple[int, int]] = []++ def responder(call: dict[str, Any]) -> types.GenerateContentResponse:+ image_part = _parts_for_call(call)[0]+ assert image_part.inline_data is not None+ sent_image = Image.open(io.BytesIO(image_part.inline_data.data or b""))+ sent_image.load()+ sent_image_sizes.append(sent_image.size)+ return _response(+ [+ {+ "request_id": "asset-0001",+ "image_index": 1,+ "box_2d": [0, 0, 1000, 1000],+ "label": "rotated photo",+ }+ ]+ )++ _calls, _clients = _install_fake_client(monkeypatch, responder)+ landscape = Image.new("RGB", (4, 2), "purple")+ exif = Image.Exif()+ exif[274] = 6 # rotate 90 degrees clockwise for display+ oriented_jpeg = _image_data_url(+ landscape,+ image_format="JPEG",+ mime_type="image/jpeg",+ save_kwargs={"exif": exif, "quality": 100},+ )++ result = await extract_assets_from_images(+ image_data_urls=[oriented_jpeg],+ asset_descriptions=["whole purple photo in screenshot 1"],+ gemini_api_key="gemini-key",+ )++ assert sent_image_sizes == [(2, 4)]+ asset = cast(list[dict[str, Any]], result["assets"])[0]+ crop = _decode_png_data_url(cast(str, asset["data_url"]))+ assert crop.size == (2, 4)+++async def test_normalizes_documented_heif_input_to_shared_png_pixels(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ sent_mime_types: list[str] = []++ def responder(call: dict[str, Any]) -> types.GenerateContentResponse:+ image_part = _parts_for_call(call)[0]+ assert image_part.inline_data is not None+ sent_mime_types.append(image_part.inline_data.mime_type or "")+ return _response(+ [+ {+ "request_id": "asset-0001",+ "image_index": 1,+ "box_2d": [0, 0, 1000, 1000],+ "label": "orange HEIF image",+ }+ ]+ )++ _calls, _clients = _install_fake_client(monkeypatch, responder)+ heif_data_url = _image_data_url(+ Image.new("RGB", (3, 2), "orange"),+ image_format="HEIF",+ mime_type="image/heif",+ )++ result = await extract_assets_from_images(+ image_data_urls=[heif_data_url],+ asset_descriptions=["orange image in screenshot 1"],+ gemini_api_key="gemini-key",+ )++ assert sent_mime_types == ["image/png"]+ asset = cast(list[dict[str, Any]], result["assets"])[0]+ crop = _decode_png_data_url(cast(str, asset["data_url"]))+ assert crop.size == (3, 2)+++async def test_semantically_normalizes_or_rejects_boxes_before_crop(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:+ return _response(+ [+ {+ "request_id": "asset-0001",+ "image_index": 1,+ "box_2d": [1200, 1100, -100, -50],+ "label": "reversed and clamped",+ },+ {+ "request_id": "asset-0002",+ "image_index": 1,+ "box_2d": [0, 0, float("nan"), 500],+ "label": "non-finite",+ },+ {+ "request_id": "asset-0003",+ "image_index": 1,+ "box_2d": [-100, -100, -50, -50],+ "label": "empty after clamp",+ },+ {+ "request_id": "asset-0004",+ "image_index": 1,+ "box_2d": [0.1, 0.1, 99.9, 99.9],+ "label": "fractional one pixel",+ },+ {+ "request_id": "asset-0005",+ "image_index": 99,+ "box_2d": [0, 0, 500, 500],+ "label": "invalid source",+ },+ ]+ )++ _calls, _clients = _install_fake_client(monkeypatch, responder)+ result = await extract_assets_from_images(+ image_data_urls=[_quadrant_png_data_url()],+ asset_descriptions=["reversed", "nan", "empty", "fractional", "bad source"],+ gemini_api_key="gemini-key",+ )++ assets = cast(list[dict[str, Any]], result["assets"])+ assert assets[0]["box_2d"] == [0, 0, 1000, 1000]+ assert assets[0]["status"] == "ok"+ assert _decode_png_data_url(cast(str, assets[0]["data_url"])).size == (10, 10)++ assert assets[1]["box_2d"] is None+ assert assets[1]["status"] == "missing"+ assert assets[2]["box_2d"] is None+ assert assets[2]["status"] == "missing"++ # Outward floor/ceil retains a real edge pixel for a small valid box.+ assert assets[3]["status"] == "ok"+ assert _decode_png_data_url(cast(str, assets[3]["data_url"])).size == (1, 1)++ assert assets[4]["box_2d"] == [0, 0, 500, 500]+ assert assets[4]["status"] == "missing"+ assert assets[4]["data_url"] is None+ assert "error" in result+++async def test_does_not_fallback_to_regex_parsing_for_fenced_free_form_json(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ def responder(_call: dict[str, Any]) -> types.GenerateContentResponse:+ return types.GenerateContentResponse(+ candidates=[+ types.Candidate(+ content=types.Content(+ role="model",+ parts=[+ types.Part(+ text=(+ "```json\n"+ '{"detections":[{"request_id":"asset-0001",'+ '"image_index":1,"box_2d":[0,0,500,500],'+ '"label":"logo"}]}\n'+ "```"+ )+ )+ ],+ )+ )+ ]+ )++ _calls, _clients = _install_fake_client(monkeypatch, responder)+ result = await extract_assets_from_images(+ image_data_urls=[_quadrant_png_data_url()],+ asset_descriptions=["logo"],+ gemini_api_key="gemini-key",+ )++ asset = cast(list[dict[str, Any]], result["assets"])[0]+ assert asset["status"] == "missing"+ assert asset["box_2d"] is None+++async def test_rejects_unsupported_or_undecodable_inputs_before_client_creation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ client_created = False++ class UnexpectedClient:+ def __init__(self, **_kwargs: Any) -> None:+ nonlocal client_created+ client_created = True++ monkeypatch.setattr("asset_extraction.genai.Client", UnexpectedClient)++ result = await extract_assets_from_images(+ image_data_urls=[+ "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==",+ "data:image/png;base64,not-valid-base64",+ ],+ asset_descriptions=["logo"],+ gemini_api_key="gemini-key",+ )++ assert client_created is False+ assert result == {+ "assets": [],+ "error": "No valid input images were available for asset extraction.",+ }diff --git a/backend/tests/test_asset_extraction_benchmark.py b/backend/tests/test_asset_extraction_benchmark.pynew file mode 100644index 00000000..58dbd5ee--- /dev/null+++ b/backend/tests/test_asset_extraction_benchmark.py@@ -0,0 +1,53 @@+from pathlib import Path++import pytest+from PIL import Image++from evals.asset_extraction_benchmark import box_iou, build_fixture+++def test_live_benchmark_fixture_covers_required_asset_cases(tmp_path: Path) -> None:+ fixture = build_fixture(tmp_path)++ assert len(fixture.image_data_urls) == 2+ assert [path.name for path in fixture.image_paths] == [+ "fixture_source_1.png",+ "fixture_source_2.png",+ ]+ assert all(path.exists() for path in fixture.image_paths)+ with Image.open(fixture.image_paths[0]) as first:+ assert first.size == (1200, 800)+ with Image.open(fixture.image_paths[1]) as second:+ assert second.size == (1000, 700)++ categories = {target.category for target in fixture.targets}+ assert categories == {+ "logo",+ "photo_or_illustration",+ "small_icon",+ "duplicate_instance",+ "multiple_source_images",+ "absent_target",+ }+ assert sum(+ target.category == "duplicate_instance" for target in fixture.targets+ ) == 2+ second_source = next(+ target+ for target in fixture.targets+ if target.category == "multiple_source_images"+ )+ assert second_source.image_index == 2+ absent = next(+ target for target in fixture.targets if target.category == "absent_target"+ )+ assert absent.image_index is None+ assert absent.expected_box_2d is None+++def test_box_iou_reports_exact_partial_and_disjoint_boxes() -> None:+ assert box_iou([0, 0, 500, 500], [0, 0, 500, 500]) == 1+ assert box_iou([0, 0, 500, 500], [500, 500, 1000, 1000]) == 0+ assert box_iou([0, 0, 500, 500], [250, 250, 750, 750]) == pytest.approx(+ 1 / 7+ )