Skip to content
Python
18.9 KB
Raw Blame
#!/usr/bin/env python3
"""
Post-process stroke overlay renderer for SketchVLM outputs.

Given a results folder containing:
  - item_00001.json
  - item_00001_orig.jpg/png (optional)
  - item_00001_grid.png/jpg (optional)

This script extracts <strokes> blocks from model_output_full (or model_output)
and renders them onto either the orig or grid image.

Supports:
  - Grid token points: 'x500y95'
  - Numeric points: (500,95) or 500,95
  - bottom-left or top-left origin interpretation
  - mapping from grid-image coordinates to orig-image coordinates using grid_config + cell_pixel_map

Example:
  python render_strokes_postprocess.py --results-dir results/mix_eval/20251221_210419 \
      --base grid --origin top-left --only "1,2,3"

"""
from __future__ import annotations

import argparse
import html
import json
import re
from pathlib import Path
from typing import Dict, List, Tuple, Optional

import io

import cairosvg
from PIL import Image

# Use the exact SVG + Bezier utilities that collab_sketch_with_label.py uses,
# so postprocessed renders match your in-run renders.
import utils


STROKE_BLOCK_RE = re.compile(r"<s(\d+)>(.*?)</s\1>", re.S | re.I)
POINTS_BLOCK_RE = re.compile(r"<points>(.*?)</points>", re.S | re.I)
TEXT_BLOCK_RE = re.compile(r"<text([^>]*)>\s*(?:'([^']+)'|\"([^\"]+)\"|([^<]+))\s*</text>", re.S | re.I)

# token style: 'x12y34' (quotes optional)
TOKEN_RE = re.compile(r"x(\d+)\s*y(\d+)", re.I)

# numeric pairs: (12,34) or 12,34
NUMPAIR_RE = re.compile(r"\(?\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)?")


def _parse_only_list(s: Optional[str]) -> Optional[List[int]]:
    if not s:
        return None
    out = []
    for part in s.split(","):
        part = part.strip()
        if not part:
            continue
        out.append(int(part))
    return out or None


def _find_image(results_dir: Path, stem: str, preferred_exts=("png", "jpg", "jpeg", "webp")) -> Optional[Path]:
    for ext in preferred_exts:
        p = results_dir / f"{stem}.{ext}"
        if p.exists():
            return p
    return None


def _extract_strokes_xml(d: Dict) -> str:
    # Prefer the cleaned field first when available.
    for k in ("model_output", "model_output_full"):
        v = d.get(k)
        if isinstance(v, str) and v.strip():
            return v
    return ""


def _svg_root_open(w: int, h: int) -> str:
    return (
        f'<svg width="{w}" height="{h}" viewBox="0 0 {w} {h}" '
        f'xmlns="http://www.w3.org/2000/svg">'
    )


def _composite_svg_on_image(base_rgb: Image.Image, svg_text: str, out_png: Path):
    """Match collab_sketch_with_label.py compositing semantics."""
    over = Image.open(io.BytesIO(cairosvg.svg2png(bytestring=svg_text.encode("utf-8")))).convert("RGBA")
    base = base_rgb.convert("RGBA")
    if over.size != base.size:
        inner = re.sub(r'^.*?<svg[^>]*>|</svg>\s*$', '', svg_text, flags=re.S)
        svg_text = (
            f'<svg width="{base.size[0]}" height="{base.size[1]}" '
            f'xmlns="http://www.w3.org/2000/svg">{inner}</svg>'
        )
        over = Image.open(io.BytesIO(cairosvg.svg2png(bytestring=svg_text.encode("utf-8")))).convert("RGBA")
    Image.alpha_composite(base, over).convert("RGB").save(out_png)


def _get_drawable_bbox_from_cell_map(cell_pixel_map: Dict[str, List[float]], cell_size: float) -> Tuple[float, float, float, float]:
    """
    Estimate the drawable (image) rectangle within a grid image from cell centers.
    Returns (left, top, right, bottom) in grid-image pixel coordinates.
    """
    xs = []
    ys = []
    for _, (cx, cy) in cell_pixel_map.items():
        xs.append(float(cx))
        ys.append(float(cy))
    if not xs or not ys:
        return (0.0, 0.0, 1.0, 1.0)

    min_x, max_x = min(xs), max(xs)
    min_y, max_y = min(ys), max(ys)

    # centers -> boundaries
    half = float(cell_size) / 2.0
    left = min_x - half
    right = max_x + half
    top = min_y - half
    bottom = max_y + half
    return (left, top, right, bottom)


def _token_to_cell_key(x: int, y: int) -> str:
    return f"x{x}y{y}"


def _flip_y_if_needed(y: int, res_y: int, origin: str) -> int:
    # token coordinates are 1..res_y in "grid token space"
    if origin == "top-left":
        # top-left means y=1 at top. If tokens were produced with bottom-left,
        # convert by flipping. (If tokens were already top-left, user should
        # pass origin=top-left and the json's cell_pixel_map would have y=1 at top,
        # in which case flipping would be wrong. But user explicitly asked for a switch,
        # so we implement flipping as the conversion.)
        return res_y - y + 1
    return y  # bottom-left


def _map_points_to_pixels(
    pts_text: str,
    base_mode: str,
    origin: str,
    res_x: int,
    res_y: int,
    base_size: Tuple[int, int],
    grid_size_px: Optional[Tuple[int, int]] = None,
    cell_pixel_map: Optional[Dict[str, List[float]]] = None,
    cell_size: Optional[float] = None,
) -> Tuple[List[Tuple[float, float]], str]:
    """
    Map points in pts_text to pixel coordinates on the chosen base image.

    Returns (points_px, kind) where kind is 'token' or 'numeric'.
    """
    Wb, Hb = base_size

    # 1) token points 'xNyM'
    tokens = [(int(a), int(b)) for a, b in TOKEN_RE.findall(pts_text)]
    if tokens:
        # If base is grid image: use cell_pixel_map directly when available.
        if base_mode == "grid" and cell_pixel_map:
            out = []
            for x, y in tokens:
                y2 = _flip_y_if_needed(y, res_y, origin)
                key = _token_to_cell_key(x, y2)
                if key not in cell_pixel_map:
                    raise KeyError(key)
                cx, cy = cell_pixel_map[key]
                out.append((float(cx), float(cy)))
            return out, "token"

        # If base is orig image (no grid), mimic collab_sketch_with_label.py's
        # "annotated_nogrid" behavior: the raw/orig image was placed onto the grid
        # canvas with a fixed offset (no scaling). So we do a pure translation from
        # grid-canvas pixel space -> orig pixel space.
        if base_mode == "orig" and cell_pixel_map and cell_size and grid_size_px:
            grid_w, grid_h = grid_size_px
            # In the main renderer, the raw image is placed at:
            #   x_offset = cell_size
            #   y_offset = grid_h - cell_size - raw_h
            x_off = float(cell_size)
            y_off = float(grid_h) - float(cell_size) - float(Hb)

            out = []
            for x, y in tokens:
                y2 = _flip_y_if_needed(y, res_y, origin)
                key = _token_to_cell_key(x, y2)
                if key not in cell_pixel_map:
                    raise KeyError(key)
                gx, gy = cell_pixel_map[key]  # grid-canvas pixel coords (center of cell)
                ox = float(gx) - x_off
                oy = float(gy) - y_off
                out.append((ox, oy))
            return out, "token"

        # If no mapping info exists, interpret tokens as normalized to res_x/res_y and scale to base image
        out = []
        for x, y in tokens:
            # tokens assumed 1..res
            xf = (x / float(res_x)) * (Wb - 1)
            if origin == "bottom-left":
                yf = ((res_y - y) / float(res_y)) * (Hb - 1)
            else:
                yf = (y / float(res_y)) * (Hb - 1)
            out.append((xf, yf))
        return out, "token"

    # 2) numeric pairs
    nums = [(float(a), float(b)) for a, b in NUMPAIR_RE.findall(pts_text)]
    if nums:
        out = []
        for x, y in nums:
            xf = (x / float(res_x)) * (Wb - 1)
            if origin == "bottom-left":
                yf = ((res_y - y) / float(res_y)) * (Hb - 1)
            else:
                yf = (y / float(res_y)) * (Hb - 1)
            out.append((xf, yf))
        return out, "numeric"

    return [], "none"


def _iter_stroke_blocks(strokes_xml: str) -> List[Tuple[int, str]]:
    blocks = []
    for m in STROKE_BLOCK_RE.finditer(strokes_xml):
        n = int(m.group(1))
        body = m.group(2)
        blocks.append((n, body))
    return blocks


def _extract_strokes_section(xml: str) -> str:
    m = re.search(r"<strokes>(.*?)</strokes>", xml, re.S | re.I)
    return m.group(1) if m else xml


def _parse_text_style(attrs: str) -> Tuple[Optional[float], Optional[str]]:
    # attrs like: size="4.0" color="black"
    size = None
    color = None
    m_size = re.search(r'size\s*=\s*"([^"]+)"', attrs)
    if m_size:
        try:
            size = float(m_size.group(1))
        except Exception:
            size = None
    m_color = re.search(r'color\s*=\s*"([^"]+)"', attrs)
    if m_color:
        color = m_color.group(1).strip()
    return size, color


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--results-dir", required=True, help="Folder containing item_*.json and images")
    ap.add_argument("--base", choices=["orig", "grid"], default="grid", help="Render strokes onto which base image")
    ap.add_argument("--origin", choices=["bottom-left", "top-left"], default="bottom-left",
                    help="How to interpret model coordinates. Use the same origin the prompt used.")
    ap.add_argument("--res-x", type=int, default=None, help="Model coordinate max X (defaults to json grid_config.res_x if present)")
    ap.add_argument("--res-y", type=int, default=None, help="Model coordinate max Y (defaults to json grid_config.res_y if present)")
    ap.add_argument("--only", type=str, default=None, help="Comma-separated item indices to process (e.g., 1,2,3)")
    ap.add_argument("--out-dir", type=str, default=None, help="Where to write outputs (default: results-dir)")
    ap.add_argument("--stroke-width", type=float, default=0.0, help="Override stroke width in px (0 = infer from cell_size)")
    ap.add_argument("--cell-size", type=float, default=15.0,
                    help="(Mimic collab_sketch bug) Base cell_size used for stroke/text sizing when rendering grid-token outputs. Default 15.")
    ap.add_argument("--use-json-cell-size", action="store_true",
                    help="Use grid_config.cell_size from each JSON for sizing (disables mimic-bug sizing).")
    ap.add_argument("--alt-colors", action="store_true", help="Alternate green/pink like colab mode (default: all green)")
    ap.add_argument("--save-svg", action="store_true", help="Also save an SVG alongside the PNG")
    args = ap.parse_args()

    results_dir = Path(args.results_dir)
    out_dir = Path(args.out_dir) if args.out_dir else results_dir
    out_dir.mkdir(parents=True, exist_ok=True)

    only = _parse_only_list(args.only)

    json_paths = sorted(results_dir.glob("item_*.json"))
    if only is not None:
        keep = set(only)
        filtered = []
        for p in json_paths:
            m = re.search(r"item_(\d+)\.json$", p.name)
            if m and int(m.group(1)) in keep:
                filtered.append(p)
        json_paths = filtered

    if not json_paths:
        raise SystemExit("No item_*.json found (or none matched --only).")

    # Match collab_sketch_with_label.py defaults
    default_stroke_color = "green"
    alt_colors = ["green", "pink"]
    text_font_family = "Arial"
    text_font_scale = 3.2

    for jp in json_paths:
        d = json.loads(jp.read_text(encoding="utf-8"))

        item_id_match = re.search(r"item_(\d+)\.json$", jp.name)
        item_id = int(item_id_match.group(1)) if item_id_match else int(d.get("index", 0))

        grid_cfg = d.get("grid_config") or {}
        cell_pixel_map = d.get("cell_pixel_map") or None

        # coordinate system size
        res_x = args.res_x if args.res_x is not None else int(grid_cfg.get("res_x") or 1000)
        res_y = args.res_y if args.res_y is not None else int(grid_cfg.get("res_y") or 1000)

        # base image
        base_img_path = _find_image(results_dir, f"item_{item_id:05d}_orig") if args.base == "orig" else _find_image(results_dir, f"item_{item_id:05d}_grid")
        if base_img_path is None:
            # fallback to paths stored in json (may be relative)
            key = "raw_image" if args.base == "orig" else "grid_image"
            maybe = d.get(key)
            if maybe:
                base_img_path = (results_dir / Path(maybe).name) if not Path(maybe).is_absolute() else Path(maybe)
        if base_img_path is None or not base_img_path.exists():
            print(f"[WARN] Missing base image for item {item_id:05d} ({args.base}). Skipping.")
            continue

        base_img = Image.open(base_img_path).convert("RGB")
        Wb, Hb = base_img.size

        strokes_xml_full = _extract_strokes_xml(d)
        strokes_xml = _extract_strokes_section(strokes_xml_full)

        # mapping info for token->pixel
        grid_size_px = None
        if "grid_size_px" in grid_cfg:
            try:
                grid_size_px = (int(grid_cfg["grid_size_px"][0]), int(grid_cfg["grid_size_px"][1]))
            except Exception:
                grid_size_px = None
        cell_size = float((grid_cfg.get("cell_size") or 0.0) if (grid_cfg and args.use_json_cell_size) else args.cell_size)

        # Infer stroke width like the in-run code: stroke_width = cell_size * 0.6
        # Note: for base=orig with grid-token runs, we mimic the in-run behavior which uses
        # a pure translation (no scaling), so stroke width should NOT be scaled.
        stroke_width = (float(args.stroke_width) if float(args.stroke_width) > 0 else (float(cell_size) * 0.6 if cell_size else 3.0))

        blocks = _iter_stroke_blocks(strokes_xml)
        if not blocks:
            print(f"[WARN] No <sN> blocks found in item {item_id:05d}.")
            continue

        # Build an SVG overlay that matches the in-run rendering (beziers, cap/join styles).
        svg_parts: List[str] = [_svg_root_open(Wb, Hb)]

        for s_idx, body in blocks:
            m_pts = POINTS_BLOCK_RE.search(body)
            if not m_pts:
                continue

            # parse optional id for stable group ids
            m_id = re.search(r"<id>(.*?)</id>", body, re.S | re.I)
            stroke_label = (m_id.group(1).strip() if m_id else f"s{s_idx}")
            stroke_label = re.sub(r"[^\w\-]", "_", stroke_label)

            pts_text = m_pts.group(1)
            pts_px, _kind = _map_points_to_pixels(
                pts_text=pts_text,
                base_mode=args.base,
                origin=args.origin,
                res_x=res_x,
                res_y=res_y,
                base_size=(Wb, Hb),
                grid_size_px=grid_size_px,
                cell_pixel_map=cell_pixel_map,
                cell_size=cell_size if cell_size > 0 else None,
            )
            if not pts_px:
                continue

            stroke_color = (
                alt_colors[(s_idx - 1) % 2] if args.alt_colors else default_stroke_color
            )

            # text stroke?
            m_txt = TEXT_BLOCK_RE.search(body)
            if m_txt:
                attrs = m_txt.group(1) or ""
                text_raw = (m_txt.group(2) or m_txt.group(3) or m_txt.group(4) or "").strip()
                text_val = html.escape(text_raw)
                size_override, color_override = _parse_text_style(attrs)
                # Match collab_sketch_with_label.py behavior:
                # - default font size is cell_size * text_font_scale
                # - if <text size="k"> is present, interpret k as a MULTIPLIER of cell_size (not pixels)
                # - clamp to a sane range like the in-run code.
                if cell_size:
                    default_px = float(cell_size) * float(text_font_scale)
                    if size_override is not None:
                        # interpret as multiplier of cell_size
                        mult = float(size_override)
                        # clamp multipliers similarly to the main code
                        if mult < 0.8:
                            mult = 0.8
                        if mult > 6.0:
                            mult = 6.0
                        base_px = float(cell_size) * mult
                    else:
                        base_px = float(default_px)
                else:
                    base_px = float(size_override) if size_override is not None else 14.0
                font_px = int(round(base_px))
                fill = color_override or stroke_color or "black"
                x0, y0 = pts_px[0]
                svg_parts.append(
                    f'<g id="{stroke_label}_s{s_idx}">'
                    f'<text x="{x0:.1f}" y="{y0:.1f}" text-anchor="middle" dominant-baseline="central" '
                    f'font-family="{text_font_family}" font-size="{font_px}" fill="{fill}">{text_val}</text>'
                    f'</g>'
                )
                continue

            # t-values (optional)
            m_t = re.search(r"<t_values>(.*?)</t_values>", body, re.S | re.I)
            t_values: List[float] = []
            if m_t:
                raw = m_t.group(1).strip().strip("[]")
                parts = [p.strip() for p in raw.split(",") if p.strip()]
                try:
                    t_values = [float(p) for p in parts]
                except Exception:
                    t_values = []
            n = len(pts_px)
            if (not t_values) or (len(t_values) != n):
                t_values = [i / (n - 1) if n > 1 else 0.0 for i in range(n)]

            sampled_points = [[float(x), float(y)] for (x, y) in pts_px]
            cps = utils.estimate_bezier_control_points(sampled_points, t_values)
            svg_parts.append(
                utils.format_svg_single_stroke(
                    cps,
                    dim=(Wb, Hb),
                    stroke_width=stroke_width,
                    stroke_counter=s_idx,
                    group_id=stroke_label,
                    stroke_color=stroke_color,
                )
            )

        svg_parts.append("</svg>")
        overlay_svg = "\n".join(svg_parts)

        out_png = out_dir / f"item_{item_id:05d}_annotated_post_{args.base}.png"
        _composite_svg_on_image(base_img, overlay_svg, out_png)
        print(f"[OK] Wrote {out_png}")

        if args.save_svg:
            out_svg = out_dir / f"item_{item_id:05d}_annotated_post_{args.base}.svg"
            out_svg.write_text(overlay_svg, encoding="utf-8")
            print(f"[OK] Wrote {out_svg}")


if __name__ == "__main__":
    main()


'''

Render onto the grid image (easy path, uses cell_pixel_map directly):

python render_strokes_postprocess.py \
  --results-dir results/mix_eval/20251221_210419 \
  --base grid \
  --origin bottom-left \
  --only "1,2,3"
  

Render onto the orig image (grid→orig mapping happens automatically if cell_pixel_map exists):

python render_strokes_postprocess.py \
  --results-dir results/mix_eval/20251221_210419 \
  --base orig \
  --origin bottom-left \
  --only "1,2,3"


If you want SVG too:

python render_strokes_postprocess.py \
  --results-dir results/mix_eval/20251221_210419 \
  --base orig \
  --origin bottom-left \
  --only "1,2,3" \
  --save-svg


If your no-grid run used res_x=res_y=1000 coords:

python render_strokes_postprocess.py \
  --results-dir results/mix_eval/20251221_210419 \
  --base orig \
  --origin top-left \
  --res-x 1000 --res-y 1000


'''