Skip to content

Local video editor for coding agents: 42 FFmpeg tools with a machine-readable contract and an MCP server, for Claude Code, Cursor and Codex. Cut, captions, reframe, loudness, platform exports. No cloud, no API keys.

feat: look.py --ink measures non-background pixels per PNG (#261)

Shogo Kajiyama7ba483fbdcf0
diff --git a/docs/roadmap.md b/docs/roadmap.md
index 94d0adb..964d955 100644
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@@ -17,7 +17,7 @@ minor and `fix` PRs into a patch, so each block below is one or two `feat` PRs p
- **planned** — not released. Nothing below a *planned* heading exists in any published version;
the feature lines are the intent, not a description of the code.
-The released version today is **1.19.2** — `multicam.py --write-project FILE` writes a
+The released version today is **1.19.3** — `multicam.py --write-project FILE` writes a
`render.py` project that reproduces the tool's own cut decision (eval 21's `mc2` gap, no more
hand-translating cuts/offsets into `clips[]`), and `--filler` alone no longer jump-cuts unrelated
dead-air silence (eval 19's `fw1`/`fw3`: "remove the ums" silently removed ~5s of unasked-for
@@ -25,8 +25,9 @@ pauses too; fixed by skipping generic silence detection unless `--speech-aware`
1.19.0 (the `--write-project` feature) is a `feat:` release, hence the minor bump; it does not
start the "1.19.0: observability, portability" theme further down this document, which remains
planned. 1.19.2 is docs-only (SKILL.md's overlong-word and MCP core-12 rows, this same truth-up).
-1.18.4 (the caption column-edge slice, `cs2`) is unchanged from before; see its own
-section below.
+1.19.3 extends 1.18.4's column-edge slice to `graphics.py`'s own `wrapped()` helper (lower-third,
+title, sticker, hook and meme labels), so an unbreakable overlong word no longer clips off-frame
+on those templates either.
| version | state | evidence |
|---|---|---|
@@ -54,6 +55,7 @@ section below.
| 1.19.0 | shipped, eval pending | `multicam.py --write-project FILE` (eval 21's `mc2` gap): writes a `render.py` project whose `clips[]` reproduces multicam's own cut decision exactly (`src`/`in`/`out` per cut, offset-shifted to each camera's own timeline), for `--switch energy`, a manual `--switch` spec, or `--auto` alike; multicam's own combined render and `--edl`/`--offsets-only` unchanged. `feat:` release, hence the minor bump — not the start of the "1.19.0" theme further down this document, which is unrelated and still planned |
| 1.19.1 | shipped, eval pending | eval 19's `fw1`/`fw3`: `silence.py --filler` alone (no `--speech-aware`) jump-cut unrelated dead-air silence gaps too, ~5s unasked. Fixed: generic silence detection is skipped unless `--speech-aware` is also given, so `--filler` alone removes only the timed filler-word spans; `--speech-aware` alone and `--filler --speech-aware` combined are unchanged |
| 1.19.2 | shipped, docs-only | SKILL.md rows for 1.18.3's MCP core-12 (previously undocumented) and 1.18.4's overlong-word column-edge slice, plus this document's own roadmap truth-up. No script changes |
+| 1.19.3 | shipped, eval pending | `graphics.py`'s `wrapped()` helper (used by the lower-third, title, sticker, hook and meme templates) turns on the same `slice_overlong` escape hatch `caption.py` turned on in 1.18.4, so a single unbreakable overlong atom drawn through a template no longer renders past the frame's safe width. `broken_inside_word` reported the same way as `caption.py`. A fitting Thai phrase or katakana run is unchanged, guarded by the same font-availability skip as `ShapingTests`. Tool count still 42, no new CLI flag for this change specifically -- `look.py --ink` is a separate, not-yet-versioned addition tracked in its own PR |
| 1.21.0, 2.0.0 | planned | — |
## 1.8.0 — one-call delivery, quieter checks, encoder flags (shipped + evaluated, eval 8)
diff --git a/scripts/look.py b/scripts/look.py
index 5029610..0604daa 100755
--- a/scripts/look.py
+++ b/scripts/look.py
@@ -14,10 +14,10 @@ import argparse
import os
import sys
from pathlib import Path
-from typing import List
+from typing import List, Optional
from _platforms import PLATFORMS, PLATFORM_CHOICES, resolve as resolve_platform
-from _common import STATE, add_common, apply_common, default_font_file, die, emit, escape_drawtext, escape_filter_path, ffmpeg_base, info, parse_time, probe, run, time_arg
+from _common import STATE, add_common, apply_common, default_font_file, die, emit, escape_drawtext, escape_filter_path, ffmpeg_base, info, parse_time, probe, run, run_analysis, time_arg
FONT = "fontcolor=white:fontsize=h/18:box=1:[email protected]:boxborderw=6:x=8:y=8"
@@ -32,6 +32,103 @@ def timecode_filter(font_prefix: str) -> str:
return f"drawtext=text='%{{pts\\:hms}}':{font_prefix}{FONT}"
+def _png_size(png_path: str) -> "Optional[tuple]":
+ # run_analysis(): the same wall-clock ceiling and #234 UTF-8 decoding as every other
+ # ffprobe/ffmpeg measurement in this codebase, check=False since a probe failure here is
+ # an unknown-metric result (has_ink: None), not a reason to die -- a stall still does,
+ # exactly like every other measurement's timeout.
+ proc = run_analysis(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
+ "stream=width,height", "-of", "csv=p=0", png_path], check=False)
+ parts = proc.stdout.strip().split(",")
+ if len(parts) != 2:
+ return None
+ try:
+ return int(parts[0]), int(parts[1])
+ except ValueError:
+ return None
+
+
+def _png_pixels(png_path: str, w: int, h: int) -> "Optional[bytes]":
+ proc = run_analysis(["ffmpeg", "-v", "error", "-i", png_path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
+ check=False, text=False)
+ data = proc.stdout
+ return data if len(data) >= w * h * 3 else None
+
+
+def measure_ink(png_path: str, *, box=None, margin_frac: float = 0.03, threshold: int = 28) -> dict:
+ """Non-background ink in `png_path` (or the `box` = (x0, y0, w, h) sub-region of it): a pixel-only
+ measurement, never a judgement of legibility. The background is estimated as the median RGB of a
+ thin strip around the region's own edge (so it works on any frame, not just a black synthetic one);
+ a pixel is "ink" when any channel differs from that estimate by more than `threshold`. Returns
+ {has_ink, bbox (x0,y0,x1,y1 within the region, or None), ink_fraction, row_bands} -- row_bands are
+ gap-merged bands of rows carrying ink, the same convention tests/_fixtures.py's _ass_ink_rows uses,
+ so a caption wrapped onto two lines shows as two bands without measuring any font."""
+ size = _png_size(png_path)
+ if not size:
+ return {"has_ink": None, "bbox": None, "ink_fraction": None, "row_bands": []}
+ full_w, full_h = size
+ data = _png_pixels(png_path, full_w, full_h)
+ if data is None:
+ return {"has_ink": None, "bbox": None, "ink_fraction": None, "row_bands": []}
+ if box:
+ bx, by, bw, bh = box
+ bx, by = max(0, bx), max(0, by)
+ bw, bh = min(bw, full_w - bx), min(bh, full_h - by)
+ else:
+ bx, by, bw, bh = 0, 0, full_w, full_h
+ if bw <= 0 or bh <= 0:
+ return {"has_ink": None, "bbox": None, "ink_fraction": None, "row_bands": []}
+
+ def px(x, y):
+ off = ((by + y) * full_w + (bx + x)) * 3
+ return data[off], data[off + 1], data[off + 2]
+
+ margin = max(1, int(min(bw, bh) * margin_frac))
+ border = []
+ for x in range(0, bw, max(1, bw // 40 or 1)):
+ border.append(px(x, 0))
+ border.append(px(x, bh - 1))
+ for y in range(0, bh, max(1, bh // 40 or 1)):
+ border.append(px(0, y))
+ border.append(px(bw - 1, y))
+ if not border:
+ border = [px(0, 0)]
+ bg = tuple(sorted(c[i] for c in border)[len(border) // 2] for i in range(3))
+
+ lit_rows = []
+ min_x = min_y = None
+ max_x = max_y = None
+ ink_count = 0
+ for y in range(bh):
+ row_lit = False
+ for x in range(bw):
+ r, g, b = px(x, y)
+ if abs(r - bg[0]) > threshold or abs(g - bg[1]) > threshold or abs(b - bg[2]) > threshold:
+ row_lit = True
+ ink_count += 1
+ min_x = x if min_x is None else min(min_x, x)
+ max_x = x if max_x is None else max(max_x, x)
+ min_y = y if min_y is None else min(min_y, y)
+ max_y = y if max_y is None else max(max_y, y)
+ if row_lit:
+ lit_rows.append(y)
+
+ bands = []
+ for y in lit_rows:
+ if bands and y - bands[-1][1] <= margin:
+ bands[-1][1] = y
+ else:
+ bands.append([y, y])
+
+ has_ink = min_x is not None
+ return {
+ "has_ink": has_ink,
+ "bbox": [min_x, min_y, max_x, max_y] if has_ink else None,
+ "ink_fraction": round(ink_count / (bw * bh), 4) if bw * bh else 0.0,
+ "row_bands": [list(b) for b in bands],
+ }
+
+
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("input")
@@ -43,6 +140,9 @@ def main() -> int:
ap.add_argument("--no-timecode", action="store_true")
ap.add_argument("--safe", choices=PLATFORM_CHOICES, help="shade the zones this platform's UI covers (its description bar, "
"like column, status bar) so you can see whether anything readable is under them")
+ ap.add_argument("--ink", action="store_true", help="measure non-background pixels in each written PNG (bbox, "
+ "ink_fraction, row_bands per tile for a contact sheet) instead of "
+ "just eyeballing it -- a pixel-only signal, not a legibility judgement")
add_common(ap)
args = ap.parse_args()
apply_common(args)
@@ -103,9 +203,15 @@ def main() -> int:
if not args.at:
die("--compare needs --at TIME")
probe(args.compare)
- for t in args.at:
+ for idx, t in enumerate(args.at):
sec = time_arg(t, "--at", meta["video"].get("fps") if meta.get("video") else None)
- out = args.output or os.path.join(outdir, f"{stem}_vs_{Path(args.compare).stem}_{sec:.3f}s.png")
+ if args.output and len(args.at) == 1:
+ out = args.output # one frame, one named image file: the caller's -o is the contract
+ else:
+ # several frames, or -o given as a stem/prefix: an occurrence index keeps every
+ # requested --at its own file even when two round to the same millisecond
+ stem_part = (args.output and Path(args.output).stem) or stem
+ out = os.path.join(outdir, f"{stem_part}_vs_{Path(args.compare).stem}_{idx}_{sec:.3f}s.png")
half = args.width // 2
stamp = "" if args.no_timecode else f",drawtext=text='{escape_drawtext(fmt_hms(sec))}':{font_prefix}{FONT}"
tcs = tc.replace("," + timecode_filter(font_prefix), "") + stamp
@@ -159,7 +265,29 @@ def main() -> int:
for o in outputs:
if STATE.dry_run or os.path.exists(o):
info(f"wrote {o}")
- emit(outputs[0] if len(outputs) == 1 else None, outputs=outputs)
+
+ ink = None
+ if args.ink and not STATE.dry_run:
+ if outputs and args.at is None and not args.compare:
+ # contact sheet: one PNG holding a cols x rows grid (tile_w:padding=2:margin=2, as built above)
+ # -- split it back into per-tile boxes so each tile is measured on its own, not the whole sheet.
+ sheet = outputs[0]
+ size = _png_size(sheet)
+ if size:
+ full_w, full_h = size
+ margin, padding = 2, 2
+ th = (full_h - 2 * margin - (rows - 1) * padding) // rows if rows else full_h
+ tiles = []
+ for r in range(rows):
+ for c in range(cols):
+ bx = margin + c * (tile_w + padding)
+ by = margin + r * (th + padding)
+ tiles.append(measure_ink(sheet, box=(bx, by, tile_w, th)))
+ ink = {"tiles": tiles}
+ else:
+ ink = {"frames": [measure_ink(o) for o in outputs]}
+
+ emit(outputs[0] if len(outputs) == 1 else None, outputs=outputs, **({"ink": ink} if ink is not None else {}))
if len(outputs) > 1 and not args.json:
for o in outputs:
print(o)
diff --git a/tests/fixtures/mcp_tools.json b/tests/fixtures/mcp_tools.json
index 9acf8d0..ec3b5ec 100644
--- a/tests/fixtures/mcp_tools.json
+++ b/tests/fixtures/mcp_tools.json
@@ -637,6 +637,7 @@
"compare",
"dry_run",
"fast",
+ "ink",
"input",
"json",
"json_brief",
diff --git a/tests/test_picture.py b/tests/test_picture.py
index 23f4cc1..e7148c0 100644
--- a/tests/test_picture.py
+++ b/tests/test_picture.py
@@ -2383,6 +2383,102 @@ class EmojiTests(unittest.TestCase):
self.assertIn("A \\{b\\} c \\ d", dialogue)
+class LookInkTests(unittest.TestCase):
+ """look.py --ink: a pixel-only non-background measurement per written PNG, so the agent can
+ check "is there ink where the caption should be" without eyeballing every frame."""
+
+ @classmethod
+ def setUpClass(cls):
+ if not shutil.which("ffmpeg"):
+ if os.environ.get("CI"):
+ raise AssertionError("ffmpeg not on PATH -- in CI this is a broken install step")
+ raise unittest.SkipTest("ffmpeg not on PATH")
+ OUT.mkdir(parents=True, exist_ok=True)
+ cls.busy = OUT / "ink_busy.mp4"
+ if not cls.busy.exists():
+ sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi",
+ "-i", "testsrc2=size=320x240:rate=10", "-t", "3", "-pix_fmt", "yuv420p", cls.busy)
+ cls.blank = OUT / "ink_blank.mp4"
+ if not cls.blank.exists():
+ sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi",
+ "-i", "color=c=black:size=320x240:rate=10", "-t", "3", "-pix_fmt", "yuv420p", cls.blank)
+
+ def test_ink_reports_a_lit_frame_via_at(self):
+ out = OUT / "ink_at.png"
+ proc = script("look.py", self.busy, "--at", "1", "--no-timecode", "--ink", "--json", "-o", out)
+ doc = json.loads(proc.stdout)
+ frame = doc["ink"]["frames"][0]
+ self.assertTrue(frame["has_ink"])
+ self.assertIsNotNone(frame["bbox"])
+ self.assertGreater(frame["ink_fraction"], 0.0)
+ self.assertTrue(frame["row_bands"])
+
+ def test_ink_reports_no_ink_on_a_blank_frame(self):
+ out = OUT / "ink_blank_at.png"
+ proc = script("look.py", self.blank, "--at", "1", "--no-timecode", "--ink", "--json", "-o", out)
+ doc = json.loads(proc.stdout)
+ frame = doc["ink"]["frames"][0]
+ self.assertFalse(frame["has_ink"])
+ self.assertIsNone(frame["bbox"])
+ self.assertEqual(frame["ink_fraction"], 0.0)
+ self.assertEqual(frame["row_bands"], [])
+
+ def test_ink_is_omitted_without_the_flag(self):
+ out = OUT / "ink_off.png"
+ proc = script("look.py", self.busy, "--at", "1", "--no-timecode", "--json", "-o", out)
+ doc = json.loads(proc.stdout)
+ self.assertNotIn("ink", doc)
+
+ def test_ink_measures_each_tile_of_a_contact_sheet(self):
+ out = OUT / "ink_sheet.png"
+ proc = script("look.py", self.busy, "--tiles", "2x2", "--ink", "--json", "-o", out)
+ doc = json.loads(proc.stdout)
+ tiles = doc["ink"]["tiles"]
+ self.assertEqual(len(tiles), 4)
+ for tile in tiles:
+ self.assertTrue(tile["has_ink"])
+
+ def test_ink_measures_both_sides_of_a_compare(self):
+ out = OUT / "ink_compare.png"
+ proc = script("look.py", self.busy, "--compare", self.blank, "--at", "1", "--ink", "--json", "-o", out)
+ doc = json.loads(proc.stdout)
+ frames = doc["ink"]["frames"]
+ self.assertEqual(len(frames), 1)
+ self.assertTrue(frames[0]["has_ink"]) # the hstacked image: busy half lights it up
+
+ def test_compare_with_repeated_at_and_o_writes_distinct_files(self):
+ """-o given alongside --compare --at T1 --at T2 must not collapse onto one file: each
+ timestamp needs its own comparison image, or ink.frames (and any other per-frame
+ measurement) silently describes the same picture twice."""
+ out = OUT / "ink_compare_multi.png"
+ proc = script("look.py", self.busy, "--compare", self.blank, "--at", "1", "--at", "2",
+ "--ink", "--json", "-o", out)
+ doc = json.loads(proc.stdout)
+ outputs = doc["outputs"]
+ self.assertEqual(len(outputs), 2)
+ self.assertNotEqual(outputs[0], outputs[1])
+ for o in outputs:
+ self.assertTrue(Path(o).exists(), o)
+ frames = doc["ink"]["frames"]
+ self.assertEqual(len(frames), 2)
+ self.assertTrue(frames[0]["has_ink"])
+ self.assertTrue(frames[1]["has_ink"])
+
+ def test_compare_with_the_same_at_value_twice_still_writes_two_files(self):
+ """The occurrence index, not just the timestamp, disambiguates the filename: two
+ identical --at values (or two that would round to the same millisecond) must not
+ collapse onto one file either."""
+ out = OUT / "ink_compare_dup.png"
+ proc = script("look.py", self.busy, "--compare", self.blank, "--at", "1", "--at", "1",
+ "--json", "-o", out)
+ doc = json.loads(proc.stdout)
+ outputs = doc["outputs"]
+ self.assertEqual(len(outputs), 2)
+ self.assertNotEqual(outputs[0], outputs[1])
+ for o in outputs:
+ self.assertTrue(Path(o).exists(), o)
+
+
class GraphicsSliceOverlongTests(unittest.TestCase):
"""graphics.py's own `wrapped()` helper (used by lower-third, title, sticker, hook and meme
labels) now turns on the same `slice_overlong` escape hatch caption.py's burn path turned on