Local video editor for coding agents: 42 FFmpeg tools with a machine-readable contract and an MCP server, for Claude Code, Cursor and Codex. Cut, captions, reframe, loudness, platform exports. No cloud, no API keys.
Python agent-skillsai-agentsaudio-processingcaptionsclaude-codecodexcoding-agentcursorffmpegloudness-normalizationmcpmcp-servermodel-context-protocolpythonsubtitlestranscodingvideovideo-editingvideo-processingwhisper
feat: look.py --ink measures non-background pixels per PNG (#261)
Shogo Kajiyama7ba483fbdcf0
diff --git a/docs/roadmap.md b/docs/roadmap.mdindex 94d0adb..964d955 100644--- a/docs/roadmap.md+++ b/docs/roadmap.md@@ -17,7 +17,7 @@ minor and `fix` PRs into a patch, so each block below is one or two `feat` PRs p- **planned** — not released. Nothing below a *planned* heading exists in any published version;the feature lines are the intent, not a description of the code.-The released version today is **1.19.2** — `multicam.py --write-project FILE` writes a+The released version today is **1.19.3** — `multicam.py --write-project FILE` writes a`render.py` project that reproduces the tool's own cut decision (eval 21's `mc2` gap, no morehand-translating cuts/offsets into `clips[]`), and `--filler` alone no longer jump-cuts unrelateddead-air silence (eval 19's `fw1`/`fw3`: "remove the ums" silently removed ~5s of unasked-for@@ -25,8 +25,9 @@ pauses too; fixed by skipping generic silence detection unless `--speech-aware`1.19.0 (the `--write-project` feature) is a `feat:` release, hence the minor bump; it does notstart the "1.19.0: observability, portability" theme further down this document, which remainsplanned. 1.19.2 is docs-only (SKILL.md's overlong-word and MCP core-12 rows, this same truth-up).-1.18.4 (the caption column-edge slice, `cs2`) is unchanged from before; see its own-section below.+1.19.3 extends 1.18.4's column-edge slice to `graphics.py`'s own `wrapped()` helper (lower-third,+title, sticker, hook and meme labels), so an unbreakable overlong word no longer clips off-frame+on those templates either.| version | state | evidence ||---|---|---|@@ -54,6 +55,7 @@ section below.| 1.19.0 | shipped, eval pending | `multicam.py --write-project FILE` (eval 21's `mc2` gap): writes a `render.py` project whose `clips[]` reproduces multicam's own cut decision exactly (`src`/`in`/`out` per cut, offset-shifted to each camera's own timeline), for `--switch energy`, a manual `--switch` spec, or `--auto` alike; multicam's own combined render and `--edl`/`--offsets-only` unchanged. `feat:` release, hence the minor bump — not the start of the "1.19.0" theme further down this document, which is unrelated and still planned || 1.19.1 | shipped, eval pending | eval 19's `fw1`/`fw3`: `silence.py --filler` alone (no `--speech-aware`) jump-cut unrelated dead-air silence gaps too, ~5s unasked. Fixed: generic silence detection is skipped unless `--speech-aware` is also given, so `--filler` alone removes only the timed filler-word spans; `--speech-aware` alone and `--filler --speech-aware` combined are unchanged || 1.19.2 | shipped, docs-only | SKILL.md rows for 1.18.3's MCP core-12 (previously undocumented) and 1.18.4's overlong-word column-edge slice, plus this document's own roadmap truth-up. No script changes |+| 1.19.3 | shipped, eval pending | `graphics.py`'s `wrapped()` helper (used by the lower-third, title, sticker, hook and meme templates) turns on the same `slice_overlong` escape hatch `caption.py` turned on in 1.18.4, so a single unbreakable overlong atom drawn through a template no longer renders past the frame's safe width. `broken_inside_word` reported the same way as `caption.py`. A fitting Thai phrase or katakana run is unchanged, guarded by the same font-availability skip as `ShapingTests`. Tool count still 42, no new CLI flag for this change specifically -- `look.py --ink` is a separate, not-yet-versioned addition tracked in its own PR || 1.21.0, 2.0.0 | planned | — |## 1.8.0 — one-call delivery, quieter checks, encoder flags (shipped + evaluated, eval 8)diff --git a/scripts/look.py b/scripts/look.pyindex 5029610..0604daa 100755--- a/scripts/look.py+++ b/scripts/look.py@@ -14,10 +14,10 @@ import argparseimport osimport sysfrom pathlib import Path-from typing import List+from typing import List, Optionalfrom _platforms import PLATFORMS, PLATFORM_CHOICES, resolve as resolve_platform-from _common import STATE, add_common, apply_common, default_font_file, die, emit, escape_drawtext, escape_filter_path, ffmpeg_base, info, parse_time, probe, run, time_arg+from _common import STATE, add_common, apply_common, default_font_file, die, emit, escape_drawtext, escape_filter_path, ffmpeg_base, info, parse_time, probe, run, run_analysis, time_argFONT = "fontcolor=white:fontsize=h/18:box=1:[email protected]:boxborderw=6:x=8:y=8"@@ -32,6 +32,103 @@ def timecode_filter(font_prefix: str) -> str:return f"drawtext=text='%{{pts\\:hms}}':{font_prefix}{FONT}"+def _png_size(png_path: str) -> "Optional[tuple]":+ # run_analysis(): the same wall-clock ceiling and #234 UTF-8 decoding as every other+ # ffprobe/ffmpeg measurement in this codebase, check=False since a probe failure here is+ # an unknown-metric result (has_ink: None), not a reason to die -- a stall still does,+ # exactly like every other measurement's timeout.+ proc = run_analysis(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",+ "stream=width,height", "-of", "csv=p=0", png_path], check=False)+ parts = proc.stdout.strip().split(",")+ if len(parts) != 2:+ return None+ try:+ return int(parts[0]), int(parts[1])+ except ValueError:+ return None+++def _png_pixels(png_path: str, w: int, h: int) -> "Optional[bytes]":+ proc = run_analysis(["ffmpeg", "-v", "error", "-i", png_path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],+ check=False, text=False)+ data = proc.stdout+ return data if len(data) >= w * h * 3 else None+++def measure_ink(png_path: str, *, box=None, margin_frac: float = 0.03, threshold: int = 28) -> dict:+ """Non-background ink in `png_path` (or the `box` = (x0, y0, w, h) sub-region of it): a pixel-only+ measurement, never a judgement of legibility. The background is estimated as the median RGB of a+ thin strip around the region's own edge (so it works on any frame, not just a black synthetic one);+ a pixel is "ink" when any channel differs from that estimate by more than `threshold`. Returns+ {has_ink, bbox (x0,y0,x1,y1 within the region, or None), ink_fraction, row_bands} -- row_bands are+ gap-merged bands of rows carrying ink, the same convention tests/_fixtures.py's _ass_ink_rows uses,+ so a caption wrapped onto two lines shows as two bands without measuring any font."""+ size = _png_size(png_path)+ if not size:+ return {"has_ink": None, "bbox": None, "ink_fraction": None, "row_bands": []}+ full_w, full_h = size+ data = _png_pixels(png_path, full_w, full_h)+ if data is None:+ return {"has_ink": None, "bbox": None, "ink_fraction": None, "row_bands": []}+ if box:+ bx, by, bw, bh = box+ bx, by = max(0, bx), max(0, by)+ bw, bh = min(bw, full_w - bx), min(bh, full_h - by)+ else:+ bx, by, bw, bh = 0, 0, full_w, full_h+ if bw <= 0 or bh <= 0:+ return {"has_ink": None, "bbox": None, "ink_fraction": None, "row_bands": []}++ def px(x, y):+ off = ((by + y) * full_w + (bx + x)) * 3+ return data[off], data[off + 1], data[off + 2]++ margin = max(1, int(min(bw, bh) * margin_frac))+ border = []+ for x in range(0, bw, max(1, bw // 40 or 1)):+ border.append(px(x, 0))+ border.append(px(x, bh - 1))+ for y in range(0, bh, max(1, bh // 40 or 1)):+ border.append(px(0, y))+ border.append(px(bw - 1, y))+ if not border:+ border = [px(0, 0)]+ bg = tuple(sorted(c[i] for c in border)[len(border) // 2] for i in range(3))++ lit_rows = []+ min_x = min_y = None+ max_x = max_y = None+ ink_count = 0+ for y in range(bh):+ row_lit = False+ for x in range(bw):+ r, g, b = px(x, y)+ if abs(r - bg[0]) > threshold or abs(g - bg[1]) > threshold or abs(b - bg[2]) > threshold:+ row_lit = True+ ink_count += 1+ min_x = x if min_x is None else min(min_x, x)+ max_x = x if max_x is None else max(max_x, x)+ min_y = y if min_y is None else min(min_y, y)+ max_y = y if max_y is None else max(max_y, y)+ if row_lit:+ lit_rows.append(y)++ bands = []+ for y in lit_rows:+ if bands and y - bands[-1][1] <= margin:+ bands[-1][1] = y+ else:+ bands.append([y, y])++ has_ink = min_x is not None+ return {+ "has_ink": has_ink,+ "bbox": [min_x, min_y, max_x, max_y] if has_ink else None,+ "ink_fraction": round(ink_count / (bw * bh), 4) if bw * bh else 0.0,+ "row_bands": [list(b) for b in bands],+ }++def main() -> int:ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)ap.add_argument("input")@@ -43,6 +140,9 @@ def main() -> int:ap.add_argument("--no-timecode", action="store_true")ap.add_argument("--safe", choices=PLATFORM_CHOICES, help="shade the zones this platform's UI covers (its description bar, ""like column, status bar) so you can see whether anything readable is under them")+ ap.add_argument("--ink", action="store_true", help="measure non-background pixels in each written PNG (bbox, "+ "ink_fraction, row_bands per tile for a contact sheet) instead of "+ "just eyeballing it -- a pixel-only signal, not a legibility judgement")add_common(ap)args = ap.parse_args()apply_common(args)@@ -103,9 +203,15 @@ def main() -> int:if not args.at:die("--compare needs --at TIME")probe(args.compare)- for t in args.at:+ for idx, t in enumerate(args.at):sec = time_arg(t, "--at", meta["video"].get("fps") if meta.get("video") else None)- out = args.output or os.path.join(outdir, f"{stem}_vs_{Path(args.compare).stem}_{sec:.3f}s.png")+ if args.output and len(args.at) == 1:+ out = args.output # one frame, one named image file: the caller's -o is the contract+ else:+ # several frames, or -o given as a stem/prefix: an occurrence index keeps every+ # requested --at its own file even when two round to the same millisecond+ stem_part = (args.output and Path(args.output).stem) or stem+ out = os.path.join(outdir, f"{stem_part}_vs_{Path(args.compare).stem}_{idx}_{sec:.3f}s.png")half = args.width // 2stamp = "" if args.no_timecode else f",drawtext=text='{escape_drawtext(fmt_hms(sec))}':{font_prefix}{FONT}"tcs = tc.replace("," + timecode_filter(font_prefix), "") + stamp@@ -159,7 +265,29 @@ def main() -> int:for o in outputs:if STATE.dry_run or os.path.exists(o):info(f"wrote {o}")- emit(outputs[0] if len(outputs) == 1 else None, outputs=outputs)++ ink = None+ if args.ink and not STATE.dry_run:+ if outputs and args.at is None and not args.compare:+ # contact sheet: one PNG holding a cols x rows grid (tile_w:padding=2:margin=2, as built above)+ # -- split it back into per-tile boxes so each tile is measured on its own, not the whole sheet.+ sheet = outputs[0]+ size = _png_size(sheet)+ if size:+ full_w, full_h = size+ margin, padding = 2, 2+ th = (full_h - 2 * margin - (rows - 1) * padding) // rows if rows else full_h+ tiles = []+ for r in range(rows):+ for c in range(cols):+ bx = margin + c * (tile_w + padding)+ by = margin + r * (th + padding)+ tiles.append(measure_ink(sheet, box=(bx, by, tile_w, th)))+ ink = {"tiles": tiles}+ else:+ ink = {"frames": [measure_ink(o) for o in outputs]}++ emit(outputs[0] if len(outputs) == 1 else None, outputs=outputs, **({"ink": ink} if ink is not None else {}))if len(outputs) > 1 and not args.json:for o in outputs:print(o)diff --git a/tests/fixtures/mcp_tools.json b/tests/fixtures/mcp_tools.jsonindex 9acf8d0..ec3b5ec 100644--- a/tests/fixtures/mcp_tools.json+++ b/tests/fixtures/mcp_tools.json@@ -637,6 +637,7 @@"compare","dry_run","fast",+ "ink","input","json","json_brief",diff --git a/tests/test_picture.py b/tests/test_picture.pyindex 23f4cc1..e7148c0 100644--- a/tests/test_picture.py+++ b/tests/test_picture.py@@ -2383,6 +2383,102 @@ class EmojiTests(unittest.TestCase):self.assertIn("A \\{b\\} c \\ d", dialogue)+class LookInkTests(unittest.TestCase):+ """look.py --ink: a pixel-only non-background measurement per written PNG, so the agent can+ check "is there ink where the caption should be" without eyeballing every frame."""++ @classmethod+ def setUpClass(cls):+ if not shutil.which("ffmpeg"):+ if os.environ.get("CI"):+ raise AssertionError("ffmpeg not on PATH -- in CI this is a broken install step")+ raise unittest.SkipTest("ffmpeg not on PATH")+ OUT.mkdir(parents=True, exist_ok=True)+ cls.busy = OUT / "ink_busy.mp4"+ if not cls.busy.exists():+ sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi",+ "-i", "testsrc2=size=320x240:rate=10", "-t", "3", "-pix_fmt", "yuv420p", cls.busy)+ cls.blank = OUT / "ink_blank.mp4"+ if not cls.blank.exists():+ sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi",+ "-i", "color=c=black:size=320x240:rate=10", "-t", "3", "-pix_fmt", "yuv420p", cls.blank)++ def test_ink_reports_a_lit_frame_via_at(self):+ out = OUT / "ink_at.png"+ proc = script("look.py", self.busy, "--at", "1", "--no-timecode", "--ink", "--json", "-o", out)+ doc = json.loads(proc.stdout)+ frame = doc["ink"]["frames"][0]+ self.assertTrue(frame["has_ink"])+ self.assertIsNotNone(frame["bbox"])+ self.assertGreater(frame["ink_fraction"], 0.0)+ self.assertTrue(frame["row_bands"])++ def test_ink_reports_no_ink_on_a_blank_frame(self):+ out = OUT / "ink_blank_at.png"+ proc = script("look.py", self.blank, "--at", "1", "--no-timecode", "--ink", "--json", "-o", out)+ doc = json.loads(proc.stdout)+ frame = doc["ink"]["frames"][0]+ self.assertFalse(frame["has_ink"])+ self.assertIsNone(frame["bbox"])+ self.assertEqual(frame["ink_fraction"], 0.0)+ self.assertEqual(frame["row_bands"], [])++ def test_ink_is_omitted_without_the_flag(self):+ out = OUT / "ink_off.png"+ proc = script("look.py", self.busy, "--at", "1", "--no-timecode", "--json", "-o", out)+ doc = json.loads(proc.stdout)+ self.assertNotIn("ink", doc)++ def test_ink_measures_each_tile_of_a_contact_sheet(self):+ out = OUT / "ink_sheet.png"+ proc = script("look.py", self.busy, "--tiles", "2x2", "--ink", "--json", "-o", out)+ doc = json.loads(proc.stdout)+ tiles = doc["ink"]["tiles"]+ self.assertEqual(len(tiles), 4)+ for tile in tiles:+ self.assertTrue(tile["has_ink"])++ def test_ink_measures_both_sides_of_a_compare(self):+ out = OUT / "ink_compare.png"+ proc = script("look.py", self.busy, "--compare", self.blank, "--at", "1", "--ink", "--json", "-o", out)+ doc = json.loads(proc.stdout)+ frames = doc["ink"]["frames"]+ self.assertEqual(len(frames), 1)+ self.assertTrue(frames[0]["has_ink"]) # the hstacked image: busy half lights it up++ def test_compare_with_repeated_at_and_o_writes_distinct_files(self):+ """-o given alongside --compare --at T1 --at T2 must not collapse onto one file: each+ timestamp needs its own comparison image, or ink.frames (and any other per-frame+ measurement) silently describes the same picture twice."""+ out = OUT / "ink_compare_multi.png"+ proc = script("look.py", self.busy, "--compare", self.blank, "--at", "1", "--at", "2",+ "--ink", "--json", "-o", out)+ doc = json.loads(proc.stdout)+ outputs = doc["outputs"]+ self.assertEqual(len(outputs), 2)+ self.assertNotEqual(outputs[0], outputs[1])+ for o in outputs:+ self.assertTrue(Path(o).exists(), o)+ frames = doc["ink"]["frames"]+ self.assertEqual(len(frames), 2)+ self.assertTrue(frames[0]["has_ink"])+ self.assertTrue(frames[1]["has_ink"])++ def test_compare_with_the_same_at_value_twice_still_writes_two_files(self):+ """The occurrence index, not just the timestamp, disambiguates the filename: two+ identical --at values (or two that would round to the same millisecond) must not+ collapse onto one file either."""+ out = OUT / "ink_compare_dup.png"+ proc = script("look.py", self.busy, "--compare", self.blank, "--at", "1", "--at", "1",+ "--json", "-o", out)+ doc = json.loads(proc.stdout)+ outputs = doc["outputs"]+ self.assertEqual(len(outputs), 2)+ self.assertNotEqual(outputs[0], outputs[1])+ for o in outputs:+ self.assertTrue(Path(o).exists(), o)++class GraphicsSliceOverlongTests(unittest.TestCase):"""graphics.py's own `wrapped()` helper (used by lower-third, title, sticker, hook and memelabels) now turns on the same `slice_overlong` escape hatch caption.py's burn path turned on