Skip to content

Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows

demos: frame-range OCR validation and build script improvements (#1778)

Maximilian Roos38307dfad458
diff --git a/docs/demos/build b/docs/demos/build
index de728bbb5..efec63387 100755
--- a/docs/demos/build
+++ b/docs/demos/build
@@ -19,6 +19,7 @@ import shutil
import subprocess
import sys
import tempfile
+import time
from concurrent.futures import ProcessPoolExecutor, as_completed
from functools import partial
from pathlib import Path
@@ -348,8 +349,9 @@ def record_demo(
vhs_binary: str,
text_mode: bool = False,
snapshot_mode: bool = False,
-):
- """Record a single demo (runs in subprocess for parallel execution)."""
+) -> float:
+ """Record a single demo. Returns elapsed seconds."""
+ t0 = time.monotonic()
tape_path = TAPES_DIR / tape_file
if not tape_path.exists():
raise FileNotFoundError(f"{tape_file} not found")
@@ -365,43 +367,51 @@ def record_demo(
if snapshot_mode:
output_snap = SNAPSHOTS_DIR / f"{output_name}.snap"
record_snapshot(env, tape_path, output_snap, REPO_ROOT)
- print(f"✓ {output_name}: Snapshot saved to {output_snap}")
- elif text_mode:
+ elapsed = time.monotonic() - t0
+ print(f"✓ {output_name}: Snapshot saved ({elapsed:.0f}s)")
+ return elapsed
+
+ if text_mode:
output_txt = mode_out_dir / f"{output_name}.txt"
record_text(env, tape_path, output_txt, replacements, REPO_ROOT)
- print(f"✓ {output_name}: Text saved to {output_txt}")
- else:
- output_gifs = {}
- for theme in themes:
- theme_dir = mode_out_dir / theme
- theme_dir.mkdir(parents=True, exist_ok=True)
- output_gifs[theme] = theme_dir / f"{output_name}.gif"
-
- record_all_themes(
- env,
- tape_path,
- output_gifs,
- REPO_ROOT,
- vhs_binary=vhs_binary,
- size=demo_size,
- )
+ elapsed = time.monotonic() - t0
+ print(f"✓ {output_name}: Text saved ({elapsed:.0f}s)")
+ return elapsed
+
+ output_gifs = {}
+ for theme in themes:
+ theme_dir = mode_out_dir / theme
+ theme_dir.mkdir(parents=True, exist_ok=True)
+ output_gifs[theme] = theme_dir / f"{output_name}.gif"
+
+ record_all_themes(
+ env,
+ tape_path,
+ output_gifs,
+ REPO_ROOT,
+ vhs_binary=vhs_binary,
+ size=demo_size,
+ )
+
+ elapsed = time.monotonic() - t0
- # Validate TUI demos with OCR after recording
- if output_name in TUI_CHECKPOINTS:
- # Use the light theme GIF for validation
- gif_path = output_gifs.get("light", list(output_gifs.values())[0])
- success, output = validate_tui_demo_verbose(output_name, gif_path)
- if success:
- print(f"✓ {output_name}: GIFs saved + validation passed")
- else:
- print(f"✓ {output_name}: GIFs saved")
- print(f"✗ {output_name}: Validation failed")
- for line in output.split("\n"):
- if line.startswith(" "):
- print(line)
- raise RuntimeError(f"TUI validation failed for {output_name}")
+ # Validate TUI demos with OCR after recording
+ if output_name in TUI_CHECKPOINTS:
+ gif_path = output_gifs.get("light", list(output_gifs.values())[0])
+ success, output = validate_tui_demo_verbose(output_name, gif_path)
+ if success:
+ print(f"✓ {output_name}: GIFs saved + validation passed ({elapsed:.0f}s)")
else:
- print(f"✓ {output_name}: GIFs saved: {', '.join(themes)}")
+ print(f"✓ {output_name}: GIFs saved ({elapsed:.0f}s)")
+ print(f"✗ {output_name}: Validation failed")
+ for line in output.split("\n"):
+ if line.startswith(" "):
+ print(line)
+ raise RuntimeError(f"TUI validation failed for {output_name}")
+ else:
+ print(f"✓ {output_name}: GIFs saved: {', '.join(themes)} ({elapsed:.0f}s)")
+
+ return elapsed
# =============================================================================
@@ -508,6 +518,7 @@ Examples:
vhs_binary = str(ensure_vhs_binary())
# Build each target sequentially (parallel targets cause Zellij conflicts)
+ t0 = time.monotonic()
all_failed = []
for target in dict.fromkeys(args.target): # deduplicate, preserve order
failed = build_target(
@@ -515,14 +526,23 @@ Examples:
)
all_failed.extend(f"{target}/{name}" for name in failed)
- if all_failed:
+ # Overall summary for multi-target builds
+ if len(args.target) > 1:
+ elapsed = time.monotonic() - t0
+ mins, secs = divmod(int(elapsed), 60)
print(f"\n{'=' * 60}")
+ print(f"Total: {mins}m{secs:02d}s")
+
+ if all_failed:
+ if len(args.target) <= 1:
+ print(f"\n{'=' * 60}")
print(f"Failed: {', '.join(all_failed)}")
sys.exit(1)
def build_target(target: str, args, vhs_binary: str) -> list[str]:
"""Build all demos for a single target. Returns list of failed demo names."""
+ t0 = time.monotonic()
mode_out_dir = OUT_DIR / target
mode_out_dir.mkdir(parents=True, exist_ok=True)
@@ -565,54 +585,82 @@ def build_target(target: str, args, vhs_binary: str) -> list[str]:
spawn_debug_shell(env, FIXTURES_DIR / "starship.toml")
return []
- # Record demos
+ # TUI demos run sequentially (they use Zellij which conflicts when parallel)
+ tui_demos = [(t, n, s) for t, n, s in demos if n in TUI_DEMOS]
+ non_tui_demos = [(t, n, s) for t, n, s in demos if n not in TUI_DEMOS]
+
failed = []
- if args.sequential or len(demos) == 1:
- print(f"\n[{target}] Recording {len(demos)} demo(s) sequentially...")
- for tape_file, output_name, setup_func in demos:
- try:
- record_demo(
- tape_file,
- output_name,
- setup_func,
- mode_out_dir,
- themes,
- demo_size,
- vhs_binary,
- args.text,
- args.snapshot,
- )
- except Exception as e:
- print(f"✗ {output_name}: {e}")
- failed.append(output_name)
- else:
- print(f"\n[{target}] Recording {len(demos)} demo(s) in parallel...")
- max_workers = min(len(demos), os.cpu_count() or 4)
-
- with ProcessPoolExecutor(max_workers=max_workers) as executor:
- futures = {
- executor.submit(
- record_demo,
- tape_file,
- output_name,
- setup_func,
- mode_out_dir,
- themes,
- demo_size,
- vhs_binary,
- args.text,
- args.snapshot,
- ): output_name
- for tape_file, output_name, setup_func in demos
- }
-
- for future in as_completed(futures):
- output_name = futures[future]
- try:
- future.result()
- except Exception as e:
- print(f"✗ {output_name}: {e}")
- failed.append(output_name)
+ succeeded = []
+
+ def _record_one(tape_file, output_name, setup_func):
+ try:
+ elapsed = record_demo(
+ tape_file,
+ output_name,
+ setup_func,
+ mode_out_dir,
+ themes,
+ demo_size,
+ vhs_binary,
+ args.text,
+ args.snapshot,
+ )
+ succeeded.append((output_name, elapsed))
+ except Exception as e:
+ print(f"✗ {output_name}: {e}")
+ failed.append(output_name)
+
+ # Record non-TUI demos (parallel unless --sequential)
+ if non_tui_demos:
+ if args.sequential or len(non_tui_demos) == 1:
+ print(f"\n[{target}] Recording {len(non_tui_demos)} demo(s) sequentially...")
+ for tape_file, output_name, setup_func in non_tui_demos:
+ _record_one(tape_file, output_name, setup_func)
+ else:
+ print(f"\n[{target}] Recording {len(non_tui_demos)} demo(s) in parallel...")
+ max_workers = min(len(non_tui_demos), os.cpu_count() or 4)
+
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
+ futures = {
+ executor.submit(
+ record_demo,
+ tape_file,
+ output_name,
+ setup_func,
+ mode_out_dir,
+ themes,
+ demo_size,
+ vhs_binary,
+ args.text,
+ args.snapshot,
+ ): output_name
+ for tape_file, output_name, setup_func in non_tui_demos
+ }
+
+ for future in as_completed(futures):
+ output_name = futures[future]
+ try:
+ elapsed = future.result()
+ succeeded.append((output_name, elapsed))
+ except Exception as e:
+ print(f"✗ {output_name}: {e}")
+ failed.append(output_name)
+
+ # Record TUI demos sequentially (Zellij conflicts when parallel)
+ if tui_demos:
+ print(f"\n[{target}] Recording {len(tui_demos)} TUI demo(s) sequentially...")
+ for tape_file, output_name, setup_func in tui_demos:
+ _record_one(tape_file, output_name, setup_func)
+
+ # Summary
+ elapsed_total = time.monotonic() - t0
+ mins, secs = divmod(int(elapsed_total), 60)
+ passed_str = ", ".join(f"{n} ({e:.0f}s)" for n, e in succeeded)
+ print(f"\n[{target}] {len(succeeded)} passed, {len(failed)} failed ({mins}m{secs:02d}s)")
+ if succeeded:
+ print(f" Passed: {passed_str}")
+ if failed:
+ print(f" Failed: {', '.join(failed)}")
return failed
diff --git a/docs/demos/shared/validation.py b/docs/demos/shared/validation.py
index 554e91a71..5077d492f 100644
--- a/docs/demos/shared/validation.py
+++ b/docs/demos/shared/validation.py
@@ -2,9 +2,15 @@
TUI demos (Zellij, interactive UIs) can't be validated via text output because
VHS only captures the outer terminal, not content rendered inside terminal
-multiplexers. Instead, we extract key frames from the GIF and use OCR to verify
+multiplexers. Instead, we extract frames from the GIF and use OCR to verify
expected content appears.
+Checkpoints specify a frame range rather than a single frame. The validator
+scans frames within the range (sampling every N frames) and passes the
+checkpoint if ANY frame in the range matches all expected patterns while
+containing none of the forbidden patterns. This makes validation resilient
+to timing shifts from UI changes.
+
Usage:
from shared.validation import validate_tui_demo, TUI_CHECKPOINTS
@@ -18,24 +24,47 @@ from __future__ import annotations
import subprocess
import tempfile
+from dataclasses import dataclass, field
from pathlib import Path
-# Checkpoint definitions per TUI demo
-# Format: {demo_name: [(frame_number, expected_patterns, forbidden_patterns), ...]}
-#
-# Frame numbers are calibrated from actual GIF content at 30fps.
-# Expected patterns must ALL be present (case-insensitive).
-# Forbidden patterns must ALL be absent (case-insensitive).
-TUI_CHECKPOINTS: dict[str, list[tuple[int, list[str], list[str]]]] = {
- # Frame numbers calibrated after removing initial wt list (demo starts with picker).
- # At 30fps: frame 200 = ~6.7s, frame 1750 = ~58.3s
+@dataclass
+class Checkpoint:
+ """A validation checkpoint that scans a range of frames."""
+
+ start: int
+ end: int
+ expected: list[str] = field(default_factory=list)
+ forbidden: list[str] = field(default_factory=list)
+ step: int = 10
+
+
+# Checkpoint definitions per TUI demo.
+# Ranges are calibrated from actual GIF content at 30fps.
+# Expected patterns must ALL be present (case-insensitive) in at least one
+# frame within the range. Forbidden patterns must ALL be absent.
+
+TUI_CHECKPOINTS: dict[str, list[Checkpoint]] = {
"wt-zellij-omnibus": [
- # Frame 200: Claude UI visible on TAB 1 (api) - shows Opus model and task
- (200, ["Opus", "acme", "Add a test"], ["command not found", "Unknown command"]),
- # Frame 1750: Near end - wt list --full showing all worktrees
- # (feature removed by wt remove in TAB 3)
- (1750, ["Branch", "main", "billing"], ["CONFLICT", "error:", "failed"]),
+ # Claude UI visible on TAB 1 (api) — shows model name and task.
+ # Range covers the window where Claude's UI is rendered and stable.
+ # Patterns kept minimal (just "Opus" + "acme") since Claude's UI
+ # layout shifts across versions — task text may wrap or truncate.
+ Checkpoint(
+ start=150,
+ end=350,
+ expected=["Opus", "acme"],
+ forbidden=["command not found", "Unknown command"],
+ ),
+ # Near end — wt list --full showing all worktrees.
+ # "billing" omitted: depends on timing of when the branch appears
+ # in the list relative to the frame window.
+ Checkpoint(
+ start=1650,
+ end=1850,
+ expected=["Branch", "main"],
+ forbidden=["CONFLICT", "error:", "failed"],
+ ),
],
}
@@ -87,50 +116,71 @@ def ocr_image(image_path: Path) -> str:
return ""
-def validate_checkpoint(
+def _check_frame(
gif_path: Path,
frame_number: int,
expected: list[str],
forbidden: list[str],
work_dir: Path,
-) -> list[str]:
- """Validate a single checkpoint. Returns list of error messages."""
- errors = []
+) -> tuple[bool, list[str]]:
+ """Check a single frame against expected/forbidden patterns.
- # Extract frame
+ Returns (passed, errors). A frame passes when all expected patterns are
+ present AND no forbidden patterns are found.
+ """
frame_path = work_dir / f"frame_{frame_number}.png"
if not extract_frame(gif_path, frame_number, frame_path):
- return [f"Failed to extract frame {frame_number}"]
+ return False, [f"Failed to extract frame {frame_number}"]
- # OCR the frame
text = ocr_image(frame_path)
if not text:
- return [f"OCR failed for frame {frame_number}"]
+ return False, [f"OCR returned no text for frame {frame_number}"]
text_lower = text.lower()
+ errors = []
- # Check expected patterns
for pattern in expected:
if pattern.lower() not in text_lower:
- errors.append(f"Expected pattern not found: '{pattern}'")
+ errors.append(f"'{pattern}' not found")
- # Check forbidden patterns
for pattern in forbidden:
if pattern.lower() in text_lower:
- errors.append(f"Forbidden pattern found: '{pattern}'")
+ errors.append(f"forbidden '{pattern}' present")
+
+ return len(errors) == 0, errors
+
- return errors
+def validate_checkpoint(
+ gif_path: Path,
+ checkpoint: Checkpoint,
+ work_dir: Path,
+) -> tuple[bool, str]:
+ """Validate a checkpoint by scanning its frame range.
+
+ Returns (passed, detail_message).
+ """
+ best_errors: list[str] = []
+ frames_checked = 0
+
+ for frame in range(checkpoint.start, checkpoint.end + 1, checkpoint.step):
+ frames_checked += 1
+ passed, errors = _check_frame(
+ gif_path, frame, checkpoint.expected, checkpoint.forbidden, work_dir
+ )
+ if passed:
+ return True, f"matched at frame {frame} ({frames_checked} checked)"
+ # Track the attempt with fewest errors (closest to passing)
+ if not best_errors or len(errors) < len(best_errors):
+ best_errors = errors
+
+ label = f"frames {checkpoint.start}-{checkpoint.end}"
+ return False, f"no match in {label} ({frames_checked} checked): {'; '.join(best_errors)}"
def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:
"""Validate a TUI demo GIF against its checkpoints.
- Args:
- demo_name: Name of the demo (e.g., "wt-zellij-omnibus")
- gif_path: Path to the GIF file to validate
-
- Returns:
- List of error messages. Empty list means validation passed.
+ Returns list of error messages. Empty list means validation passed.
"""
if demo_name not in TUI_CHECKPOINTS:
return [f"No checkpoints defined for demo: {demo_name}"]
@@ -138,7 +188,6 @@ def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:
if not gif_path.exists():
return [f"GIF not found: {gif_path}"]
- # Check dependencies
missing = check_dependencies()
if missing:
return [f"Missing required tools: {', '.join(missing)}"]
@@ -149,12 +198,10 @@ def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:
with tempfile.TemporaryDirectory(prefix="wt-validate-") as work_dir:
work_path = Path(work_dir)
- for frame_number, expected, forbidden in checkpoints:
- errors = validate_checkpoint(
- gif_path, frame_number, expected, forbidden, work_path
- )
- if errors:
- all_errors.append(f"Frame {frame_number}: {'; '.join(errors)}")
+ for checkpoint in checkpoints:
+ passed, detail = validate_checkpoint(gif_path, checkpoint, work_path)
+ if not passed:
+ all_errors.append(detail)
return all_errors
@@ -162,8 +209,7 @@ def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:
def validate_tui_demo_verbose(demo_name: str, gif_path: Path) -> tuple[bool, str]:
"""Validate a TUI demo with verbose output.
- Returns:
- (success, output_message)
+ Returns (success, output_message).
"""
lines = [f"Validating {demo_name}: {gif_path}"]
@@ -183,17 +229,14 @@ def validate_tui_demo_verbose(demo_name: str, gif_path: Path) -> tuple[bool, str
with tempfile.TemporaryDirectory(prefix="wt-validate-") as work_dir:
work_path = Path(work_dir)
- for frame_number, expected, forbidden in checkpoints:
- errors = validate_checkpoint(
- gif_path, frame_number, expected, forbidden, work_path
- )
- if errors:
- lines.append(f" ✗ Frame {frame_number}")
- for error in errors:
- lines.append(f" - {error}")
- all_passed = False
+ for checkpoint in checkpoints:
+ passed, detail = validate_checkpoint(gif_path, checkpoint, work_path)
+ label = f"frames {checkpoint.start}-{checkpoint.end}"
+ if passed:
+ lines.append(f" ✓ {label}: {detail}")
else:
- lines.append(f" ✓ Frame {frame_number}")
+ lines.append(f" ✗ {label}: {detail}")
+ all_passed = False
if all_passed:
lines.append("✓ All checkpoints passed")