Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows
demos: frame-range OCR validation and build script improvements (#1778)
Maximilian Roos38307dfad458
diff --git a/docs/demos/build b/docs/demos/buildindex de728bbb5..efec63387 100755--- a/docs/demos/build+++ b/docs/demos/build@@ -19,6 +19,7 @@ import shutilimport subprocessimport sysimport tempfile+import timefrom concurrent.futures import ProcessPoolExecutor, as_completedfrom functools import partialfrom pathlib import Path@@ -348,8 +349,9 @@ def record_demo(vhs_binary: str,text_mode: bool = False,snapshot_mode: bool = False,-):- """Record a single demo (runs in subprocess for parallel execution)."""+) -> float:+ """Record a single demo. Returns elapsed seconds."""+ t0 = time.monotonic()tape_path = TAPES_DIR / tape_fileif not tape_path.exists():raise FileNotFoundError(f"{tape_file} not found")@@ -365,43 +367,51 @@ def record_demo(if snapshot_mode:output_snap = SNAPSHOTS_DIR / f"{output_name}.snap"record_snapshot(env, tape_path, output_snap, REPO_ROOT)- print(f"✓ {output_name}: Snapshot saved to {output_snap}")- elif text_mode:+ elapsed = time.monotonic() - t0+ print(f"✓ {output_name}: Snapshot saved ({elapsed:.0f}s)")+ return elapsed++ if text_mode:output_txt = mode_out_dir / f"{output_name}.txt"record_text(env, tape_path, output_txt, replacements, REPO_ROOT)- print(f"✓ {output_name}: Text saved to {output_txt}")- else:- output_gifs = {}- for theme in themes:- theme_dir = mode_out_dir / theme- theme_dir.mkdir(parents=True, exist_ok=True)- output_gifs[theme] = theme_dir / f"{output_name}.gif"-- record_all_themes(- env,- tape_path,- output_gifs,- REPO_ROOT,- vhs_binary=vhs_binary,- size=demo_size,- )+ elapsed = time.monotonic() - t0+ print(f"✓ {output_name}: Text saved ({elapsed:.0f}s)")+ return elapsed++ output_gifs = {}+ for theme in themes:+ theme_dir = mode_out_dir / theme+ theme_dir.mkdir(parents=True, exist_ok=True)+ output_gifs[theme] = theme_dir / f"{output_name}.gif"++ record_all_themes(+ env,+ tape_path,+ output_gifs,+ REPO_ROOT,+ vhs_binary=vhs_binary,+ size=demo_size,+ )++ elapsed = time.monotonic() - t0- # Validate TUI demos with OCR after recording- if output_name in TUI_CHECKPOINTS:- # Use the light theme GIF for validation- gif_path = output_gifs.get("light", list(output_gifs.values())[0])- success, output = validate_tui_demo_verbose(output_name, gif_path)- if success:- print(f"✓ {output_name}: GIFs saved + validation passed")- else:- print(f"✓ {output_name}: GIFs saved")- print(f"✗ {output_name}: Validation failed")- for line in output.split("\n"):- if line.startswith(" "):- print(line)- raise RuntimeError(f"TUI validation failed for {output_name}")+ # Validate TUI demos with OCR after recording+ if output_name in TUI_CHECKPOINTS:+ gif_path = output_gifs.get("light", list(output_gifs.values())[0])+ success, output = validate_tui_demo_verbose(output_name, gif_path)+ if success:+ print(f"✓ {output_name}: GIFs saved + validation passed ({elapsed:.0f}s)")else:- print(f"✓ {output_name}: GIFs saved: {', '.join(themes)}")+ print(f"✓ {output_name}: GIFs saved ({elapsed:.0f}s)")+ print(f"✗ {output_name}: Validation failed")+ for line in output.split("\n"):+ if line.startswith(" "):+ print(line)+ raise RuntimeError(f"TUI validation failed for {output_name}")+ else:+ print(f"✓ {output_name}: GIFs saved: {', '.join(themes)} ({elapsed:.0f}s)")++ return elapsed# =============================================================================@@ -508,6 +518,7 @@ Examples:vhs_binary = str(ensure_vhs_binary())# Build each target sequentially (parallel targets cause Zellij conflicts)+ t0 = time.monotonic()all_failed = []for target in dict.fromkeys(args.target): # deduplicate, preserve orderfailed = build_target(@@ -515,14 +526,23 @@ Examples:)all_failed.extend(f"{target}/{name}" for name in failed)- if all_failed:+ # Overall summary for multi-target builds+ if len(args.target) > 1:+ elapsed = time.monotonic() - t0+ mins, secs = divmod(int(elapsed), 60)print(f"\n{'=' * 60}")+ print(f"Total: {mins}m{secs:02d}s")++ if all_failed:+ if len(args.target) <= 1:+ print(f"\n{'=' * 60}")print(f"Failed: {', '.join(all_failed)}")sys.exit(1)def build_target(target: str, args, vhs_binary: str) -> list[str]:"""Build all demos for a single target. Returns list of failed demo names."""+ t0 = time.monotonic()mode_out_dir = OUT_DIR / targetmode_out_dir.mkdir(parents=True, exist_ok=True)@@ -565,54 +585,82 @@ def build_target(target: str, args, vhs_binary: str) -> list[str]:spawn_debug_shell(env, FIXTURES_DIR / "starship.toml")return []- # Record demos+ # TUI demos run sequentially (they use Zellij which conflicts when parallel)+ tui_demos = [(t, n, s) for t, n, s in demos if n in TUI_DEMOS]+ non_tui_demos = [(t, n, s) for t, n, s in demos if n not in TUI_DEMOS]+failed = []- if args.sequential or len(demos) == 1:- print(f"\n[{target}] Recording {len(demos)} demo(s) sequentially...")- for tape_file, output_name, setup_func in demos:- try:- record_demo(- tape_file,- output_name,- setup_func,- mode_out_dir,- themes,- demo_size,- vhs_binary,- args.text,- args.snapshot,- )- except Exception as e:- print(f"✗ {output_name}: {e}")- failed.append(output_name)- else:- print(f"\n[{target}] Recording {len(demos)} demo(s) in parallel...")- max_workers = min(len(demos), os.cpu_count() or 4)-- with ProcessPoolExecutor(max_workers=max_workers) as executor:- futures = {- executor.submit(- record_demo,- tape_file,- output_name,- setup_func,- mode_out_dir,- themes,- demo_size,- vhs_binary,- args.text,- args.snapshot,- ): output_name- for tape_file, output_name, setup_func in demos- }-- for future in as_completed(futures):- output_name = futures[future]- try:- future.result()- except Exception as e:- print(f"✗ {output_name}: {e}")- failed.append(output_name)+ succeeded = []++ def _record_one(tape_file, output_name, setup_func):+ try:+ elapsed = record_demo(+ tape_file,+ output_name,+ setup_func,+ mode_out_dir,+ themes,+ demo_size,+ vhs_binary,+ args.text,+ args.snapshot,+ )+ succeeded.append((output_name, elapsed))+ except Exception as e:+ print(f"✗ {output_name}: {e}")+ failed.append(output_name)++ # Record non-TUI demos (parallel unless --sequential)+ if non_tui_demos:+ if args.sequential or len(non_tui_demos) == 1:+ print(f"\n[{target}] Recording {len(non_tui_demos)} demo(s) sequentially...")+ for tape_file, output_name, setup_func in non_tui_demos:+ _record_one(tape_file, output_name, setup_func)+ else:+ print(f"\n[{target}] Recording {len(non_tui_demos)} demo(s) in parallel...")+ max_workers = min(len(non_tui_demos), os.cpu_count() or 4)++ with ProcessPoolExecutor(max_workers=max_workers) as executor:+ futures = {+ executor.submit(+ record_demo,+ tape_file,+ output_name,+ setup_func,+ mode_out_dir,+ themes,+ demo_size,+ vhs_binary,+ args.text,+ args.snapshot,+ ): output_name+ for tape_file, output_name, setup_func in non_tui_demos+ }++ for future in as_completed(futures):+ output_name = futures[future]+ try:+ elapsed = future.result()+ succeeded.append((output_name, elapsed))+ except Exception as e:+ print(f"✗ {output_name}: {e}")+ failed.append(output_name)++ # Record TUI demos sequentially (Zellij conflicts when parallel)+ if tui_demos:+ print(f"\n[{target}] Recording {len(tui_demos)} TUI demo(s) sequentially...")+ for tape_file, output_name, setup_func in tui_demos:+ _record_one(tape_file, output_name, setup_func)++ # Summary+ elapsed_total = time.monotonic() - t0+ mins, secs = divmod(int(elapsed_total), 60)+ passed_str = ", ".join(f"{n} ({e:.0f}s)" for n, e in succeeded)+ print(f"\n[{target}] {len(succeeded)} passed, {len(failed)} failed ({mins}m{secs:02d}s)")+ if succeeded:+ print(f" Passed: {passed_str}")+ if failed:+ print(f" Failed: {', '.join(failed)}")return faileddiff --git a/docs/demos/shared/validation.py b/docs/demos/shared/validation.pyindex 554e91a71..5077d492f 100644--- a/docs/demos/shared/validation.py+++ b/docs/demos/shared/validation.py@@ -2,9 +2,15 @@TUI demos (Zellij, interactive UIs) can't be validated via text output becauseVHS only captures the outer terminal, not content rendered inside terminal-multiplexers. Instead, we extract key frames from the GIF and use OCR to verify+multiplexers. Instead, we extract frames from the GIF and use OCR to verifyexpected content appears.+Checkpoints specify a frame range rather than a single frame. The validator+scans frames within the range (sampling every N frames) and passes the+checkpoint if ANY frame in the range matches all expected patterns while+containing none of the forbidden patterns. This makes validation resilient+to timing shifts from UI changes.+Usage:from shared.validation import validate_tui_demo, TUI_CHECKPOINTS@@ -18,24 +24,47 @@ from __future__ import annotationsimport subprocessimport tempfile+from dataclasses import dataclass, fieldfrom pathlib import Path-# Checkpoint definitions per TUI demo-# Format: {demo_name: [(frame_number, expected_patterns, forbidden_patterns), ...]}-#-# Frame numbers are calibrated from actual GIF content at 30fps.-# Expected patterns must ALL be present (case-insensitive).-# Forbidden patterns must ALL be absent (case-insensitive).-TUI_CHECKPOINTS: dict[str, list[tuple[int, list[str], list[str]]]] = {- # Frame numbers calibrated after removing initial wt list (demo starts with picker).- # At 30fps: frame 200 = ~6.7s, frame 1750 = ~58.3s+@dataclass+class Checkpoint:+ """A validation checkpoint that scans a range of frames."""++ start: int+ end: int+ expected: list[str] = field(default_factory=list)+ forbidden: list[str] = field(default_factory=list)+ step: int = 10+++# Checkpoint definitions per TUI demo.+# Ranges are calibrated from actual GIF content at 30fps.+# Expected patterns must ALL be present (case-insensitive) in at least one+# frame within the range. Forbidden patterns must ALL be absent.++TUI_CHECKPOINTS: dict[str, list[Checkpoint]] = {"wt-zellij-omnibus": [- # Frame 200: Claude UI visible on TAB 1 (api) - shows Opus model and task- (200, ["Opus", "acme", "Add a test"], ["command not found", "Unknown command"]),- # Frame 1750: Near end - wt list --full showing all worktrees- # (feature removed by wt remove in TAB 3)- (1750, ["Branch", "main", "billing"], ["CONFLICT", "error:", "failed"]),+ # Claude UI visible on TAB 1 (api) — shows model name and task.+ # Range covers the window where Claude's UI is rendered and stable.+ # Patterns kept minimal (just "Opus" + "acme") since Claude's UI+ # layout shifts across versions — task text may wrap or truncate.+ Checkpoint(+ start=150,+ end=350,+ expected=["Opus", "acme"],+ forbidden=["command not found", "Unknown command"],+ ),+ # Near end — wt list --full showing all worktrees.+ # "billing" omitted: depends on timing of when the branch appears+ # in the list relative to the frame window.+ Checkpoint(+ start=1650,+ end=1850,+ expected=["Branch", "main"],+ forbidden=["CONFLICT", "error:", "failed"],+ ),],}@@ -87,50 +116,71 @@ def ocr_image(image_path: Path) -> str:return ""-def validate_checkpoint(+def _check_frame(gif_path: Path,frame_number: int,expected: list[str],forbidden: list[str],work_dir: Path,-) -> list[str]:- """Validate a single checkpoint. Returns list of error messages."""- errors = []+) -> tuple[bool, list[str]]:+ """Check a single frame against expected/forbidden patterns.- # Extract frame+ Returns (passed, errors). A frame passes when all expected patterns are+ present AND no forbidden patterns are found.+ """frame_path = work_dir / f"frame_{frame_number}.png"if not extract_frame(gif_path, frame_number, frame_path):- return [f"Failed to extract frame {frame_number}"]+ return False, [f"Failed to extract frame {frame_number}"]- # OCR the frametext = ocr_image(frame_path)if not text:- return [f"OCR failed for frame {frame_number}"]+ return False, [f"OCR returned no text for frame {frame_number}"]text_lower = text.lower()+ errors = []- # Check expected patternsfor pattern in expected:if pattern.lower() not in text_lower:- errors.append(f"Expected pattern not found: '{pattern}'")+ errors.append(f"'{pattern}' not found")- # Check forbidden patternsfor pattern in forbidden:if pattern.lower() in text_lower:- errors.append(f"Forbidden pattern found: '{pattern}'")+ errors.append(f"forbidden '{pattern}' present")++ return len(errors) == 0, errors+- return errors+def validate_checkpoint(+ gif_path: Path,+ checkpoint: Checkpoint,+ work_dir: Path,+) -> tuple[bool, str]:+ """Validate a checkpoint by scanning its frame range.++ Returns (passed, detail_message).+ """+ best_errors: list[str] = []+ frames_checked = 0++ for frame in range(checkpoint.start, checkpoint.end + 1, checkpoint.step):+ frames_checked += 1+ passed, errors = _check_frame(+ gif_path, frame, checkpoint.expected, checkpoint.forbidden, work_dir+ )+ if passed:+ return True, f"matched at frame {frame} ({frames_checked} checked)"+ # Track the attempt with fewest errors (closest to passing)+ if not best_errors or len(errors) < len(best_errors):+ best_errors = errors++ label = f"frames {checkpoint.start}-{checkpoint.end}"+ return False, f"no match in {label} ({frames_checked} checked): {'; '.join(best_errors)}"def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:"""Validate a TUI demo GIF against its checkpoints.- Args:- demo_name: Name of the demo (e.g., "wt-zellij-omnibus")- gif_path: Path to the GIF file to validate-- Returns:- List of error messages. Empty list means validation passed.+ Returns list of error messages. Empty list means validation passed."""if demo_name not in TUI_CHECKPOINTS:return [f"No checkpoints defined for demo: {demo_name}"]@@ -138,7 +188,6 @@ def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:if not gif_path.exists():return [f"GIF not found: {gif_path}"]- # Check dependenciesmissing = check_dependencies()if missing:return [f"Missing required tools: {', '.join(missing)}"]@@ -149,12 +198,10 @@ def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:with tempfile.TemporaryDirectory(prefix="wt-validate-") as work_dir:work_path = Path(work_dir)- for frame_number, expected, forbidden in checkpoints:- errors = validate_checkpoint(- gif_path, frame_number, expected, forbidden, work_path- )- if errors:- all_errors.append(f"Frame {frame_number}: {'; '.join(errors)}")+ for checkpoint in checkpoints:+ passed, detail = validate_checkpoint(gif_path, checkpoint, work_path)+ if not passed:+ all_errors.append(detail)return all_errors@@ -162,8 +209,7 @@ def validate_tui_demo(demo_name: str, gif_path: Path) -> list[str]:def validate_tui_demo_verbose(demo_name: str, gif_path: Path) -> tuple[bool, str]:"""Validate a TUI demo with verbose output.- Returns:- (success, output_message)+ Returns (success, output_message)."""lines = [f"Validating {demo_name}: {gif_path}"]@@ -183,17 +229,14 @@ def validate_tui_demo_verbose(demo_name: str, gif_path: Path) -> tuple[bool, strwith tempfile.TemporaryDirectory(prefix="wt-validate-") as work_dir:work_path = Path(work_dir)- for frame_number, expected, forbidden in checkpoints:- errors = validate_checkpoint(- gif_path, frame_number, expected, forbidden, work_path- )- if errors:- lines.append(f" ✗ Frame {frame_number}")- for error in errors:- lines.append(f" - {error}")- all_passed = False+ for checkpoint in checkpoints:+ passed, detail = validate_checkpoint(gif_path, checkpoint, work_path)+ label = f"frames {checkpoint.start}-{checkpoint.end}"+ if passed:+ lines.append(f" ✓ {label}: {detail}")else:- lines.append(f" ✓ Frame {frame_number}")+ lines.append(f" ✗ {label}: {detail}")+ all_passed = Falseif all_passed:lines.append("✓ All checkpoints passed")