Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
#!/usr/bin/env python3
"""Plans engine verification runs from scripts/verify/suites.json.
plan.py select --tier fast --platform linux [--suites a,b] [--local]
Prints four lines for run.sh: the suite names, the cargo target
selection, the nextest filter expression and the case globs to skip.
plan.py changes --tier fast [--files FILE | --all] [--metadata FILE]
Maps changed paths to suites through the workspace dependency graph
(`cargo metadata`) and prints GitHub step outputs: run, reason,
suites, matrix.
plan.py recording [--files FILE | --all]
run=, reason= for the real-capture workflow: whether a change reaches
the recorder's or the stimulus's dependency closure or the harness.
plan.py risk --files F [--summary FILE] [--output FILE]
changed files -> high-risk regions and guards (scripts/verify/guards.toml):
escalate=, regions=, guards= lines plus ::warning annotations.
plan.py test
Self-test of the change mapping.
"""
import argparse
import fnmatch
import json
import os
import subprocess
import sys
ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".."))
PLATFORMS = ["linux", "macos", "windows"]
def load_guards():
import tomllib
with open(os.path.join(ROOT, "scripts", "verify", "guards.toml"), "rb") as f:
return tomllib.load(f)
# Engine crates: a pull request that changes any of them runs the FULL tier
# (with Blacksmith runners there is no queue to save).
ENGINE_CRATES = [
"crates/rendering/**",
"crates/editor/**",
"crates/export/**",
"crates/recording/**",
"crates/enc-*/**",
"crates/audio/**",
"crates/video-decode/**",
"crates/project/**",
"crates/scap-*/**",
"crates/camera*/**",
"crates/media/**",
"crates/media-info/**",
"crates/frame-converter/**",
"crates/gpu-converters/**",
"crates/cursor-capture/**",
"crates/timestamp/**",
"crates/cap-muxer/**",
]
def engine_files(files):
return sorted({p for p in files for pattern in ENGINE_CRATES if match(p, pattern)})
def risk(files, guards=None):
"""(regions touched, guards whose files are touched) for changed files."""
guards = guards or load_guards()
regions = []
for region in guards.get("region", []):
hits = sorted({p for p in files for pattern in region["paths"] if match(p, pattern)})
if hits:
regions.append((region["name"], hits))
touched = [
g for g in guards.get("guard", []) if any(p in g["files"] for p in files)
]
return regions, touched
def risk_report(regions, touched):
"""Job-summary markdown and annotation lines for a risky diff."""
if not regions and not touched:
return "", []
md = ["## Needs deep review", ""]
notes = []
if regions:
md.append("This change touches high-risk engine code. A pull request that does runs the FULL tier, with real capture on both macOS versions:")
md.append("")
for name, hits in regions:
md.append(f"- **{name}**: " + ", ".join(f"`{h}`" for h in hits[:6]) + (" ..." if len(hits) > 6 else ""))
notes.append(f"::warning title=Needs deep review ({name})::{', '.join(hits[:6])}")
md.append("")
if touched:
md.append("### Guard changed: needs deep review")
md.append("")
md.append("These guards protect users, but removing them changes no output the suites check (equivalent mutants), so only review can catch a regression:")
md.append("")
for g in touched:
md.append(f"- **{g['id']}** ({g['area']}; {', '.join(g['symbols'])}): {g['why']} Prevents: {g['prevents']}")
notes.append(f"::warning title=Guard changed: {g['id']} ({g['area']})::{g['prevents']}")
md.append("")
return "\n".join(md), notes
def load_config():
with open(os.path.join(ROOT, "scripts", "verify", "suites.json")) as f:
return json.load(f)
def match(path, pattern):
if pattern.endswith("/**"):
prefix = pattern[:-2]
if any(c in prefix for c in "*?["):
return fnmatch.fnmatchcase(path, prefix + "*")
return path.startswith(prefix)
return fnmatch.fnmatchcase(path, pattern)
def suite_filter(suite):
parts = []
for binary in suite["binaries"]:
expr = f"binary_id({binary})"
if suite.get("test"):
expr = f"({expr} and test(={suite['test']}))"
parts.append(expr)
return " or ".join(parts)
def select(config, tier, platform, names=None, local=False):
# Locally, platform-independent suites (CI runs them on linux only) run
# on every platform too.
suites = [
s for s in config["suites"]
if platform in s["platforms"][tier] or (local and s["platforms"][tier] == ["linux"])
]
if names:
unknown = set(names) - {s["name"] for s in config["suites"]}
if unknown:
sys.exit(f"unknown suites: {', '.join(sorted(unknown))}")
suites = [s for s in suites if s["name"] in names]
return suites
def cargo_args(suites):
packages, flags = [], []
for suite in suites:
args = suite["cargo"]
i = 0
while i < len(args):
if args[i] == "-p":
if args[i + 1] not in packages:
packages.append(args[i + 1])
i += 2
continue
if args[i] == "--test":
flag = ["--test", args[i + 1]]
i += 2
else:
flag = [args[i]]
i += 1
if flag not in flags:
flags.append(flag)
out = []
for package in packages:
out += ["-p", package]
for flag in flags:
out += flag
return out
def cmd_select(args):
config = load_config()
names = [n for n in (args.suites or "").split(",") if n]
suites = select(config, args.tier, args.platform, names or None, args.local)
print(" ".join(s["name"] for s in suites))
print(" ".join(cargo_args(suites)))
print(" or ".join(f"({suite_filter(s)})" for s in suites))
skips = []
for suite in suites:
for glob in suite.get("skip", {}).get(args.tier, {}).get(args.platform, []):
if glob not in skips:
skips.append(glob)
print(",".join(skips))
def cmd_env(args):
"""Shell `export` lines for the env of the selected suites (suites.json
`env`: {"KEY": "value"} or {"KEY": {"<platform>": "value"}}) and the
platform's unsupported-case allowlist. Variables already set in the
environment win."""
import shlex
config = load_config()
names = [n for n in (args.suites or "").split(",") if n]
allow = ",".join(config.get("unsupported", {}).get(args.platform, {}).keys())
if allow and "CAP_VERIFY_UNSUPPORTED_ALLOW" not in os.environ:
print(f"export CAP_VERIFY_UNSUPPORTED_ALLOW={shlex.quote(allow)}")
for suite in select(config, args.tier, args.platform, names or None, args.local):
for key, value in suite.get("env", {}).items():
if isinstance(value, dict):
value = value.get(args.platform)
if value is None or key in os.environ:
continue
print(f"export {key}={shlex.quote(str(value))}")
def workspace_graph(metadata):
"""Package dirs and reverse dependencies (normal, dev and build)."""
dirs, deps = {}, {}
for package in metadata["packages"]:
name = package["name"]
dirs[name] = os.path.relpath(os.path.dirname(package["manifest_path"]), metadata["workspace_root"]).replace(os.sep, "/")
deps[name] = [
(d["name"], d.get("kind") or "normal")
for d in package["dependencies"]
if d.get("path")
]
return dirs, deps
def closure(root, deps):
"""Packages whose code ends up in `root`'s tests: root's normal, dev and
build dependencies, then normal and build dependencies transitively."""
seen = {root}
stack = [(name, kind) for name, kind in deps.get(root, [])]
while stack:
name, _kind = stack.pop()
if name in seen or name not in deps:
continue
seen.add(name)
stack.extend((n, k) for n, k in deps[name] if k != "dev")
return seen
def owning_package(path, dirs):
best = None
for name, d in dirs.items():
if d == ".":
continue
if path == d or path.startswith(d + "/"):
if best is None or len(d) > len(dirs[best]):
best = name
return best
def affected(config, files, metadata, tier):
"""Returns (suite names, reason)."""
all_names = [s["name"] for s in config["suites"]]
for path in files:
if path.startswith("apps/desktop-gpui/"):
return all_names, f"{path} changes the standalone GPUI application"
for pattern in config["always"]:
if match(path, pattern):
return all_names, f"{path} affects every suite"
dirs, deps = workspace_graph(metadata)
closures = {s["name"]: set().union(*(closure(r, deps) for r in s["roots"])) for s in config["suites"]}
owned = {}
for suite in config["suites"]:
for pattern in suite["own"]:
owned.setdefault(pattern, set()).add(suite["name"])
selected, reasons = set(), []
for path in files:
package = owning_package(path, dirs)
if package is None:
continue
owners = set()
for pattern, names in owned.items():
if match(path, pattern):
owners |= names
hits = owners if owners else {n for n, c in closures.items() if package in c}
if hits - selected:
reasons.append(f"{path} ({package}) -> {', '.join(sorted(hits))}")
selected |= hits
names = [n for n in all_names if n in selected]
if not names:
return [], "no engine code changed"
return names, "; ".join(reasons[:8]) + (" ..." if len(reasons) > 8 else "")
def extra_runners(config, names, tier, files):
"""Additional platform entries (suites.json `extra_runners`): the same
platform's suites on another machine class, for a path the main runner
cannot cover (macOS export on the VideoToolbox hardware encoder, which
Blacksmith VMs lack). Included for every changed path that matches the
entry's `paths`, and always when `files` is None (a full run)."""
out = []
for name, entry in config.get("extra_runners", {}).items():
suites = [
s["name"] for s in config["suites"]
if s["name"] in names and s["name"] in entry["suites"] and entry["os"] in s["platforms"][tier]
]
if not suites:
continue
if files is not None and not any(match(f, p) for f in files for p in entry["paths"]):
continue
shards = entry["shards"][tier] if isinstance(entry["shards"], dict) else entry["shards"]
out.append({
"platform": name,
"os": entry["os"],
"runner": os.environ.get(f"ENGINE_VERIFY_{name.upper().replace('-', '_')}_RUNNER", "").strip() or entry["runner"],
"target": entry["target"],
"shards": shards,
"shard_list": json.dumps(list(range(1, shards + 1))),
"archive": entry.get("archive", False),
"suites": ",".join(suites),
})
return out
def matrix(config, names, tier, files=None):
include = []
for platform in PLATFORMS:
suites = [s["name"] for s in config["suites"] if s["name"] in names and platform in s["platforms"][tier]]
if not suites:
continue
runner = dict(config["runners"][platform])
# One switch per platform without a code change: the repository
# variable ENGINE_VERIFY_<PLATFORM>_RUNNER (passed in by
# engine-verify.yml) overrides suites.json, e.g. macos-15 to go back
# to GitHub-hosted runners.
override = os.environ.get(f"ENGINE_VERIFY_{platform.upper()}_RUNNER", "").strip()
if override:
runner["runner"] = override
if isinstance(runner["shards"], dict):
runner["shards"] = runner["shards"][tier]
include.append({
"platform": platform,
"os": platform,
"runner": runner["runner"],
"target": runner["target"],
"shards": runner["shards"],
"shard_list": json.dumps(list(range(1, runner["shards"] + 1))),
"archive": runner.get("archive", False),
"suites": ",".join(suites),
})
include += extra_runners(config, names, tier, files)
return {"include": include}
def load_metadata(path):
if path:
with open(path) as f:
return json.load(f)
env = dict(os.environ)
env.setdefault("RUSTUP_TOOLCHAIN", "stable")
out = subprocess.run(
["cargo", "metadata", "--no-deps", "--format-version", "1"],
cwd=ROOT, env=env, check=True, capture_output=True, text=True,
).stdout
return json.loads(out)
def cmd_changes(args):
config = load_config()
files = None
if args.all:
names, reason = [s["name"] for s in config["suites"]], "full run"
else:
with open(args.files) as f:
files = [line.strip() for line in f if line.strip()]
if len(files) >= args.max_files:
names, reason = [s["name"] for s in config["suites"]], f"{len(files)} changed files (list may be truncated)"
files = None
else:
names, reason = affected(config, files, load_metadata(args.metadata), args.tier)
if args.suites:
wanted = [n for n in args.suites.split(",") if n]
unknown = set(wanted) - {s["name"] for s in config["suites"]}
if unknown:
sys.exit(f"unknown suites: {', '.join(sorted(unknown))}")
names = [n for n in names if n in wanted]
reason += f" (limited to {args.suites})"
m = matrix(config, names, args.tier, files)
lines = {
"run": "true" if m["include"] else "false",
"reason": reason.replace("\n", " "),
"suites": ",".join(names),
"matrix": json.dumps(m, separators=(",", ":")),
}
for key, value in lines.items():
print(f"{key}={value}")
# Real capture (crates/verify/tests/recording.rs): the recorder and the
# stimulus, their whole dependency closure, and the harness itself.
RECORDING = {
"roots": ["cap-recording", "cap-verify-stimulus"],
"own": [
"crates/verify/tests/recording.rs",
"crates/verify/src/**",
"crates/verify/Cargo.toml",
"crates/verify/expectations.toml",
"crates/verify-expect/**",
"crates/verify-stimulus/**",
".github/workflows/verify-recording.yml",
],
}
def recording_affected(config, files, metadata):
"""(run, reason) for the real-capture workflow."""
for path in files:
for pattern in config["always"] + RECORDING["own"]:
if match(path, pattern):
return True, f"{path} affects real capture"
dirs, deps = workspace_graph(metadata)
reach = set().union(*(closure(r, deps) for r in RECORDING["roots"]))
for path in files:
# Other suites' test files share the cap-verify package but not the
# recording test binary.
if path.startswith("crates/verify/tests/"):
continue
package = owning_package(path, dirs)
if package in reach:
return True, f"{path} ({package}) is in the recorder's dependency closure"
return False, "no recording code changed"
def cmd_recording(args):
config = load_config()
if args.all:
run, reason = True, "full run"
else:
with open(args.files) as f:
files = [line.strip() for line in f if line.strip()]
if len(files) >= args.max_files:
run, reason = True, f"{len(files)} changed files (list may be truncated)"
else:
run, reason = recording_affected(config, files, load_metadata(args.metadata))
print(f"run={'true' if run else 'false'}")
print(f"reason={reason}")
def cmd_risk(args):
with open(args.files) as f:
files = [line.strip() for line in f if line.strip()]
regions, touched = risk(files)
md, notes = risk_report(regions, touched)
if args.summary and md:
with open(args.summary, "a") as f:
f.write(md + "\n")
for note in notes:
print(note)
engine = engine_files(files)
if engine:
print(f"::notice title=Engine crates changed::FULL tier for {', '.join(engine[:6])}{' ...' if len(engine) > 6 else ''}")
outputs = [
f"escalate={'true' if regions or touched else 'false'}",
f"engine={'true' if engine else 'false'}",
f"regions={','.join(name for name, _ in regions)}",
f"guards={','.join(g['id'] for g in touched)}",
]
if args.output:
with open(args.output, "a") as f:
f.write("\n".join(outputs) + "\n")
else:
print("\n".join(outputs))
def cmd_test(_args):
config = load_config()
metadata = load_metadata(None)
cases = [
(["apps/web/app/page.tsx", "README.md"], []),
(["crates/verify/tests/seek.rs"], ["seek"]),
(["crates/verify/tests/seek.rs", "crates/verify/tests/export.rs"], ["export", "seek"]),
(["crates/verify/src/oracle.rs"], [s["name"] for s in config["suites"] if "cap-verify" in s["roots"]]),
(["crates/verify/src/audio_model.rs"], ["audio"]),
(["crates/rendering/src/decoder/ffmpeg.rs"], [s["name"] for s in config["suites"]]),
(["crates/video-decode/src/ffmpeg.rs"], [s["name"] for s in config["suites"]]),
(["Cargo.lock"], [s["name"] for s in config["suites"]]),
(["scripts/verify/run.sh"], [s["name"] for s in config["suites"]]),
(["apps/desktop-gpui/src/main.rs"], [s["name"] for s in config["suites"]]),
(["apps/desktop-gpui/src/editor.rs"], [s["name"] for s in config["suites"]]),
(["apps/desktop-gpui/src/recording.rs"], [s["name"] for s in config["suites"]]),
(["apps/desktop-gpui/src/upload/verify_tests.rs"], [s["name"] for s in config["suites"]]),
(["apps/desktop-gpui/Cargo.toml"], [s["name"] for s in config["suites"]]),
]
failed = 0
for files, expected in cases:
names, reason = affected(config, files, metadata, "fast")
ok = sorted(names) == sorted(expected)
failed += not ok
print(f"{'ok ' if ok else 'BAD'} {files} -> {names} ({reason})")
guards = load_guards()
for g in guards.get("guard", []):
if not any(match(f, pattern) for f in g["files"] for r in guards["region"] for pattern in r["paths"]):
failed += 1
print(f"BAD guard {g['id']}: its files lie outside every high-risk region")
for files, expected in [
(["crates/enc-ffmpeg/src/base.rs"], (["encoder timestamps"], ["E02"])),
(["crates/editor/src/audio.rs", "README.md"], (["audio mixing and Studio Sound"], ["A09"])),
(["apps/desktop-gpui/src/upload/queue.rs"], (["upload retry"], [])),
(["crates/verify/src/oracle.rs", "apps/web/app/page.tsx"], ([], [])),
]:
regions, touched = risk(files, guards)
got = ([n for n, _ in regions], [g["id"] for g in touched])
ok = got == expected
failed += not ok
print(f"{'ok ' if ok else 'BAD'} risk {files} -> {got}")
for files, expected in [
(["crates/rendering/src/lib.rs"], True),
(["crates/enc-ffmpeg/src/lib.rs"], True),
(["crates/camera-ffmpeg/src/lib.rs"], True),
(["apps/desktop-gpui/src/main.rs", "crates/verify/src/oracle.rs"], False),
]:
ok = bool(engine_files(files)) == expected
failed += not ok
print(f"{'ok ' if ok else 'BAD'} engine {files} -> {bool(engine_files(files))}")
for files, expected in [
(["crates/frame-converter/src/lib.rs"], True),
(["crates/recording/src/studio_recording.rs"], True),
(["crates/verify/tests/recording.rs"], True),
(["vendor/wgpu-hal/src/lib.rs"], True),
(["Cargo.toml"], True),
(["crates/verify/tests/export.rs", "apps/web/app/page.tsx"], False),
]:
got = recording_affected(config, files, metadata)[0]
ok = got == expected
failed += not ok
print(f"{'ok ' if ok else 'BAD'} recording {files} -> {got}")
names = [s["name"] for s in config["suites"]]
for files, expected in [
(["crates/enc-ffmpeg/src/video/h264.rs"], True),
(["crates/export/src/mp4.rs"], True),
(["crates/rendering/src/decoder/ffmpeg.rs"], False),
(["crates/editor/src/playback.rs"], False),
(None, True),
]:
got = any(e["platform"] == "macos-vt" for e in matrix(config, names, "fast", files)["include"])
ok = got == expected
failed += not ok
print(f"{'ok ' if ok else 'BAD'} macos-vt {files} -> {got}")
rendering_only = affected(config, ["crates/rendering/src/decoder/model/mod.rs"], metadata, "fast")[0]
if "decoder-model-ffmpeg" not in rendering_only:
failed += 1
print(f"BAD decoder model change -> {rendering_only}")
sys.exit(1 if failed else 0)
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
p = sub.add_parser("select")
p.add_argument("--tier", choices=["fast", "full"], required=True)
p.add_argument("--platform", choices=PLATFORMS, required=True)
p.add_argument("--suites")
p.add_argument("--local", action="store_true")
p.set_defaults(func=cmd_select)
p = sub.add_parser("env")
p.add_argument("--tier", choices=["fast", "full"], required=True)
p.add_argument("--platform", choices=PLATFORMS, required=True)
p.add_argument("--suites")
p.add_argument("--local", action="store_true")
p.set_defaults(func=cmd_env)
p = sub.add_parser("changes")
p.add_argument("--tier", choices=["fast", "full"], required=True)
p.add_argument("--files")
p.add_argument("--all", action="store_true")
p.add_argument("--metadata")
p.add_argument("--max-files", type=int, default=3000)
p.add_argument("--suites")
p.set_defaults(func=cmd_changes)
p = sub.add_parser("recording")
p.add_argument("--files")
p.add_argument("--all", action="store_true")
p.add_argument("--metadata")
p.add_argument("--max-files", type=int, default=3000)
p.set_defaults(func=cmd_recording)
p = sub.add_parser("risk")
p.add_argument("--files", required=True)
p.add_argument("--summary", help="append the markdown report to this file")
p.add_argument("--output", help="append the escalate/regions/guards lines to this file (GITHUB_OUTPUT)")
p.set_defaults(func=cmd_risk)
p = sub.add_parser("test")
p.set_defaults(func=cmd_test)
args = parser.parse_args()
args.func(args)
if __name__ == "__main__":
main()