Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat(verify-openhands): record videos of the browser with `browser record` (#18211)

Engel Nyst683f6592555a
diff --git a/.agents/skills/verify-openhands/SKILL.md b/.agents/skills/verify-openhands/SKILL.md
index f749fd0f6b..4b24fec546 100644
--- a/.agents/skills/verify-openhands/SKILL.md
+++ b/.agents/skills/verify-openhands/SKILL.md
@@ -44,7 +44,9 @@ Requirements: Node >=24 (the launcher's engine), npm dependencies installed
(`npm ci --ignore-scripts`), `uv`/`uvx`, and a Chromium Playwright can launch
(set `CONTROL_OPENHANDS_BROWSER=/path/to/chrome` when the pinned browser is not
installed; `launch` then reports the running stack and `browser start` picks it
-up). Every command prints one JSON object; exit 0 ok, 1 action failed, 2 usage,
+up). `browser record` also needs an ffmpeg: one with libx264 on `PATH` writes
+MP4 (and GIF), and Playwright's bundled one (`npx playwright install ffmpeg`)
+writes WebM. Every command prints one JSON object; exit 0 ok, 1 action failed, 2 usage,
3 environment.
If a user path cannot be driven with the CLI, that is a **harness gap**: extend
@@ -134,7 +136,14 @@ control-openhands stop # stops only this run; evidence s
prints; `evidence report` renders the table from
[the report contract](references/report.md), fail and blocked rows first with
their `--note` (a blocked row names its missing prerequisite there), and
- `--baseline <yesterday's run>` lists what changed since that ledger. Keys, logs, browser profile and
+ `--baseline <yesterday's run>` lists what changed since that ledger. When
+ timing or a transition is the point (a menu closing on Escape, a reply
+ streaming in), record it: `browser record start --feature ID --name N`,
+ drive the steps, then `browser record stop` saves a video of the active tab
+ under the same folder (`--gif` adds one that GitHub shows inline). Around a
+ long wait, such as the agent working, run `browser record pause` and `browser
+ record resume`: the video cuts there. The caret is hidden in recordings, as
+ in screenshots. Keys, logs, browser profile and
downloads stay in `<run>/private/`. Evidence is not automatically public:
review every image before publishing it. The CLI masks password fields in
`snapshot`, `value` and `testids`; a screenshot of a visible key field is
diff --git a/.agents/skills/verify-openhands/references/report.md b/.agents/skills/verify-openhands/references/report.md
index 73eff0337b..6a294388d7 100644
--- a/.agents/skills/verify-openhands/references/report.md
+++ b/.agents/skills/verify-openhands/references/report.md
@@ -36,7 +36,8 @@ Intent status: documented / undocumented / contradictory. Runtime classification
intended-change / introduced-in-range / reproduced-on-both /
observed-origin-unconfirmed / environment-or-harness-gap. Keep these independent.
A screenshot proves appearance, not provenance or a side effect; include reloads,
-real files, actual run status, or read-only state as appropriate.
+real files, actual run status, or read-only state as appropriate. A recording
+(`browser record`) shows order and timing; say where it was paused.
## Findings and verdict
diff --git a/.agents/skills/verify-openhands/scripts/browser-daemon.mjs b/.agents/skills/verify-openhands/scripts/browser-daemon.mjs
index f72e6c6d17..81b34ebb7a 100755
--- a/.agents/skills/verify-openhands/scripts/browser-daemon.mjs
+++ b/.agents/skills/verify-openhands/scripts/browser-daemon.mjs
@@ -15,6 +15,7 @@ import {
existsSync,
mkdirSync,
readFileSync,
+ rmSync,
writeFileSync,
} from "node:fs";
import { createRequire } from "node:module";
@@ -23,6 +24,15 @@ import { fileURLToPath } from "node:url";
import { redactStorage } from "./lib/redact-storage.mjs";
import { SECRET_KEY, SECRET_PATH, redactBody } from "./lib/network-bodies.mjs";
import { buildLocator, toCss } from "./lib/selectors.mjs";
+import {
+ RECORD_DEFAULTS,
+ canvasSize,
+ encodeRecording,
+ findFfmpeg,
+ frameDurations,
+ videoToGif,
+ withoutPauses,
+} from "./lib/recording.mjs";
const here = dirname(fileURLToPath(import.meta.url));
const repoRoot = resolve(here, "../../../..");
@@ -517,6 +527,49 @@ async function failureShot() {
}
}
+// One recording at a time (`browser record`). The loop captures whichever tab
+// is active, so it follows `browser tab`; see lib/recording.mjs for why it
+// polls instead of using a screencast. The text caret is hidden in each
+// capture, as in screenshots: its blink would make an idle page look busy.
+let recording = null;
+
+async function captureLoop(rec) {
+ while (!rec.stopping && Date.now() - rec.startedAt < rec.maxMs) {
+ const t = Date.now();
+ const page = activePage;
+ if (rec.pausedAt) {
+ await new Promise((r) => setTimeout(r, rec.intervalMs));
+ continue;
+ }
+ try {
+ const data = await page.screenshot({
+ type: "jpeg",
+ quality: 85,
+ caret: "hide",
+ timeout: 5_000,
+ });
+ rec.captures += 1;
+ const hash = createHash("sha1").update(data).digest("hex");
+ if (hash !== rec.lastHash) {
+ rec.lastHash = hash;
+ const file = join(
+ rec.dir,
+ `frame-${String(rec.frames.length).padStart(5, "0")}.jpg`,
+ );
+ writeFileSync(file, data);
+ const size = page.viewportSize() ?? {};
+ rec.frames.push({ file, t, width: size.width, height: size.height });
+ }
+ } catch {
+ // A navigation or a closed tab; the next tick captures the active page.
+ rec.misses += 1;
+ }
+ const wait = rec.intervalMs - (Date.now() - t);
+ if (wait > 0) await new Promise((r) => setTimeout(r, wait));
+ }
+ rec.endedAt = Date.now();
+}
+
async function collectTestids(scopeSelector, includeHidden) {
const scope = scopeSelector
? locate(scopeSelector)
@@ -1354,6 +1407,153 @@ const handlers = {
await probe.close();
}
},
+ async "record-start"({ feature, name, fps, maxSeconds }) {
+ if (recording)
+ throw new Error(
+ "A recording is already running: end it with `browser record stop`.",
+ );
+ const rate = Number(fps ?? RECORD_DEFAULTS.fps);
+ if (!(rate >= 1 && rate <= 25))
+ throw new Error("--fps must be between 1 and 25");
+ const limit = Number(maxSeconds ?? RECORD_DEFAULTS.maxSeconds);
+ if (!(limit > 0 && limit <= 3600))
+ throw new Error("--max-seconds must be between 1 and 3600");
+ const ffmpeg = findFfmpeg();
+ if (!ffmpeg)
+ throw new Error(
+ "browser record needs ffmpeg: install it (MP4, --gif), or run `npx playwright install ffmpeg` (WebM). CONTROL_OPENHANDS_FFMPEG=/path picks one.",
+ );
+ const dir = join(privateDir, "recordings", String(Date.now()));
+ mkdirSync(dir, { recursive: true });
+ recording = {
+ feature,
+ name,
+ dir,
+ ffmpeg,
+ fps: rate,
+ intervalMs: 1000 / rate,
+ maxMs: limit * 1000,
+ startedAt: Date.now(),
+ frames: [],
+ cuts: [],
+ captures: 0,
+ misses: 0,
+ };
+ recording.loop = captureLoop(recording);
+ return {
+ recording: true,
+ fps: rate,
+ maxSeconds: limit,
+ format: ffmpeg.format,
+ ffmpeg: ffmpeg.path,
+ url: activePage.url(),
+ };
+ },
+ async "record-pause"() {
+ if (!recording || recording.endedAt)
+ throw new Error("No recording is running.");
+ if (recording.pausedAt) throw new Error("The recording is already paused.");
+ recording.pausedAt = Date.now();
+ return { paused: true };
+ },
+ async "record-resume"() {
+ if (!recording?.pausedAt) throw new Error("No recording is paused.");
+ if (recording.endedAt)
+ throw new Error(
+ "The recording reached --max-seconds while paused; stop it.",
+ );
+ const ms = Date.now() - recording.pausedAt;
+ recording.cuts.push({ at: recording.pausedAt, ms });
+ recording.pausedAt = undefined;
+ // The first capture after the cut is kept even if nothing changed.
+ recording.lastHash = undefined;
+ return { resumed: true, cutSeconds: Number((ms / 1000).toFixed(1)) };
+ },
+ async "record-status"() {
+ if (!recording) return { recording: false };
+ return {
+ recording: !recording.endedAt,
+ paused: Boolean(recording.pausedAt),
+ capped: Boolean(recording.endedAt),
+ seconds: ((recording.endedAt ?? Date.now()) - recording.startedAt) / 1000,
+ captures: recording.captures,
+ changes: recording.frames.length,
+ misses: recording.misses,
+ };
+ },
+ async "record-stop"({ gif, keepFrames, discard }) {
+ const rec = recording;
+ if (!rec)
+ throw new Error(
+ "No recording is running: start one with `browser record start`.",
+ );
+ rec.stopping = true;
+ await rec.loop;
+ recording = null;
+ if (discard) {
+ rmSync(rec.dir, { recursive: true, force: true });
+ return { discarded: true, captures: rec.captures };
+ }
+ if (!rec.frames.length)
+ throw new Error(
+ `No frame was captured (${rec.misses} failed captures); frames dir ${rec.dir}`,
+ );
+ // Stopped while paused: the video ends where the pause began.
+ const timeline = withoutPauses(
+ rec.frames,
+ rec.pausedAt ?? rec.endedAt,
+ rec.cuts,
+ );
+ const durations = frameDurations(timeline.frames, timeline.endT);
+ const { width, height } = canvasSize(rec.frames);
+ const path = evidencePath(rec.feature, rec.name, `.${rec.ffmpeg.format}`);
+ const written = await encodeRecording({
+ ffmpeg: rec.ffmpeg,
+ frames: rec.frames,
+ durations,
+ fps: rec.fps,
+ width,
+ height,
+ out: path,
+ }).catch((error) => {
+ throw new Error(`${error.message} (frames kept in ${rec.dir})`);
+ });
+ const result = {
+ path,
+ seconds: Number((written / rec.fps).toFixed(1)),
+ recordedSeconds: Number(
+ ((rec.endedAt - rec.startedAt) / 1000).toFixed(1),
+ ),
+ size: `${width}x${height}`,
+ fps: rec.fps,
+ captures: rec.captures,
+ changes: rec.frames.length,
+ misses: rec.misses,
+ capped: rec.endedAt - rec.startedAt >= rec.maxMs,
+ cutSeconds: Number(
+ (rec.cuts.reduce((sum, c) => sum + c.ms, 0) / 1000).toFixed(1),
+ ),
+ url: activePage.url(),
+ };
+ if (gif && !rec.ffmpeg.gif) {
+ result.gifError = `${rec.ffmpeg.path} cannot write GIF; install ffmpeg for --gif`;
+ } else if (gif) {
+ const out = evidencePath(rec.feature, rec.name, ".gif");
+ await videoToGif({
+ ffmpeg: rec.ffmpeg,
+ video: path,
+ out,
+ fps: Math.min(rec.fps, 12),
+ width: RECORD_DEFAULTS.gifWidth,
+ }).then(
+ () => (result.gif = out),
+ (error) => (result.gifError = error.message),
+ );
+ }
+ if (keepFrames) result.framesDir = rec.dir;
+ else rmSync(rec.dir, { recursive: true, force: true });
+ return result;
+ },
async shutdown() {
setTimeout(async () => {
await context.close().catch(() => {});
diff --git a/.agents/skills/verify-openhands/scripts/control-openhands.mjs b/.agents/skills/verify-openhands/scripts/control-openhands.mjs
index c4d4658ecf..b21f47d640 100755
--- a/.agents/skills/verify-openhands/scripts/control-openhands.mjs
+++ b/.agents/skills/verify-openhands/scripts/control-openhands.mjs
@@ -2908,6 +2908,44 @@ async function cmdBrowser({ positional, flags }) {
fullPage: Boolean(flags["full-page"]),
});
break;
+ case "record": {
+ const action = rest[0];
+ if (action === "start") {
+ if (!flags.feature || !flags.name)
+ usage(
+ "browser record start needs --feature <ID> and --name <label>",
+ "control-openhands browser record start --feature F05.overflow-menu --name escape",
+ );
+ result = await browserCall(run, "record-start", {
+ feature: flags.feature,
+ name: flags.name,
+ fps: flags.fps === undefined ? undefined : intFlag(flags.fps),
+ maxSeconds:
+ flags["max-seconds"] === undefined
+ ? undefined
+ : intFlag(flags["max-seconds"]),
+ });
+ } else if (action === "stop") {
+ result = await browserCall(
+ run,
+ "record-stop",
+ {
+ gif: Boolean(flags.gif),
+ keepFrames: Boolean(flags["keep-frames"]),
+ discard: Boolean(flags.discard),
+ },
+ { timeout: 600_000 },
+ );
+ } else if (["pause", "resume", "status"].includes(action)) {
+ result = await browserCall(run, `record-${action}`);
+ } else {
+ usage(
+ "browser record start|pause|resume|stop|status",
+ "control-openhands browser record start --feature F05.overflow-menu --name escape",
+ );
+ }
+ break;
+ }
case "clock":
if (
!flags["offset-ms"] &&
@@ -3306,6 +3344,7 @@ const BROWSER_VERBS = new Set([
"snapshot",
"testids",
"screenshot",
+ "record",
"viewport",
"clock",
"errors",
@@ -4139,6 +4178,12 @@ Verbs
snapshot [<sel>] [--max-lines N] [--feature ID --name N] ARIA tree (saved as evidence)
testids [<sel>] [--hidden] [--filter part] discover on-screen data-testid handles (--hidden adds hidden/off-screen)
screenshot [<sel>] --feature ID --name N [--full-page] PNG under evidence/<ID>/
+ record start --feature ID --name N [--fps 10] [--max-seconds 600] video of the active tab
+ record pause | record resume cut a wait (the agent working) out of the video
+ record stop [--gif] [--keep-frames] [--discard] | record status
+ (MP4 under evidence/<ID>/ when ffmpeg has libx264, else WebM with Playwright's ffmpeg;
+ --gif adds a GIF at most 960 px wide; the still lead-in is cut to 1 s, the end held 1 s;
+ --max-seconds is wall time; browser stop or reset ends the daemon and the recording)
viewport desktop|phone|narrow|tablet|WxH 1440x1000, 390x844, 320x700, 820x1180
clock --offset-ms N | --system ISO|+MS | --fixed ISO|+MS skew the page's clock (install before the goto
whose page should see it; the server's clock is untouched)
diff --git a/.agents/skills/verify-openhands/scripts/control-openhands.test.mjs b/.agents/skills/verify-openhands/scripts/control-openhands.test.mjs
index eb042cbb92..c7dd417fc5 100755
--- a/.agents/skills/verify-openhands/scripts/control-openhands.test.mjs
+++ b/.agents/skills/verify-openhands/scripts/control-openhands.test.mjs
@@ -36,6 +36,17 @@ import { routePattern } from "./lib/route-pattern.mjs";
import { resolveTestids } from "./lib/testids.mjs";
import { tmuxPathFor } from "./lib/tmux-path.mjs";
import { browserCallLimit } from "./lib/call-limit.mjs";
+import {
+ canvasSize,
+ encodeRecording,
+ encoderSupport,
+ ffmpegArgs,
+ findFfmpeg,
+ frameDurations,
+ frameRepeats,
+ playwrightFfmpegPaths,
+ withoutPauses,
+} from "./lib/recording.mjs";
import { MAX_PAGES, collectEvents, countImages } from "./lib/events-paging.mjs";
import { redactBody, redactUrl } from "./lib/network-bodies.mjs";
import { buildLocator, parseRole, toCss } from "./lib/selectors.mjs";
@@ -1432,3 +1443,131 @@ test("fixture skill --commit records the project skill once and reports a re-run
assert.equal(personal.status, 2);
assert.match(personal.json.error, /needs --repo/);
});
+
+test("a recording's timeline cuts the lead-in and pauses, and holds the end", () => {
+ // Changes at 0, 3, 3.2 and 10 s; stopped at 10.5 s.
+ const frames = [{ t: 0 }, { t: 3000 }, { t: 3200 }, { t: 10000 }];
+ assert.deepEqual(frameDurations(frames, 10500), [1, 0.2, 6.8, 1]);
+ // Paused from 4 s to 9 s: the frame shown at 3.2 s lasts 1.8 s.
+ const cut = withoutPauses(frames, 10500, [{ at: 4000, ms: 5000 }]);
+ assert.deepEqual(
+ cut.frames.map((f) => f.t),
+ [0, 3000, 3200, 5000],
+ );
+ assert.equal(cut.endT, 5500);
+ assert.deepEqual(frameDurations(cut.frames, cut.endT), [1, 0.2, 1.8, 1]);
+ // One unchanging frame keeps its real length, and is shown for at least 1 s.
+ assert.deepEqual(frameDurations([{ t: 0 }], 5000), [5]);
+ assert.deepEqual(frameDurations([{ t: 0 }], 200), [1]);
+ // Constant frame rate: ends rounded on the running total, at least one each.
+ assert.deepEqual(frameRepeats([1, 0.2, 6.8, 1], 10), [10, 2, 68, 10]);
+ assert.deepEqual(frameRepeats([0.15, 0.15, 0.15], 10), [2, 1, 2]);
+ assert.deepEqual(frameRepeats([0.01, 0.01], 10), [1, 1]);
+ // The canvas is the largest frame with even sides; smaller frames are padded.
+ assert.deepEqual(
+ canvasSize([
+ { width: 1440, height: 1000 },
+ { width: 391, height: 845 },
+ ]),
+ { width: 1440, height: 1000 },
+ );
+ assert.deepEqual(canvasSize([{ width: 391, height: 845 }]), {
+ width: 392,
+ height: 846,
+ });
+});
+
+test("the recorder picks MP4 with libx264, else WebM, and finds Playwright's ffmpeg", () => {
+ const system = [
+ " V....D libx264 libx264 H.264 / AVC (codec h264)",
+ " V....D libx264rgb libx264 H.264 RGB (codec h264)",
+ " V....D gif GIF (Graphics Interchange Format)",
+ ].join("\n");
+ assert.deepEqual(encoderSupport(system), { format: "mp4", gif: true });
+ const bundled =
+ " V....D libvpx libvpx VP8 (codec vp8)\n A....D aac x";
+ assert.deepEqual(encoderSupport(bundled), { format: "webm", gif: false });
+ assert.equal(encoderSupport(" V....D libx264rgb rgb only"), null);
+ const root = mkdtempSync(join(tmpdir(), "pw-browsers-"));
+ for (const dir of ["chromium-1194", "ffmpeg-1009", "ffmpeg-1011"])
+ mkdirSync(join(root, dir));
+ assert.deepEqual(
+ playwrightFfmpegPaths({ PLAYWRIGHT_BROWSERS_PATH: root }, "darwin"),
+ [
+ join(root, "ffmpeg-1011", "ffmpeg-mac"),
+ join(root, "ffmpeg-1009", "ffmpeg-mac"),
+ ],
+ );
+ assert.deepEqual(
+ playwrightFfmpegPaths({ XDG_CACHE_HOME: root }, "linux"),
+ [],
+ );
+ const mp4 = ffmpegArgs("mp4", {
+ fps: 10,
+ width: 320,
+ height: 700,
+ out: "o.mp4",
+ });
+ assert.equal(mp4[mp4.indexOf("-i") + 1], "pipe:0");
+ assert.ok(mp4.includes("libx264") && mp4.at(-1) === "o.mp4");
+ assert.match(mp4[mp4.indexOf("-vf") + 1], /pad=320:700:.*format=yuv420p$/);
+ assert.ok(
+ ffmpegArgs("webm", {
+ fps: 10,
+ width: 2,
+ height: 2,
+ out: "o.webm",
+ }).includes("vp8"),
+ );
+});
+
+const RED_16X16_JPEG =
+ "/9j/4AAQSkZJRgABAgAAAQABAAD//gAQTGF2YzYwLjMxLjEwMgD/2wBDAAgUFBcUFxsbGxsbGyAeICEhISAgICAhISEkJCQqKiokJCQhISQkKCgqKi4vLisrKisvLzIyMjw8OTlGRkhWVmf/xABMAAEBAAAAAAAAAAAAAAAAAAAABgEBAQAAAAAAAAAAAAAAAAAABgcQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCAAQABADASIAAhEAAxEA/9oADAMBAAIRAxEAPwCLAFF/f//Z";
+const BLUE_16X12_JPEG =
+ "/9j/4AAQSkZJRgABAgAAAQABAAD//gAQTGF2YzYwLjMxLjEwMgD/2wBDAAgUFBcUFxsbGxsbGyAeICEhISAgICAhISEkJCQqKiokJCQhISQkKCgqKi4vLisrKisvLzIyMjw8OTlGRkhWVmf/xABMAAEBAAAAAAAAAAAAAAAAAAAABwEBAQAAAAAAAAAAAAAAAAAABQcQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCAAMABADASIAAhEAAxEA/9oADAMBAAIRAxEAPwCOAL+Lf//Z";
+
+test.skipIf(!findFfmpeg())(
+ "recorded frames of two sizes encode at the real timing",
+ async () => {
+ const dir = mkdtempSync(join(tmpdir(), "record-"));
+ const frames = [
+ [RED_16X16_JPEG, 0, 16, 16],
+ [BLUE_16X12_JPEG, 1500, 16, 12],
+ ].map(([data, t, width, height], i) => {
+ const file = join(dir, `frame-${i}.jpg`);
+ writeFileSync(file, Buffer.from(data, "base64"));
+ return { file, t, width, height };
+ });
+ const ffmpeg = findFfmpeg();
+ const out = join(dir, `out.${ffmpeg.format}`);
+ const written = await encodeRecording({
+ ffmpeg,
+ frames,
+ durations: frameDurations(frames, 2000),
+ fps: 10,
+ ...canvasSize(frames),
+ out,
+ });
+ // 1 s lead-in (cut from 1.5 s) + 1 s hold on the last frame.
+ assert.equal(written, 20);
+ assert.ok(readFileSync(out).length > 0);
+ },
+);
+
+test("browser help documents record", () => {
+ const help = spawnSync(process.execPath, [cli, "help", "browser"], {
+ encoding: "utf8",
+ });
+ assert.equal(help.status, 0);
+ assert.match(help.stdout, /record start --feature ID --name N/);
+ assert.match(help.stdout, /record pause \| record resume/);
+ assert.match(help.stdout, /record stop \[--gif\]/);
+ const bad = spawnSync(
+ process.execPath,
+ [cli, "browser", "record", "start", "--name", "x", "--run", tmpdir()],
+ {
+ encoding: "utf8",
+ },
+ );
+ assert.notEqual(bad.status, 0);
+});
diff --git a/.agents/skills/verify-openhands/scripts/lib/recording.mjs b/.agents/skills/verify-openhands/scripts/lib/recording.mjs
new file mode 100644
index 0000000000..6b18957c4d
--- /dev/null
+++ b/.agents/skills/verify-openhands/scripts/lib/recording.mjs
@@ -0,0 +1,266 @@
+// Timeline and encoding for `browser record`.
+//
+// The daemon polls screenshots: a headless screencast emits frames only while
+// something animates, so a single repaint (a menu closing on Escape) would
+// never arrive. It keeps a frame only when its bytes differ from
+// the previous one, so each kept frame shows the page from its capture time
+// until the next change. On stop the timeline is resampled to a constant frame
+// rate and piped as JPEG into ffmpeg. An ffmpeg with libx264 (the usual system
+// build) writes MP4 and can add a GIF. Playwright's bundled ffmpeg encodes only
+// VP8, so with it the recording is WebM.
+
+import { spawn, spawnSync } from "node:child_process";
+import { once } from "node:events";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export const RECORD_DEFAULTS = { fps: 10, maxSeconds: 600, gifWidth: 960 };
+
+/**
+ * Take paused time out of the timeline (`browser record pause|resume`).
+ * @param {{ t: number }[]} frames capture times in ms; none falls in a pause
+ * @param {number} endT when capture stopped, in ms
+ * @param {{ at: number, ms: number }[]} cuts each pause's start and length
+ */
+export function withoutPauses(frames, endT, cuts) {
+ const shift = (t) =>
+ t - cuts.filter((c) => c.at + c.ms <= t).reduce((sum, c) => sum + c.ms, 0);
+ return {
+ frames: frames.map((frame) => ({ ...frame, t: shift(frame.t) })),
+ endT: shift(endT),
+ };
+}
+
+/**
+ * Seconds each kept frame stays on screen.
+ * @param {{ t: number }[]} frames capture times in ms, ascending
+ * @param {number} endT when capture stopped, in ms
+ * The still before the first change is cut to `leadIn` (when the recording
+ * started is arbitrary), and the last frame is held for at least `hold`.
+ * Anything that animates (a running conversation's pulsing dot) is a change,
+ * so long waits are cut with pause/resume rather than detected.
+ */
+export function frameDurations(frames, endT, { leadIn = 1, hold = 1 } = {}) {
+ return frames.map((frame, i) => {
+ const last = i === frames.length - 1;
+ const next = last ? endT : frames[i + 1].t;
+ let seconds = Math.max(0, (next - frame.t) / 1000);
+ if (i === 0 && !last) seconds = Math.min(seconds, leadIn);
+ if (last) seconds = Math.max(seconds, hold);
+ return seconds;
+ });
+}
+
+/**
+ * How many times to repeat each frame at `fps`. Frame ends are rounded on the
+ * running total, so the video does not drift from the real timeline, and every
+ * kept frame is shown at least once.
+ */
+export function frameRepeats(durations, fps) {
+ let total = 0;
+ let shown = 0;
+ return durations.map((seconds) => {
+ total += seconds;
+ const n = Math.max(1, Math.round(total * fps) - shown);
+ shown += n;
+ return n;
+ });
+}
+
+/** The output size: the largest frame, with even sides (yuv420p needs them). */
+export function canvasSize(frames) {
+ const width = Math.max(...frames.map((f) => f.width || 0));
+ const height = Math.max(...frames.map((f) => f.height || 0));
+ return { width: width + (width % 2), height: height + (height % 2) };
+}
+
+/** What an ffmpeg build can write, from its `-encoders` listing. */
+export function encoderSupport(encoders) {
+ const has = (name) =>
+ new RegExp(`^\\s*V\\S*\\s+${name}\\s`, "m").test(encoders);
+ if (has("libx264")) return { format: "mp4", gif: has("gif") };
+ if (has("libvpx")) return { format: "webm", gif: has("gif") };
+ return null;
+}
+
+/** Where Playwright installs its ffmpeg (the browsers directory). */
+export function playwrightFfmpegPaths(
+ env = process.env,
+ platform = process.platform,
+ home = homedir(),
+) {
+ const exe = {
+ linux: "ffmpeg-linux",
+ darwin: "ffmpeg-mac",
+ win32: "ffmpeg-win64.exe",
+ }[platform];
+ if (!exe) return [];
+ const cache =
+ platform === "darwin"
+ ? join(home, "Library", "Caches")
+ : platform === "win32"
+ ? env.LOCALAPPDATA || join(home, "AppData", "Local")
+ : env.XDG_CACHE_HOME || join(home, ".cache");
+ const root =
+ env.PLAYWRIGHT_BROWSERS_PATH && env.PLAYWRIGHT_BROWSERS_PATH !== "0"
+ ? env.PLAYWRIGHT_BROWSERS_PATH
+ : join(cache, "ms-playwright");
+ if (!existsSync(root)) return [];
+ return readdirSync(root)
+ .filter((dir) => dir.startsWith("ffmpeg-"))
+ .sort()
+ .reverse()
+ .map((dir) => join(root, dir, exe));
+}
+
+/**
+ * The first usable ffmpeg: $CONTROL_OPENHANDS_FFMPEG, `ffmpeg` on PATH, then
+ * Playwright's bundled build. Returns { path, format, gif } or null.
+ */
+export function findFfmpeg(env = process.env) {
+ const candidates = [
+ env.CONTROL_OPENHANDS_FFMPEG,
+ "ffmpeg",
+ ...playwrightFfmpegPaths(env),
+ ].filter(Boolean);
+ for (const path of candidates) {
+ const probe = spawnSync(path, ["-hide_banner", "-encoders"], {
+ encoding: "utf8",
+ env,
+ timeout: 10_000,
+ });
+ if (probe.status !== 0) continue;
+ const support = encoderSupport(probe.stdout);
+ if (support) return { path, ...support };
+ }
+ return null;
+}
+
+export function ffmpegArgs(format, { fps, width, height, out }) {
+ const fit = `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`;
+ const input = [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-y",
+ "-f",
+ "image2pipe",
+ "-c:v",
+ "mjpeg",
+ "-framerate",
+ String(fps),
+ "-i",
+ "pipe:0", // Playwright's build has no "-" shorthand
+ "-an",
+ ];
+ if (format === "mp4")
+ return [
+ ...input,
+ "-vf",
+ `${fit},format=yuv420p`,
+ "-c:v",
+ "libx264",
+ "-preset",
+ "veryfast",
+ "-crf",
+ "23",
+ "-movflags",
+ "+faststart",
+ out,
+ ];
+ // Playwright's own recorder settings for its VP8-only build.
+ return [
+ ...input,
+ "-vf",
+ fit,
+ "-c:v",
+ "vp8",
+ "-qmin",
+ "0",
+ "-qmax",
+ "50",
+ "-crf",
+ "8",
+ "-deadline",
+ "realtime",
+ "-speed",
+ "8",
+ "-b:v",
+ "1M",
+ out,
+ ];
+}
+
+/** Pipe the frames into ffmpeg at a constant rate; resolves to the frame count. */
+export async function encodeRecording({
+ ffmpeg,
+ frames,
+ durations,
+ fps,
+ width,
+ height,
+ out,
+}) {
+ const repeats = frameRepeats(durations, fps);
+ const child = spawn(
+ ffmpeg.path,
+ ffmpegArgs(ffmpeg.format, { fps, width, height, out }),
+ { stdio: ["pipe", "ignore", "pipe"] },
+ );
+ let stderr = "";
+ child.stderr.on("data", (chunk) => (stderr += chunk));
+ // A write after ffmpeg exits fails with EPIPE; its stderr says why.
+ child.stdin.on("error", () => {});
+ let closed = false;
+ const done = new Promise((resolve, reject) => {
+ child.on("error", reject);
+ child.on("close", (code) => {
+ closed = true;
+ if (code === 0) resolve();
+ else
+ reject(
+ new Error(`ffmpeg exited ${code}: ${stderr.trim().slice(-600)}`),
+ );
+ });
+ });
+ const exited = done.catch(() => {});
+ for (const [i, frame] of frames.entries()) {
+ const data = readFileSync(frame.file);
+ for (let n = 0; n < repeats[i] && !closed; n += 1)
+ if (!child.stdin.write(data))
+ await Promise.race([
+ once(child.stdin, "drain").catch(() => {}),
+ exited,
+ ]);
+ }
+ child.stdin.end();
+ await done;
+ return repeats.reduce((sum, n) => sum + n, 0);
+}
+
+/** A GIF from the MP4, with its own palette, at most `width` wide. */
+export async function videoToGif({ ffmpeg, video, out, fps, width }) {
+ const child = spawn(
+ ffmpeg.path,
+ [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-y",
+ "-i",
+ video,
+ "-vf",
+ `fps=${fps},scale='min(${width},iw)':-2:flags=lanczos,split[a][b];[a]palettegen=max_colors=128[p];[b][p]paletteuse`,
+ out,
+ ],
+ { stdio: ["ignore", "ignore", "pipe"] },
+ );
+ let stderr = "";
+ child.stderr.on("data", (chunk) => (stderr += chunk));
+ const [code] = await once(child, "close");
+ if (code !== 0)
+ throw new Error(
+ `ffmpeg (gif) exited ${code}: ${stderr.trim().slice(-600)}`,
+ );
+}