Skip to content

Desktop app that records your on-screen work session and uses the GitHub Copilot SDK to reconstruct it as an intent + ordered steps, then builds a reusable Skill or Automation for Microsoft Autopilots like Scout, Microsoft Copilot Cowork, or Copilot Studio.

Remove legacy ffmpeg frame fallback (#69)

adilei712384b6401a
diff --git a/electron/describer/describer.ts b/electron/describer/describer.ts
index 7f88289..53a228e 100644
--- a/electron/describer/describer.ts
+++ b/electron/describer/describer.ts
@@ -1,7 +1,6 @@
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
import path from "node:path";
-import { CAPTURED_FRAME_MANIFEST_VERSION } from "../../common/frames";
import { approveAll, CopilotClient, type CopilotSession } from "@github/copilot-sdk";
import {
@@ -39,11 +38,9 @@ const NUDGE_PROMPT =
const msg = (err: unknown) => (err instanceof Error ? err.message : String(err));
interface VideoMeta {
- file: string;
startEpoch: number;
durationMs: number;
framesFile?: string;
- framesVersion?: number;
}
/** A live, resumable analysis session for one recording. */
@@ -372,17 +369,10 @@ export class Describer {
function buildExtractor(sessionDir: string): FrameExtractor | null {
const video = readJson<VideoMeta>(path.join(sessionDir, "video.json"));
if (!video) return null;
- const videoPath = path.join(sessionDir, video.file);
- const capturedFramesPath = video.framesFile
- ? path.join(sessionDir, video.framesFile)
- : undefined;
- const hasVideo = existsSync(videoPath);
- const hasCapturedFrames = Boolean(capturedFramesPath && existsSync(capturedFramesPath));
- if (!hasVideo && !hasCapturedFrames) return null;
+ const capturedFramesPath = video.framesFile && path.join(sessionDir, video.framesFile);
+ if (!capturedFramesPath || !existsSync(capturedFramesPath)) return null;
return new FrameExtractor({
- ...(hasVideo ? { videoPath } : {}),
- ...(hasCapturedFrames ? { capturedFramesPath } : {}),
- capturedFramesExpected: video.framesVersion === CAPTURED_FRAME_MANIFEST_VERSION,
+ capturedFramesPath,
framesDir: path.join(sessionDir, "frames"),
anchorEpochMs: video.startEpoch,
durationSec: video.durationMs > 0 ? video.durationMs / 1000 : undefined,
diff --git a/electron/frames/extractor.ts b/electron/frames/extractor.ts
index 43bb184..c3c64c7 100644
--- a/electron/frames/extractor.ts
+++ b/electron/frames/extractor.ts
@@ -1,10 +1,8 @@
-import { execFile, execFileSync } from "node:child_process";
import { randomUUID } from "node:crypto";
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { copyFile, unlink } from "node:fs/promises";
import { createRequire } from "node:module";
import path from "node:path";
-import { promisify } from "node:util";
import {
CAPTURED_FRAME_MANIFEST_VERSION,
@@ -16,7 +14,6 @@ import { createLogger } from "../logger";
const log = createLogger("Frames");
const require = createRequire(import.meta.url);
-const execFileAsync = promisify(execFile);
type Sharp = (typeof import("sharp"))["default"];
let sharpMod: Sharp | null | undefined;
@@ -37,12 +34,8 @@ interface SourceFrame extends CapturedVideoFrame {
}
export interface ExtractorOptions {
- /** Retained for pre-change recordings that need a system-FFmpeg fallback. */
- videoPath?: string;
/** Absolute path to the versioned source-frame manifest. */
capturedFramesPath?: string;
- /** Suppress the system-FFmpeg legacy path for current recordings with no snapshots. */
- capturedFramesExpected?: boolean;
framesDir: string;
/** `video.json` startEpoch — the wall-clock anchor for offset↔epoch mapping. */
anchorEpochMs: number;
@@ -128,12 +121,10 @@ export function sampleCapturedFrames<T extends CapturedVideoFrame>(
/**
* Extracts sparse, visually distinct JPEGs from the snapshots captured alongside
- * the WebM. Pre-change sessions can still use a user-installed FFmpeg, but the
- * application no longer downloads or distributes any FFmpeg binary.
+ * the WebM.
*/
export class FrameExtractor {
private readonly opts: {
- videoPath?: string;
capturedFramesPath?: string;
framesDir: string;
anchorEpochMs: number;
@@ -144,9 +135,7 @@ export class FrameExtractor {
};
private readonly manifestPath: string;
private readonly sourceFrames: SourceFrame[];
- private readonly usesCapturedFramePipeline: boolean;
private frames: FrameRecord[] = [];
- private warnedLegacy = false;
constructor(opts: ExtractorOptions) {
this.opts = {
@@ -156,9 +145,6 @@ export class FrameExtractor {
frameGridSec: opts.frameGridSec ?? DEFAULTS.frameGridSec,
};
this.manifestPath = path.join(opts.framesDir, "frames.json");
- this.usesCapturedFramePipeline =
- opts.capturedFramesExpected === true ||
- Boolean(opts.capturedFramesPath && existsSync(opts.capturedFramesPath));
this.sourceFrames = loadSourceFrames(opts.capturedFramesPath);
if (existsSync(this.manifestPath)) {
this.frames = loadRetainedFrames(this.manifestPath);
@@ -193,17 +179,12 @@ export class FrameExtractor {
if (seen.has(cell)) continue;
seen.add(cell);
try {
- const record =
- this.sourceFrames.length > 0
- ? await this.extractCapturedAt(
- event.tMs,
- source,
- event.reason,
- seenCapturedFiles,
- )
- : this.usesCapturedFramePipeline
- ? null
- : await this.extractLegacySingle(offsetSec, source, event.reason);
+ const record = await this.extractCapturedAt(
+ event.tMs,
+ source,
+ event.reason,
+ seenCapturedFiles,
+ );
if (record) added.push(record);
} catch (err) {
log.warn(`frame extraction at ${offsetSec.toFixed(2)}s failed:`, message(err));
@@ -214,9 +195,7 @@ export class FrameExtractor {
}
async extractWindow(req: WindowRequest): Promise<FrameRecord[]> {
- if (this.sourceFrames.length === 0) {
- return this.usesCapturedFramePipeline ? [] : this.extractLegacyWindow(req);
- }
+ if (this.sourceFrames.length === 0) return [];
const startMs = Math.max(this.opts.anchorEpochMs, req.startMs);
const endMs = Math.max(startMs, req.endMs);
@@ -327,102 +306,6 @@ export class FrameExtractor {
return true;
}
- private async extractLegacyWindow(req: WindowRequest): Promise<FrameRecord[]> {
- const ffmpegPath = this.legacyFfmpegPath();
- const videoPath = this.opts.videoPath;
- if (!ffmpegPath || !videoPath) return [];
-
- const fps = req.fps ?? DEFAULT_WINDOW_FPS;
- const startSec = this.offsetForEpoch(req.startMs);
- const endSec = Math.max(startSec, this.offsetForEpoch(req.endMs));
- const cap = req.maxFrames ?? DEFAULT_WINDOW_MAX_FRAMES;
- const stamp = randomUUID();
- const pattern = path.join(this.opts.framesDir, `probe_${stamp}_%04d.jpg`);
- const filters = [`fps=${fps}`];
- if (req.crop) filters.push(`crop=${req.crop.w}:${req.crop.h}:${req.crop.x}:${req.crop.y}`);
-
- try {
- await execFileAsync(
- ffmpegPath,
- [
- "-hide_banner",
- "-ss", startSec.toFixed(3),
- "-to", endSec.toFixed(3),
- "-i", videoPath,
- "-vf", filters.join(","),
- "-vsync", "vfr",
- "-frames:v", String(cap),
- "-q:v", "3",
- pattern,
- ],
- { maxBuffer: 32 * 1024 * 1024 },
- );
- } catch (err) {
- log.warn("legacy probe window failed:", message(err));
- return [];
- }
-
- const added: FrameRecord[] = [];
- for (let index = 1; index <= cap; index++) {
- const file = path.join(
- this.opts.framesDir,
- `probe_${stamp}_${String(index).padStart(4, "0")}.jpg`,
- );
- if (!existsSync(file)) break;
- const offsetSec = startSec + (index - 1) / fps;
- const record = await this.keepOrDrop(
- file,
- offsetSec,
- "probe",
- req.reason ?? "probe:legacy-system-ffmpeg",
- );
- if (record) added.push(record);
- }
- this.persist();
- return added;
- }
-
- private async extractLegacySingle(
- offsetSec: number,
- source: FrameSource,
- reason?: string,
- ): Promise<FrameRecord | null> {
- const ffmpegPath = this.legacyFfmpegPath();
- const videoPath = this.opts.videoPath;
- if (!ffmpegPath || !videoPath) return null;
- const file = path.join(this.opts.framesDir, `${source}_${Math.round(offsetSec * 1000)}.jpg`);
- try {
- await execFileAsync(
- ffmpegPath,
- [
- "-hide_banner",
- "-i", videoPath,
- "-ss", offsetSec.toFixed(3),
- "-frames:v", "1",
- "-q:v", "3",
- "-y", file,
- ],
- { maxBuffer: 16 * 1024 * 1024 },
- );
- } catch (err) {
- log.warn(`legacy frame at ${offsetSec.toFixed(2)}s failed:`, message(err));
- return null;
- }
- if (!existsSync(file)) return null;
- return this.keepOrDrop(file, offsetSec, source, reason);
- }
-
- private legacyFfmpegPath(): string | null {
- const resolved = systemFfmpegPath();
- if (!resolved && !this.warnedLegacy) {
- this.warnedLegacy = true;
- log.warn(
- "This recording predates captured source frames. Install FFmpeg to extract legacy video frames.",
- );
- }
- return resolved;
- }
-
private async keepOrDrop(
file: string,
offsetSec: number,
@@ -566,21 +449,6 @@ function finitePositiveNumber(value: unknown): value is number {
return finiteNumber(value) && value > 0;
}
-let ffmpegPath: string | null | undefined;
-function systemFfmpegPath(): string | null {
- if (ffmpegPath !== undefined) return ffmpegPath;
- try {
- const finder = process.platform === "win32" ? "where" : "which";
- ffmpegPath =
- execFileSync(finder, ["ffmpeg"], { encoding: "utf8" })
- .trim()
- .split(/\r?\n/)[0] || null;
- } catch {
- ffmpegPath = null;
- }
- return ffmpegPath;
-}
-
async function dhash(file: string): Promise<string> {
const image = sharp();
if (!image) return "";
diff --git a/electron/pipeline.test.ts b/electron/pipeline.test.ts
new file mode 100644
index 0000000..2ac486e
--- /dev/null
+++ b/electron/pipeline.test.ts
@@ -0,0 +1,113 @@
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import {
+ chmod,
+ copyFile,
+ link,
+ mkdir,
+ mkdtemp,
+ readFile,
+ rm,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import type { SessionBundle } from "../common/bundle";
+import type { RecEvent, SessionMeta } from "../common/types";
+import { processSession } from "./pipeline";
+
+test("processSession skips legacy video-only frame extraction", async (t) => {
+ const root = await mkdtemp(path.join(tmpdir(), "skill-recorder-pipeline-"));
+ t.after(() => rm(root, { recursive: true, force: true }));
+
+ const startedAt = 1_000;
+ const meta: SessionMeta = {
+ id: "legacy-video-only",
+ startedAt,
+ stoppedAt: startedAt + 1_000,
+ platform: process.platform,
+ appVersion: "0.6.0-test",
+ };
+ const event: RecEvent = {
+ seq: 1,
+ t: 100,
+ epoch: startedAt + 100,
+ type: "app.activate",
+ source: "test",
+ payload: { app: "Example", title: "Legacy recording" },
+ };
+ await Promise.all([
+ writeFile(path.join(root, "session.json"), JSON.stringify(meta)),
+ writeFile(path.join(root, "events.jsonl"), `${JSON.stringify(event)}\n`),
+ writeFile(
+ path.join(root, "video.json"),
+ JSON.stringify({
+ file: "screen.webm",
+ startEpoch: startedAt,
+ durationMs: 1_000,
+ }),
+ ),
+ writeFile(path.join(root, "screen.webm"), "legacy video placeholder"),
+ ]);
+
+ const lookupMarker = path.join(root, "ffmpeg-lookup-attempted");
+ const restoreLookup = await installFfmpegLookupTrap(root, lookupMarker);
+
+ try {
+ await processSession(root);
+ } finally {
+ restoreLookup();
+ }
+
+ assert.equal(existsSync(lookupMarker), false, "legacy ffmpeg discovery must not run");
+ const bundle = JSON.parse(
+ await readFile(path.join(root, "bundle.json"), "utf8"),
+ ) as SessionBundle;
+ assert.equal(bundle.stats.frameCount, 0);
+ assert.equal(bundle.stats.stepCount, 1);
+ assert.equal(existsSync(path.join(root, "description.md")), true);
+});
+
+function shellQuote(value: string): string {
+ return `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+async function installFfmpegLookupTrap(root: string, marker: string): Promise<() => void> {
+ const binDir = path.join(root, "bin");
+ await mkdir(binDir);
+ const previousPath = process.env.PATH;
+ const previousNodeOptions = process.env.NODE_OPTIONS;
+
+ if (process.platform === "win32") {
+ const preload = path.join(root, "mark-ffmpeg-lookup.cjs");
+ await writeFile(
+ preload,
+ `require("node:fs").writeFileSync(${JSON.stringify(marker)}, "called");\n`,
+ );
+ const fakeWhere = path.join(binDir, "where.exe");
+ try {
+ await link(process.execPath, fakeWhere);
+ } catch {
+ await copyFile(process.execPath, fakeWhere);
+ }
+ const requirePreload = `--require=${JSON.stringify(preload)}`;
+ process.env.NODE_OPTIONS = [previousNodeOptions, requirePreload].filter(Boolean).join(" ");
+ } else {
+ const fakeWhich = path.join(binDir, "which");
+ await writeFile(
+ fakeWhich,
+ `#!/bin/sh\nprintf called > ${shellQuote(marker)}\nexit 1\n`,
+ );
+ await chmod(fakeWhich, 0o755);
+ }
+ process.env.PATH = `${binDir}${path.delimiter}${previousPath ?? ""}`;
+
+ return () => {
+ if (previousPath === undefined) delete process.env.PATH;
+ else process.env.PATH = previousPath;
+ if (previousNodeOptions === undefined) delete process.env.NODE_OPTIONS;
+ else process.env.NODE_OPTIONS = previousNodeOptions;
+ };
+}
diff --git a/electron/pipeline.ts b/electron/pipeline.ts
index 1149833..a84605e 100644
--- a/electron/pipeline.ts
+++ b/electron/pipeline.ts
@@ -5,7 +5,6 @@ import { buildBundle } from "../common/bundle";
import { MEANINGFUL_EVENT_TYPES } from "../common/correlation";
import type { CorrelationResult } from "../common/correlation";
import { renderDescription } from "../common/describe";
-import { CAPTURED_FRAME_MANIFEST_VERSION } from "../common/frames";
import type { SessionMeta } from "../common/types";
import { CorrelationEngine, readEvents } from "./frames/correlate";
import { FrameExtractor } from "./frames/extractor";
@@ -26,11 +25,11 @@ function readMeta(sessionDir: string): SessionMeta | null {
}
/**
- * Opportunistic video stage: if a video was recorded, grab one frame at each
+ * Opportunistic frame stage: if source snapshots were captured, grab one at each
* meaningful non-video event and correlate. We deliberately DO NOT scan the whole
- * video — events are the primary signal; anything they miss is surfaced as probe
+ * recording — events are the primary signal; anything they miss is surfaced as probe
* *suggestions* and harvested later, only where confidence is low. Best-effort:
- * returns null when there's no usable video.
+ * returns null when there are no usable captured frames.
*/
async function runFrameStage(sessionDir: string): Promise<CorrelationResult | null> {
const videoJsonPath = path.join(sessionDir, "video.json");
@@ -44,21 +43,14 @@ async function runFrameStage(sessionDir: string): Promise<CorrelationResult | nu
return null;
}
- const videoPath = path.join(sessionDir, video.file);
- const capturedFramesPath = video.framesFile
- ? path.join(sessionDir, video.framesFile)
- : undefined;
- const hasVideo = existsSync(videoPath);
- const hasCapturedFrames = Boolean(capturedFramesPath && existsSync(capturedFramesPath));
- if (!hasVideo && !hasCapturedFrames) {
- log.warn("video and captured frames missing; skipping frame stage");
+ const capturedFramesPath = video.framesFile && path.join(sessionDir, video.framesFile);
+ if (!capturedFramesPath || !existsSync(capturedFramesPath)) {
+ log.warn("captured frames missing; skipping frame stage");
return null;
}
const extractor = new FrameExtractor({
- ...(hasVideo ? { videoPath } : {}),
- ...(hasCapturedFrames ? { capturedFramesPath } : {}),
- capturedFramesExpected: video.framesVersion === CAPTURED_FRAME_MANIFEST_VERSION,
+ capturedFramesPath,
framesDir: path.join(sessionDir, "frames"),
anchorEpochMs: video.startEpoch,
durationSec: video.durationMs > 0 ? video.durationMs / 1000 : undefined,
@@ -85,7 +77,7 @@ async function runFrameStage(sessionDir: string): Promise<CorrelationResult | nu
/**
* Post-stop processing for a completed session. Always produces `bundle.json`
* (segmented steps) and `description.md` (baseline narrative) from the primary
- * event stream; enriches them with correlated frames when a video is present.
+ * event stream; enriches them with correlated captured frames when available.
* Strictly best-effort — never throws into the recorder.
*/
export async function processSession(sessionDir: string): Promise<void> {
diff --git a/package.json b/package.json
index 9bb88cd..23b9223 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,7 @@
"check:lockfile": "node scripts/check-lockfile-portability.mjs",
"typecheck": "tsc --noEmit",
"typecheck:evals": "tsc --noEmit -p evals/tsconfig.json",
- "test": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs --test evals/builder-imports.test.ts common/architecture-registry.test.ts electron/architectures/catalogue-registry.test.ts common/audio.test.ts common/microphone.test.ts common/screen.test.ts common/narration.test.ts common/sensitive.test.ts common/skill.test.ts electron/recording-controls-bounds.test.ts electron/recorder-window-sizing.test.ts electron/recording-privacy.test.ts electron/crash-guards.test.ts electron/recorder/controller.test.ts electron/recorder/session-store.test.ts electron/frames/extractor.test.ts electron/narration/audio-analysis.test.ts electron/narration/analyze-gate.test.ts electron/narration/transcribe.test.ts electron/narration/whisper.test.ts electron/sensitive/scanner.test.ts electron/sensitive/secrets.test.ts electron/sensitive/tessdata-source.test.ts electron/sensitive/ocr.test.ts electron/sensitive/frame-redact.test.ts electron/sensitive/frame-heuristics.test.ts electron/terminal/terminal.test.ts electron/sessions.test.ts electron/debug-bundle.test.ts electron/skillbuilder/plan-tools.test.ts electron/skillbuilder/placement.test.ts src/skill-placement.test.ts scripts/compliance.test.mjs",
+ "test": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs --test evals/builder-imports.test.ts common/architecture-registry.test.ts electron/architectures/catalogue-registry.test.ts common/audio.test.ts common/microphone.test.ts common/screen.test.ts common/narration.test.ts common/sensitive.test.ts common/skill.test.ts electron/recording-controls-bounds.test.ts electron/recorder-window-sizing.test.ts electron/recording-privacy.test.ts electron/crash-guards.test.ts electron/recorder/controller.test.ts electron/recorder/session-store.test.ts electron/frames/extractor.test.ts electron/pipeline.test.ts electron/narration/audio-analysis.test.ts electron/narration/analyze-gate.test.ts electron/narration/transcribe.test.ts electron/narration/whisper.test.ts electron/sensitive/scanner.test.ts electron/sensitive/secrets.test.ts electron/sensitive/tessdata-source.test.ts electron/sensitive/ocr.test.ts electron/sensitive/frame-redact.test.ts electron/sensitive/frame-heuristics.test.ts electron/terminal/terminal.test.ts electron/sessions.test.ts electron/debug-bundle.test.ts electron/skillbuilder/plan-tools.test.ts electron/skillbuilder/placement.test.ts src/skill-placement.test.ts scripts/compliance.test.mjs",
"eval": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/run.ts",
"eval:terminal": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/run.ts --only=terminal-output-required,terminal-failure-recovery,terminal-sensitive-output",
"eval:builder": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/builder/run.ts",