Desktop app that records your on-screen work session and uses the GitHub Copilot SDK to reconstruct it as an intent + ordered steps, then builds a reusable Skill or Automation for Microsoft Autopilots like Scout, Microsoft Copilot Cowork, or Copilot Studio.
TypeScript agent-skillsai-agentsautomationcopilotcopilot-clicopilot-coworkcopilot-studioelectronmicrosoft-scoutscreen-recording
Remove legacy ffmpeg frame fallback (#69)
adilei712384b6401a
diff --git a/electron/describer/describer.ts b/electron/describer/describer.tsindex 7f88289..53a228e 100644--- a/electron/describer/describer.ts+++ b/electron/describer/describer.ts@@ -1,7 +1,6 @@import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";import path from "node:path";-import { CAPTURED_FRAME_MANIFEST_VERSION } from "../../common/frames";import { approveAll, CopilotClient, type CopilotSession } from "@github/copilot-sdk";import {@@ -39,11 +38,9 @@ const NUDGE_PROMPT =const msg = (err: unknown) => (err instanceof Error ? err.message : String(err));interface VideoMeta {- file: string;startEpoch: number;durationMs: number;framesFile?: string;- framesVersion?: number;}/** A live, resumable analysis session for one recording. */@@ -372,17 +369,10 @@ export class Describer {function buildExtractor(sessionDir: string): FrameExtractor | null {const video = readJson<VideoMeta>(path.join(sessionDir, "video.json"));if (!video) return null;- const videoPath = path.join(sessionDir, video.file);- const capturedFramesPath = video.framesFile- ? path.join(sessionDir, video.framesFile)- : undefined;- const hasVideo = existsSync(videoPath);- const hasCapturedFrames = Boolean(capturedFramesPath && existsSync(capturedFramesPath));- if (!hasVideo && !hasCapturedFrames) return null;+ const capturedFramesPath = video.framesFile && path.join(sessionDir, video.framesFile);+ if (!capturedFramesPath || !existsSync(capturedFramesPath)) return null;return new FrameExtractor({- ...(hasVideo ? { videoPath } : {}),- ...(hasCapturedFrames ? { capturedFramesPath } : {}),- capturedFramesExpected: video.framesVersion === CAPTURED_FRAME_MANIFEST_VERSION,+ capturedFramesPath,framesDir: path.join(sessionDir, "frames"),anchorEpochMs: video.startEpoch,durationSec: video.durationMs > 0 ? video.durationMs / 1000 : undefined,diff --git a/electron/frames/extractor.ts b/electron/frames/extractor.tsindex 43bb184..c3c64c7 100644--- a/electron/frames/extractor.ts+++ b/electron/frames/extractor.ts@@ -1,10 +1,8 @@-import { execFile, execFileSync } from "node:child_process";import { randomUUID } from "node:crypto";import { existsSync, readFileSync, writeFileSync } from "node:fs";import { copyFile, unlink } from "node:fs/promises";import { createRequire } from "node:module";import path from "node:path";-import { promisify } from "node:util";import {CAPTURED_FRAME_MANIFEST_VERSION,@@ -16,7 +14,6 @@ import { createLogger } from "../logger";const log = createLogger("Frames");const require = createRequire(import.meta.url);-const execFileAsync = promisify(execFile);type Sharp = (typeof import("sharp"))["default"];let sharpMod: Sharp | null | undefined;@@ -37,12 +34,8 @@ interface SourceFrame extends CapturedVideoFrame {}export interface ExtractorOptions {- /** Retained for pre-change recordings that need a system-FFmpeg fallback. */- videoPath?: string;/** Absolute path to the versioned source-frame manifest. */capturedFramesPath?: string;- /** Suppress the system-FFmpeg legacy path for current recordings with no snapshots. */- capturedFramesExpected?: boolean;framesDir: string;/** `video.json` startEpoch — the wall-clock anchor for offset↔epoch mapping. */anchorEpochMs: number;@@ -128,12 +121,10 @@ export function sampleCapturedFrames<T extends CapturedVideoFrame>(/*** Extracts sparse, visually distinct JPEGs from the snapshots captured alongside- * the WebM. Pre-change sessions can still use a user-installed FFmpeg, but the- * application no longer downloads or distributes any FFmpeg binary.+ * the WebM.*/export class FrameExtractor {private readonly opts: {- videoPath?: string;capturedFramesPath?: string;framesDir: string;anchorEpochMs: number;@@ -144,9 +135,7 @@ export class FrameExtractor {};private readonly manifestPath: string;private readonly sourceFrames: SourceFrame[];- private readonly usesCapturedFramePipeline: boolean;private frames: FrameRecord[] = [];- private warnedLegacy = false;constructor(opts: ExtractorOptions) {this.opts = {@@ -156,9 +145,6 @@ export class FrameExtractor {frameGridSec: opts.frameGridSec ?? DEFAULTS.frameGridSec,};this.manifestPath = path.join(opts.framesDir, "frames.json");- this.usesCapturedFramePipeline =- opts.capturedFramesExpected === true ||- Boolean(opts.capturedFramesPath && existsSync(opts.capturedFramesPath));this.sourceFrames = loadSourceFrames(opts.capturedFramesPath);if (existsSync(this.manifestPath)) {this.frames = loadRetainedFrames(this.manifestPath);@@ -193,17 +179,12 @@ export class FrameExtractor {if (seen.has(cell)) continue;seen.add(cell);try {- const record =- this.sourceFrames.length > 0- ? await this.extractCapturedAt(- event.tMs,- source,- event.reason,- seenCapturedFiles,- )- : this.usesCapturedFramePipeline- ? null- : await this.extractLegacySingle(offsetSec, source, event.reason);+ const record = await this.extractCapturedAt(+ event.tMs,+ source,+ event.reason,+ seenCapturedFiles,+ );if (record) added.push(record);} catch (err) {log.warn(`frame extraction at ${offsetSec.toFixed(2)}s failed:`, message(err));@@ -214,9 +195,7 @@ export class FrameExtractor {}async extractWindow(req: WindowRequest): Promise<FrameRecord[]> {- if (this.sourceFrames.length === 0) {- return this.usesCapturedFramePipeline ? [] : this.extractLegacyWindow(req);- }+ if (this.sourceFrames.length === 0) return [];const startMs = Math.max(this.opts.anchorEpochMs, req.startMs);const endMs = Math.max(startMs, req.endMs);@@ -327,102 +306,6 @@ export class FrameExtractor {return true;}- private async extractLegacyWindow(req: WindowRequest): Promise<FrameRecord[]> {- const ffmpegPath = this.legacyFfmpegPath();- const videoPath = this.opts.videoPath;- if (!ffmpegPath || !videoPath) return [];-- const fps = req.fps ?? DEFAULT_WINDOW_FPS;- const startSec = this.offsetForEpoch(req.startMs);- const endSec = Math.max(startSec, this.offsetForEpoch(req.endMs));- const cap = req.maxFrames ?? DEFAULT_WINDOW_MAX_FRAMES;- const stamp = randomUUID();- const pattern = path.join(this.opts.framesDir, `probe_${stamp}_%04d.jpg`);- const filters = [`fps=${fps}`];- if (req.crop) filters.push(`crop=${req.crop.w}:${req.crop.h}:${req.crop.x}:${req.crop.y}`);-- try {- await execFileAsync(- ffmpegPath,- [- "-hide_banner",- "-ss", startSec.toFixed(3),- "-to", endSec.toFixed(3),- "-i", videoPath,- "-vf", filters.join(","),- "-vsync", "vfr",- "-frames:v", String(cap),- "-q:v", "3",- pattern,- ],- { maxBuffer: 32 * 1024 * 1024 },- );- } catch (err) {- log.warn("legacy probe window failed:", message(err));- return [];- }-- const added: FrameRecord[] = [];- for (let index = 1; index <= cap; index++) {- const file = path.join(- this.opts.framesDir,- `probe_${stamp}_${String(index).padStart(4, "0")}.jpg`,- );- if (!existsSync(file)) break;- const offsetSec = startSec + (index - 1) / fps;- const record = await this.keepOrDrop(- file,- offsetSec,- "probe",- req.reason ?? "probe:legacy-system-ffmpeg",- );- if (record) added.push(record);- }- this.persist();- return added;- }-- private async extractLegacySingle(- offsetSec: number,- source: FrameSource,- reason?: string,- ): Promise<FrameRecord | null> {- const ffmpegPath = this.legacyFfmpegPath();- const videoPath = this.opts.videoPath;- if (!ffmpegPath || !videoPath) return null;- const file = path.join(this.opts.framesDir, `${source}_${Math.round(offsetSec * 1000)}.jpg`);- try {- await execFileAsync(- ffmpegPath,- [- "-hide_banner",- "-i", videoPath,- "-ss", offsetSec.toFixed(3),- "-frames:v", "1",- "-q:v", "3",- "-y", file,- ],- { maxBuffer: 16 * 1024 * 1024 },- );- } catch (err) {- log.warn(`legacy frame at ${offsetSec.toFixed(2)}s failed:`, message(err));- return null;- }- if (!existsSync(file)) return null;- return this.keepOrDrop(file, offsetSec, source, reason);- }-- private legacyFfmpegPath(): string | null {- const resolved = systemFfmpegPath();- if (!resolved && !this.warnedLegacy) {- this.warnedLegacy = true;- log.warn(- "This recording predates captured source frames. Install FFmpeg to extract legacy video frames.",- );- }- return resolved;- }-private async keepOrDrop(file: string,offsetSec: number,@@ -566,21 +449,6 @@ function finitePositiveNumber(value: unknown): value is number {return finiteNumber(value) && value > 0;}-let ffmpegPath: string | null | undefined;-function systemFfmpegPath(): string | null {- if (ffmpegPath !== undefined) return ffmpegPath;- try {- const finder = process.platform === "win32" ? "where" : "which";- ffmpegPath =- execFileSync(finder, ["ffmpeg"], { encoding: "utf8" })- .trim()- .split(/\r?\n/)[0] || null;- } catch {- ffmpegPath = null;- }- return ffmpegPath;-}-async function dhash(file: string): Promise<string> {const image = sharp();if (!image) return "";diff --git a/electron/pipeline.test.ts b/electron/pipeline.test.tsnew file mode 100644index 0000000..2ac486e--- /dev/null+++ b/electron/pipeline.test.ts@@ -0,0 +1,113 @@+import assert from "node:assert/strict";+import { existsSync } from "node:fs";+import {+ chmod,+ copyFile,+ link,+ mkdir,+ mkdtemp,+ readFile,+ rm,+ writeFile,+} from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";++import type { SessionBundle } from "../common/bundle";+import type { RecEvent, SessionMeta } from "../common/types";+import { processSession } from "./pipeline";++test("processSession skips legacy video-only frame extraction", async (t) => {+ const root = await mkdtemp(path.join(tmpdir(), "skill-recorder-pipeline-"));+ t.after(() => rm(root, { recursive: true, force: true }));++ const startedAt = 1_000;+ const meta: SessionMeta = {+ id: "legacy-video-only",+ startedAt,+ stoppedAt: startedAt + 1_000,+ platform: process.platform,+ appVersion: "0.6.0-test",+ };+ const event: RecEvent = {+ seq: 1,+ t: 100,+ epoch: startedAt + 100,+ type: "app.activate",+ source: "test",+ payload: { app: "Example", title: "Legacy recording" },+ };+ await Promise.all([+ writeFile(path.join(root, "session.json"), JSON.stringify(meta)),+ writeFile(path.join(root, "events.jsonl"), `${JSON.stringify(event)}\n`),+ writeFile(+ path.join(root, "video.json"),+ JSON.stringify({+ file: "screen.webm",+ startEpoch: startedAt,+ durationMs: 1_000,+ }),+ ),+ writeFile(path.join(root, "screen.webm"), "legacy video placeholder"),+ ]);++ const lookupMarker = path.join(root, "ffmpeg-lookup-attempted");+ const restoreLookup = await installFfmpegLookupTrap(root, lookupMarker);++ try {+ await processSession(root);+ } finally {+ restoreLookup();+ }++ assert.equal(existsSync(lookupMarker), false, "legacy ffmpeg discovery must not run");+ const bundle = JSON.parse(+ await readFile(path.join(root, "bundle.json"), "utf8"),+ ) as SessionBundle;+ assert.equal(bundle.stats.frameCount, 0);+ assert.equal(bundle.stats.stepCount, 1);+ assert.equal(existsSync(path.join(root, "description.md")), true);+});++function shellQuote(value: string): string {+ return `'${value.replaceAll("'", "'\\''")}'`;+}++async function installFfmpegLookupTrap(root: string, marker: string): Promise<() => void> {+ const binDir = path.join(root, "bin");+ await mkdir(binDir);+ const previousPath = process.env.PATH;+ const previousNodeOptions = process.env.NODE_OPTIONS;++ if (process.platform === "win32") {+ const preload = path.join(root, "mark-ffmpeg-lookup.cjs");+ await writeFile(+ preload,+ `require("node:fs").writeFileSync(${JSON.stringify(marker)}, "called");\n`,+ );+ const fakeWhere = path.join(binDir, "where.exe");+ try {+ await link(process.execPath, fakeWhere);+ } catch {+ await copyFile(process.execPath, fakeWhere);+ }+ const requirePreload = `--require=${JSON.stringify(preload)}`;+ process.env.NODE_OPTIONS = [previousNodeOptions, requirePreload].filter(Boolean).join(" ");+ } else {+ const fakeWhich = path.join(binDir, "which");+ await writeFile(+ fakeWhich,+ `#!/bin/sh\nprintf called > ${shellQuote(marker)}\nexit 1\n`,+ );+ await chmod(fakeWhich, 0o755);+ }+ process.env.PATH = `${binDir}${path.delimiter}${previousPath ?? ""}`;++ return () => {+ if (previousPath === undefined) delete process.env.PATH;+ else process.env.PATH = previousPath;+ if (previousNodeOptions === undefined) delete process.env.NODE_OPTIONS;+ else process.env.NODE_OPTIONS = previousNodeOptions;+ };+}diff --git a/electron/pipeline.ts b/electron/pipeline.tsindex 1149833..a84605e 100644--- a/electron/pipeline.ts+++ b/electron/pipeline.ts@@ -5,7 +5,6 @@ import { buildBundle } from "../common/bundle";import { MEANINGFUL_EVENT_TYPES } from "../common/correlation";import type { CorrelationResult } from "../common/correlation";import { renderDescription } from "../common/describe";-import { CAPTURED_FRAME_MANIFEST_VERSION } from "../common/frames";import type { SessionMeta } from "../common/types";import { CorrelationEngine, readEvents } from "./frames/correlate";import { FrameExtractor } from "./frames/extractor";@@ -26,11 +25,11 @@ function readMeta(sessionDir: string): SessionMeta | null {}/**- * Opportunistic video stage: if a video was recorded, grab one frame at each+ * Opportunistic frame stage: if source snapshots were captured, grab one at each* meaningful non-video event and correlate. We deliberately DO NOT scan the whole- * video — events are the primary signal; anything they miss is surfaced as probe+ * recording — events are the primary signal; anything they miss is surfaced as probe* *suggestions* and harvested later, only where confidence is low. Best-effort:- * returns null when there's no usable video.+ * returns null when there are no usable captured frames.*/async function runFrameStage(sessionDir: string): Promise<CorrelationResult | null> {const videoJsonPath = path.join(sessionDir, "video.json");@@ -44,21 +43,14 @@ async function runFrameStage(sessionDir: string): Promise<CorrelationResult | nureturn null;}- const videoPath = path.join(sessionDir, video.file);- const capturedFramesPath = video.framesFile- ? path.join(sessionDir, video.framesFile)- : undefined;- const hasVideo = existsSync(videoPath);- const hasCapturedFrames = Boolean(capturedFramesPath && existsSync(capturedFramesPath));- if (!hasVideo && !hasCapturedFrames) {- log.warn("video and captured frames missing; skipping frame stage");+ const capturedFramesPath = video.framesFile && path.join(sessionDir, video.framesFile);+ if (!capturedFramesPath || !existsSync(capturedFramesPath)) {+ log.warn("captured frames missing; skipping frame stage");return null;}const extractor = new FrameExtractor({- ...(hasVideo ? { videoPath } : {}),- ...(hasCapturedFrames ? { capturedFramesPath } : {}),- capturedFramesExpected: video.framesVersion === CAPTURED_FRAME_MANIFEST_VERSION,+ capturedFramesPath,framesDir: path.join(sessionDir, "frames"),anchorEpochMs: video.startEpoch,durationSec: video.durationMs > 0 ? video.durationMs / 1000 : undefined,@@ -85,7 +77,7 @@ async function runFrameStage(sessionDir: string): Promise<CorrelationResult | nu/*** Post-stop processing for a completed session. Always produces `bundle.json`* (segmented steps) and `description.md` (baseline narrative) from the primary- * event stream; enriches them with correlated frames when a video is present.+ * event stream; enriches them with correlated captured frames when available.* Strictly best-effort — never throws into the recorder.*/export async function processSession(sessionDir: string): Promise<void> {diff --git a/package.json b/package.jsonindex 9bb88cd..23b9223 100644--- a/package.json+++ b/package.json@@ -19,7 +19,7 @@"check:lockfile": "node scripts/check-lockfile-portability.mjs","typecheck": "tsc --noEmit","typecheck:evals": "tsc --noEmit -p evals/tsconfig.json",- "test": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs --test evals/builder-imports.test.ts common/architecture-registry.test.ts electron/architectures/catalogue-registry.test.ts common/audio.test.ts common/microphone.test.ts common/screen.test.ts common/narration.test.ts common/sensitive.test.ts common/skill.test.ts electron/recording-controls-bounds.test.ts electron/recorder-window-sizing.test.ts electron/recording-privacy.test.ts electron/crash-guards.test.ts electron/recorder/controller.test.ts electron/recorder/session-store.test.ts electron/frames/extractor.test.ts electron/narration/audio-analysis.test.ts electron/narration/analyze-gate.test.ts electron/narration/transcribe.test.ts electron/narration/whisper.test.ts electron/sensitive/scanner.test.ts electron/sensitive/secrets.test.ts electron/sensitive/tessdata-source.test.ts electron/sensitive/ocr.test.ts electron/sensitive/frame-redact.test.ts electron/sensitive/frame-heuristics.test.ts electron/terminal/terminal.test.ts electron/sessions.test.ts electron/debug-bundle.test.ts electron/skillbuilder/plan-tools.test.ts electron/skillbuilder/placement.test.ts src/skill-placement.test.ts scripts/compliance.test.mjs",+ "test": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs --test evals/builder-imports.test.ts common/architecture-registry.test.ts electron/architectures/catalogue-registry.test.ts common/audio.test.ts common/microphone.test.ts common/screen.test.ts common/narration.test.ts common/sensitive.test.ts common/skill.test.ts electron/recording-controls-bounds.test.ts electron/recorder-window-sizing.test.ts electron/recording-privacy.test.ts electron/crash-guards.test.ts electron/recorder/controller.test.ts electron/recorder/session-store.test.ts electron/frames/extractor.test.ts electron/pipeline.test.ts electron/narration/audio-analysis.test.ts electron/narration/analyze-gate.test.ts electron/narration/transcribe.test.ts electron/narration/whisper.test.ts electron/sensitive/scanner.test.ts electron/sensitive/secrets.test.ts electron/sensitive/tessdata-source.test.ts electron/sensitive/ocr.test.ts electron/sensitive/frame-redact.test.ts electron/sensitive/frame-heuristics.test.ts electron/terminal/terminal.test.ts electron/sessions.test.ts electron/debug-bundle.test.ts electron/skillbuilder/plan-tools.test.ts electron/skillbuilder/placement.test.ts src/skill-placement.test.ts scripts/compliance.test.mjs","eval": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/run.ts","eval:terminal": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/run.ts --only=terminal-output-required,terminal-failure-recovery,terminal-sensitive-output","eval:builder": "node --experimental-transform-types --no-warnings --import ./evals/register.mjs evals/builder/run.ts",