Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(cli): openscreen captions writes a transcript and turns captions on

EtienneLescot7aefe3afbab2
diff --git a/electron/cli/args.ts b/electron/cli/args.ts
index 9b621b660..6c1323010 100644
--- a/electron/cli/args.ts
+++ b/electron/cli/args.ts
@@ -84,7 +84,7 @@ Usage:
openscreen record [options] Record the screen headlessly
openscreen sources [--json] [-o <file>] List displays, windows, microphones and cameras
openscreen pack <project.openscreen> --out <dir> Copy project + media into one portable folder
- openscreen captions <project.openscreen> Add auto-captions (on-device Whisper) to a project
+ openscreen captions <project.openscreen> Transcribe (on-device Whisper) and turn captions on
[--min-words <1-12>] [--max-words <1-12>]
openscreen info <project.openscreen> [--json] Inspect a project file
openscreen mcp Serve the agent tools over MCP on stdio
diff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsx
index a87f729e2..046637440 100644
--- a/src/cli/CliCaptionsRunner.tsx
+++ b/src/cli/CliCaptionsRunner.tsx
@@ -1,149 +1,58 @@
-// Hidden-window runner for `openscreen captions`: transcribes the project's
-// audio with the on-device Whisper worker and writes the resulting caption
-// annotations back into the project. Mirrors VideoEditor.generateAutoCaptions.
+// Hidden-window runner for `openscreen captions`: transcribes the project's primary media
+// with the editor's own on-device Whisper path (`transcribeAsset`) and turns its captions
+// on (`captionProject.ts`). The main process writes the project back.
-import { useEffect, useRef, useState } from "react";
-import {
- normalizeProjectEditor,
- resolveProjectMedia,
- toFileUrl,
- validateProjectData,
-} from "@/components/video-editor/projectPersistence";
-import type { AnnotationRegion, TrimRegion } from "@/components/video-editor/types";
-import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k";
-import { type SttRendererStatus, transcribeMono16kToSegments } from "@/lib/captioning/transcribe";
+import { useEffect, useRef } from "react";
+import { type TranscribeStatus, transcribeAsset } from "@/lib/ai-edition/document/transcribe";
+import { classifyTranscriptionError } from "@/lib/ai-edition/transcription/status";
import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts";
-import { nativeBridgeClient } from "@/native";
-import { captionSegmentsToAnnotationRegions } from "./captionAnnotations";
-import {
- shiftTrimRegionsMsForCaptionBuffer,
- trimLeadingSilenceMono16k,
-} from "./vendor/leadingSilence";
-
-/** Highest trailing number across existing region ids, so new ids never collide. */
-function nextNumericIdFrom(regions: { id: string }[]): number {
- let max = 0;
- for (const region of regions) {
- const match = /(\d+)$/.exec(region.id);
- if (match) max = Math.max(max, Number(match[1]));
- }
- return max + 1;
-}
+import { captionProject } from "./captionProject";
+import { loadCliProject } from "./loadCliProject";
async function runCaptions(request: CliCaptionsRequest): Promise<CliDoneResult> {
- const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath);
- if (!loaded.success || loaded.project === undefined) {
- throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file");
- }
- if (!validateProjectData(loaded.project)) {
- throw new Error("Project file is not a valid .openscreen project");
- }
- const project = loaded.project;
- const media = resolveProjectMedia(project);
- if (!media) {
- throw new Error("Project file does not reference any recorded media");
- }
- const editor = normalizeProjectEditor(project.editor ?? {});
- const trimRegions: TrimRegion[] = editor.trimRegions;
-
- window.electronAPI.cliLog("info", "Extracting audio…");
- const videoUrl = toFileUrl(media.screenVideoPath);
- const { samples, durationSec } = await extractMono16kFromVideoUrl(videoUrl);
- if (!Number.isFinite(durationSec) || durationSec <= 0 || samples.length < 800) {
- throw new Error("The project's video has no usable audio track to transcribe");
- }
-
- const { samples: speechSamples, trimSec } = trimLeadingSilenceMono16k(samples);
- if (speechSamples.length < 800) {
- throw new Error("No speech detected in the project's audio");
- }
-
- const trimMs = Math.round(trimSec * 1000);
- const trimRegionsForTranscribe = shiftTrimRegionsMsForCaptionBuffer(trimRegions, trimMs);
-
- // `onStatus` now fires once per transcribed chunk, not once per phase, so log
- // only on a phase change — otherwise a long transcription spams the CLI with
- // one identical line per chunk.
- let loggedPhase: SttRendererStatus["phase"] | null = null;
- const transcribeOptions = {
- onStatus: ({ phase }: SttRendererStatus) => {
- if (phase === loggedPhase) return;
- loggedPhase = phase;
- window.electronAPI.cliLog(
- "info",
- phase === "model" ? "Loading caption model…" : "Transcribing…",
- );
- },
+ const { document } = await loadCliProject(request.projectPath);
+
+ // `onStatus` fires once per transcribed chunk, so log only on a phase change: a long
+ // transcription would otherwise print one identical line per chunk.
+ let loggedPhase: TranscribeStatus["phase"] | null = null;
+ const onStatus = ({ phase }: TranscribeStatus) => {
+ if (phase === loggedPhase) return;
+ loggedPhase = phase;
+ window.electronAPI.cliLog(
+ "info",
+ phase === "extracting-audio"
+ ? "Extracting audio…"
+ : phase === "loading-model"
+ ? "Loading the speech model…"
+ : "Transcribing…",
+ );
};
- let { segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(speechSamples, {
- trimRegions: trimRegionsForTranscribe,
- ...transcribeOptions,
- });
- let transcribedFromTrimmedBuffer = true;
-
- // Leading-silence trimming can return empty even when the full source has
- // speech. Retry once against the untrimmed buffer before giving up.
- if (segmentsRaw.length === 0 && trimSec > 0) {
- ({ segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(samples, {
- trimRegions,
- ...transcribeOptions,
- }));
- transcribedFromTrimmedBuffer = false;
- }
-
- const segments =
- transcribedFromTrimmedBuffer && trimSec > 0
- ? segmentsRaw.map((segment) => ({
- ...segment,
- startSec: segment.startSec + trimSec,
- endSec: segment.endSec + trimSec,
- }))
- : segmentsRaw;
-
- // Re-running the command replaces earlier auto-captions instead of stacking
- // duplicates; manually added annotations are preserved.
- const manualAnnotations: AnnotationRegion[] = editor.annotationRegions.filter(
- (annotation) => annotation.annotationSource !== "auto-caption",
+ const result = await captionProject(
+ document,
+ {
+ minWordsPerLine: request.minWordsPerCaption,
+ maxWordsPerLine: request.maxWordsPerCaption,
+ },
+ (doc, assetId) =>
+ transcribeAsset(doc, assetId, { onStatus }).catch((error: unknown) => {
+ throw new Error(`Transcription failed: ${classifyTranscriptionError(error).message}`);
+ }),
);
- const startNumericId = nextNumericIdFrom([...editor.annotationRegions, ...editor.zoomRegions]);
- const startZIndex = manualAnnotations.reduce((max, a) => Math.max(max, a.zIndex + 1), 1);
-
- let regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, {
- minWordsPerCaption: request.minWordsPerCaption,
- maxWordsPerCaption: request.maxWordsPerCaption,
- timestampGranularity: granularity,
- });
- if (regions.length === 0 && segments.length > 0) {
- regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, {
- minWordsPerCaption: 1,
- maxWordsPerCaption: Number.MAX_SAFE_INTEGER,
- timestampGranularity: granularity,
- });
- }
- if (regions.length === 0) {
- throw new Error("Transcription produced no caption segments");
+ if (!result.transcribed) {
+ window.electronAPI.cliLog("info", "Using the transcript the project already has.");
}
- const updatedProject = {
- ...project,
- editor: {
- ...editor,
- annotationRegions: [...manualAnnotations, ...regions],
- },
- };
-
return {
success: true,
projectPath: request.projectPath,
- projectData: updatedProject,
- captionCount: regions.length,
+ projectData: result.document,
+ captionCount: result.captionCount,
};
}
export function CliCaptionsRunner() {
const startedRef = useRef(false);
- const [status] = useState("Generating captions…");
useEffect(() => {
if (startedRef.current) return;
@@ -166,7 +75,7 @@ export function CliCaptionsRunner() {
return (
<div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm">
- {status}
+ Generating captions…
</div>
);
}
diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx
index e206e1b83..685f80f14 100644
--- a/src/cli/CliExportRunner.tsx
+++ b/src/cli/CliExportRunner.tsx
@@ -6,22 +6,14 @@
// builds the clip list + scene JSON and relays progress.
import { useEffect, useRef, useState } from "react";
-import {
- type EditorProjectData,
- normalizeProjectEditor,
- resolveProjectMedia,
- toFileUrl,
- validateProjectData,
-} from "@/components/video-editor/projectPersistence";
+import { toFileUrl } from "@/components/video-editor/projectPersistence";
import { prepareClickSound } from "@/lib/ai-edition/clickSound";
-import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate";
import {
collectEffectiveClipDims,
pickExtremeDims,
resolveAspectRatioValue,
} from "@/lib/ai-edition/document/outputFormat";
-import { applyProbedDuration } from "@/lib/ai-edition/document/timeline";
-import { type AxcutDocument, isAxcutDocumentFile } from "@/lib/ai-edition/schema";
+import type { AxcutDocument } from "@/lib/ai-edition/schema";
import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings";
import {
appendAutoZoomSuggestions,
@@ -38,38 +30,7 @@ import { exportGifNative, exportMultiNative, nativeBridgeClient } from "@/native
import type { CompositorClipInput } from "@/native/contracts";
import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription";
import { type ExportProject, loadDocumentProject } from "./exportProject";
-
-function probeVideoDimensions(
- url: string,
-): Promise<{ width: number; height: number; durationMs: number }> {
- return new Promise((resolve, reject) => {
- const video = document.createElement("video");
- video.preload = "metadata";
- video.muted = true;
- const cleanup = () => {
- clearTimeout(timer);
- video.removeAttribute("src");
- video.load();
- };
- // A stalled load fires neither event; without a deadline the CLI hangs.
- const timer = setTimeout(() => {
- cleanup();
- reject(new Error(`Timed out reading video metadata: ${url}`));
- }, 30_000);
- video.onloadedmetadata = () => {
- const width = video.videoWidth;
- const height = video.videoHeight;
- const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0;
- cleanup();
- resolve({ width, height, durationMs });
- };
- video.onerror = () => {
- cleanup();
- reject(new Error(`Failed to load video metadata: ${url}`));
- };
- video.src = url;
- });
-}
+import { loadCliProject, probeVideoDimensions } from "./loadCliProject";
function replaceExtension(filePath: string, newExtension: string): string {
return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension;
@@ -115,126 +76,18 @@ function gifOutputDims(
return { width: even(sourceDims.width), height: even(sourceDims.height) };
}
-/**
- * A legacy v2 project (`{ version, media, editor }`), as written before the editor saved the
- * document itself: migrated to an AxcutDocument, with what the migration cannot know probed.
- */
-async function loadLegacyProject(project: EditorProjectData): Promise<ExportProject> {
- const media = resolveProjectMedia(project);
- if (!media) {
- throw new Error("Project file does not reference any recorded media");
- }
- // Prefer the main process's approved session paths: they carry the
- // packed-project sibling fallback when the stored absolute paths are stale.
- try {
- const sessionResult = await window.electronAPI.getCurrentRecordingSession();
- const session = sessionResult?.session;
- if (session?.screenVideoPath) {
- media.screenVideoPath = session.screenVideoPath;
- if (media.webcamVideoPath && session.webcamVideoPath) {
- media.webcamVideoPath = session.webcamVideoPath;
- }
- }
- } catch {
- // Fall back to the paths stored in the project file.
- }
- const editor = normalizeProjectEditor(project.editor ?? {});
-
- const probed = await probeVideoDimensions(toFileUrl(media.screenVideoPath));
-
- // Migrate the .openscreen project onto the AxcutDocument the native
- // compositor consumes. The migration is pure and carries zooms, annotations,
- // trims and the legacy editor settings; the clip's duration is unknown until
- // probed, so applyProbedDuration must run or the export is a single frame.
- let axcutDocument = migrateProjectDataToAxcutDocument({
- ...project,
- media,
- editor,
- });
- const primaryAssetId = axcutDocument.project.primaryAssetId ?? axcutDocument.assets[0]?.id;
- if (!primaryAssetId) {
- throw new Error("Project migration produced no media asset");
- }
- if (probed.durationMs > 0) {
- axcutDocument = applyProbedDuration(axcutDocument, primaryAssetId, probed.durationMs / 1000);
- }
- // The screen's dimensions, for the same reason as the camera's below: the pure migration
- // cannot know them, and the scene's screen box, the output frame and Auto all read them off
- // `asset.video`. Without them the CLI laid every recording out as 1920x1080 while sizing the
- // export from the probe, two answers to one question.
- if (probed.width > 0 && probed.height > 0) {
- axcutDocument = {
- ...axcutDocument,
- assets: axcutDocument.assets.map((asset) =>
- asset.id === primaryAssetId
- ? {
- ...asset,
- video: {
- codec: "unknown",
- fps: 0,
- ...asset.video,
- width: probed.width,
- height: probed.height,
- },
- }
- : asset,
- ),
- };
- }
-
- // The camera's dimensions decide the PiP's layout box, and this is the one caller the
- // document cannot answer for: there is no editor session here to have probed and saved
- // them, so without this the CLI lays the box out from a hardcoded 4:3 and a 16:9 camera
- // exports framed differently from the same project opened in the app.
- //
- // Failure-tolerant on purpose, unlike the screen probe above which is allowed to reject:
- // a camera file that has gone missing should cost the export its camera, not the export.
- if (media.webcamVideoPath) {
- const camera = await probeVideoDimensions(toFileUrl(media.webcamVideoPath)).catch(() => null);
- if (camera) {
- axcutDocument = {
- ...axcutDocument,
- assets: axcutDocument.assets.map((asset) =>
- asset.cameraTrack
- ? {
- ...asset,
- cameraTrack: {
- ...asset.cameraTrack,
- width: camera.width,
- height: camera.height,
- },
- }
- : asset,
- ),
- };
- }
- }
- return {
- document: axcutDocument,
- editor,
- screenVideoPath: media.screenVideoPath,
- durationMs: probed.durationMs,
- sourceDims: { width: probed.width, height: probed.height },
- };
-}
-
/** The project to export: by id when the MCP server names one, else the file at its path. */
async function loadExportProject(request: CliExportRequest): Promise<ExportProject> {
- const probe = (videoPath: string) => probeVideoDimensions(toFileUrl(videoPath));
if (request.projectId) {
const stored = await nativeBridgeClient.aiEdition.get(request.projectId);
if (!stored.success || !stored.document) {
throw new Error(stored.error ?? `No project has the id "${request.projectId}"`);
}
- return loadDocumentProject(stored.document, probe);
- }
- const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath);
- if (!loaded.success || loaded.project === undefined) {
- throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file");
+ return loadDocumentProject(stored.document, (videoPath) =>
+ probeVideoDimensions(toFileUrl(videoPath)),
+ );
}
- if (isAxcutDocumentFile(loaded.project)) return loadDocumentProject(loaded.project, probe);
- if (validateProjectData(loaded.project)) return loadLegacyProject(loaded.project);
- throw new Error("Project file is not a valid .openscreen project");
+ return loadCliProject(request.projectPath);
}
async function runExport(request: CliExportRequest): Promise<CliDoneResult> {
diff --git a/src/cli/captionAnnotations.test.ts b/src/cli/captionAnnotations.test.ts
deleted file mode 100644
index 79f336a69..000000000
--- a/src/cli/captionAnnotations.test.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { DEFAULT_CAPTION_SETTINGS } from "@/lib/ai-edition/captions/settings";
-import type { CaptionSegment } from "@/lib/captioning/transcribe";
-import { captionSegmentsToAnnotationRegions } from "./captionAnnotations";
-
-const words = (...texts: string[]): CaptionSegment[] =>
- texts.map((text, i) => ({ text, startSec: i * 0.5, endSec: i * 0.5 + 0.4 }));
-
-describe("captionSegmentsToAnnotationRegions", () => {
- it("numbers ids and z-indexes from the given start, with non-empty spans", () => {
- const regions = captionSegmentsToAnnotationRegions(words("one", "two", "three", "four"), 7, 3);
-
- expect(regions.length).toBeGreaterThan(0);
- expect(regions.map((r) => r.id)).toEqual(regions.map((_, i) => `annotation-${7 + i}`));
- expect(regions.map((r) => r.zIndex)).toEqual(regions.map((_, i) => 3 + i));
- for (const region of regions) {
- expect(region.endMs).toBeGreaterThan(region.startMs);
- expect(region.annotationSource).toBe("auto-caption");
- expect(region.content.trim()).not.toBe("");
- }
- });
-
- it("groups phrase-granularity segments one line at a time", () => {
- const regions = captionSegmentsToAnnotationRegions(
- [
- { text: "hello there", startSec: 0, endSec: 1 },
- { text: "second line", startSec: 1.2, endSec: 2 },
- ],
- 1,
- 1,
- { timestampGranularity: "phrase" },
- );
-
- expect(regions.map((r) => r.content)).toEqual(["hello there", "second line"]);
- });
-
- it("looks like the editor's captions, not a faint annotation", () => {
- const [region] = captionSegmentsToAnnotationRegions(words("one", "two"), 1, 1);
- expect(region.style).toMatchObject({
- fontSize: DEFAULT_CAPTION_SETTINGS.fontSize,
- fontWeight: DEFAULT_CAPTION_SETTINGS.fontWeight,
- color: DEFAULT_CAPTION_SETTINGS.color,
- backgroundColor: "rgba(0, 0, 0, 0.55)",
- });
- // The editor's caption box: 16:9 safe column, bottom edge 1.5% off the frame's, room for
- // a caption that wraps. The text is bottom-anchored in it (see sceneDescription).
- expect(region.position.x).toBe(16);
- expect(region.size.width).toBe(68);
- expect(region.position.y + region.size.height).toBeCloseTo(98.5);
- // Three 48 px lines of 1.5 em plus the plate padding: a wrapped caption is not clipped.
- expect(region.size.height).toBeCloseTo(((48 * (3 * 1.5 + 0.2)) / 1080) * 100);
- });
-});
diff --git a/src/cli/captionAnnotations.ts b/src/cli/captionAnnotations.ts
deleted file mode 100644
index 91e3baa96..000000000
--- a/src/cli/captionAnnotations.ts
+++ /dev/null
@@ -1,84 +0,0 @@
-// CLI-only: the v4 editor renders captions natively, but `openscreen captions`
-// still writes caption *annotations* into .openscreen projects. Only the
-// annotation conversion lives here — the segment grouping/dedupe helpers come
-// from the live captioning module so the CLI can't drift from the editor.
-
-import type { AnnotationRegion, AnnotationTextStyle } from "@/components/video-editor/types";
-import {
- captionBackgroundCss,
- captionBoxRect,
- DEFAULT_CAPTION_SETTINGS,
-} from "@/lib/ai-edition/captions/settings";
-import {
- type CaptionSegmentLayoutOptions,
- dedupeAdjacentCaptionRepeats,
- finalizeCaptionSegmentsForPlayback,
- groupPhraseCaptionSegmentsIntoLines,
- groupTimedCaptionWordsIntoLines,
-} from "@/lib/captioning/annotationsFromCaptions";
-import type { CaptionSegment } from "@/lib/captioning/transcribe";
-
-// The look and place of the editor's own captions (`DEFAULT_CAPTION_SETTINGS`): bold 48 px
-// white on a 55% black plate, in the landscape safe column, just off the bottom edge. The CLI
-// used to write 24 px regular text on no plate, which read as faint over any bright frame.
-// `.openscreen` projects carry no transcript, so the CLI cannot write the editor's transcript-
-// driven captions; matching their appearance is what keeps the two from looking different.
-//
-// Placed with the editor's own caption box: room for three lines, pinned by its BOTTOM edge
-// 1.5% off the frame's. The scene description bottom-anchors `auto-caption` text in it, so
-// the plate ends at the inset whether the caption holds one line or wraps to three.
-const CAPTION_BOX = captionBoxRect(DEFAULT_CAPTION_SETTINGS, 16 / 9);
-
-const CAPTION_POSITION = { x: CAPTION_BOX.x, y: CAPTION_BOX.y };
-
-const CAPTION_SIZE = { width: CAPTION_BOX.width, height: CAPTION_BOX.height };
-
-const CAPTION_STYLE: AnnotationTextStyle = {
- color: DEFAULT_CAPTION_SETTINGS.color,
- backgroundColor: captionBackgroundCss(DEFAULT_CAPTION_SETTINGS),
- fontSize: DEFAULT_CAPTION_SETTINGS.fontSize,
- fontFamily: DEFAULT_CAPTION_SETTINGS.fontFamily,
- fontWeight: DEFAULT_CAPTION_SETTINGS.fontWeight,
- fontStyle: "normal",
- textDecoration: "none",
- textAlign: "center",
-};
-
-export function captionSegmentsToAnnotationRegions(
- segments: CaptionSegment[],
- startNumericId: number,
- startZIndex: number,
- layout?: CaptionSegmentLayoutOptions,
-): AnnotationRegion[] {
- // Don't echo-collapse raw word tokens before grouping: repeated words ("I … I") share a
- // normalized key and would merge spans while keeping only the first token's text.
- const minW = layout?.minWordsPerCaption ?? DEFAULT_CAPTION_SETTINGS.minWordsPerLine;
- const maxW = layout?.maxWordsPerCaption ?? DEFAULT_CAPTION_SETTINGS.maxWordsPerLine;
- const granularity = layout?.timestampGranularity ?? "word";
-
- const grouped =
- granularity === "phrase"
- ? groupPhraseCaptionSegmentsIntoLines(segments, minW, maxW)
- : groupTimedCaptionWordsIntoLines(segments, minW, maxW);
-
- const finalized = finalizeCaptionSegmentsForPlayback(dedupeAdjacentCaptionRepeats(grouped));
-
- let nid = startNumericId;
- let z = startZIndex;
- return finalized.map((seg) => {
- const startMs = Math.round(seg.startSec * 1000);
- const endMs = Math.max(Math.round(seg.endSec * 1000), startMs + 1);
- return {
- id: `annotation-${nid++}`,
- startMs,
- endMs,
- type: "text",
- content: seg.text,
- annotationSource: "auto-caption",
- position: { ...CAPTION_POSITION },
- size: { ...CAPTION_SIZE },
- style: { ...CAPTION_STYLE },
- zIndex: z++,
- };
- });
-}
diff --git a/src/cli/captionProject.test.ts b/src/cli/captionProject.test.ts
new file mode 100644
index 000000000..a7f09f032
--- /dev/null
+++ b/src/cli/captionProject.test.ts
@@ -0,0 +1,108 @@
+import { describe, expect, it, vi } from "vitest";
+import { getCaptionSettings } from "@/lib/ai-edition/captions";
+import {
+ type AxcutDocument,
+ type AxcutTranscript,
+ createEmptyDocument,
+ documentSchema,
+} from "@/lib/ai-edition/schema";
+import { captionProject } from "./captionProject";
+
+function project(extra: Partial<AxcutDocument> = {}): AxcutDocument {
+ const base = createEmptyDocument({
+ title: "CLI",
+ projectId: "proj_1",
+ createdAt: "2026-01-01T00:00:00.000Z",
+ });
+ return documentSchema.parse({
+ ...base,
+ project: { ...base.project, primaryAssetId: "asset_1" },
+ assets: [
+ { id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 10 },
+ ],
+ timeline: {
+ ...base.timeline,
+ clips: [
+ {
+ id: "clip_1",
+ assetId: "asset_1",
+ sourceStartSec: 0,
+ sourceEndSec: 10,
+ timelineStartSec: 0,
+ timelineEndSec: 10,
+ wordRefs: [],
+ origin: "user",
+ reason: "",
+ },
+ ],
+ },
+ ...extra,
+ });
+}
+
+function transcript(text: string): AxcutTranscript {
+ const words = text.split(" ");
+ return {
+ assetId: "asset_1",
+ language: "en",
+ segments: words.map((word, i) => ({
+ id: `seg_${i + 1}`,
+ kind: "speech" as const,
+ startSec: i,
+ endSec: i + 1,
+ text: word,
+ wordIds: [`word_${i + 1}`],
+ })),
+ words: words.map((word, i) => ({
+ id: `word_${i + 1}`,
+ segmentId: `seg_${i + 1}`,
+ startSec: i,
+ endSec: i + 1,
+ text: word,
+ })),
+ };
+}
+
+const LINE = { minWordsPerLine: 1, maxWordsPerLine: 2 };
+
+describe("captionProject", () => {
+ it("transcribes the primary media and turns the editor's captions on", async () => {
+ const transcribe = vi.fn(async () => transcript("one two three four"));
+ const result = await captionProject(project(), LINE, transcribe);
+
+ expect(transcribe).toHaveBeenCalledWith(expect.anything(), "asset_1");
+ expect(result.transcribed).toBe(true);
+ expect(result.document.transcripts).toHaveLength(1);
+ expect(getCaptionSettings(result.document)).toMatchObject({
+ enabled: true,
+ minWordsPerLine: 1,
+ maxWordsPerLine: 2,
+ });
+ expect(result.captionCount).toBe(2);
+ });
+
+ it("reuses a transcript the project has, and clears the old caption annotations", async () => {
+ const withLegacy = {
+ ...project({ transcripts: [transcript("hello there")] }),
+ annotations: [
+ { id: "annotation-1", annotationSource: "auto-caption" },
+ ] as unknown as AxcutDocument["annotations"],
+ };
+ const transcribe = vi.fn();
+ const result = await captionProject(withLegacy, LINE, transcribe);
+
+ expect(transcribe).not.toHaveBeenCalled();
+ expect(result.transcribed).toBe(false);
+ expect(result.document.annotations).toEqual([]);
+ });
+
+ it("fails on media with no speech, writing nothing", async () => {
+ const silent: AxcutTranscript = {
+ assetId: "asset_1",
+ language: "auto",
+ segments: [],
+ words: [],
+ };
+ await expect(captionProject(project(), LINE, async () => silent)).rejects.toThrow(/No speech/);
+ });
+});
diff --git a/src/cli/captionProject.ts b/src/cli/captionProject.ts
new file mode 100644
index 000000000..44ac48042
--- /dev/null
+++ b/src/cli/captionProject.ts
@@ -0,0 +1,59 @@
+// What `openscreen captions` does to a project, kept out of the runner window so it is
+// testable: the editor's own captions, not annotations. The primary media gets a
+// transcript (the one it has, or a fresh one), then the captions are turned on with the
+// requested line length, as the Captions pane would. They are the transcript drawn at
+// render time, so `openscreen export` burns them in and the editor shows them editable.
+
+import {
+ deriveCaptionCues,
+ getCaptionSettings,
+ getCaptionTranslations,
+ patchCaptionSettings,
+} from "@/lib/ai-edition/captions";
+import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat";
+import { applyTranscriptionResult } from "@/lib/ai-edition/document/transcript";
+import type { AxcutDocument, AxcutTranscript } from "@/lib/ai-edition/schema";
+import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings";
+import { findAssetTranscript, transcriptHasSpeech } from "@/lib/ai-edition/transcription/status";
+
+export interface CaptionProjectOptions {
+ minWordsPerLine: number;
+ maxWordsPerLine: number;
+}
+
+export async function captionProject(
+ document: AxcutDocument,
+ options: CaptionProjectOptions,
+ transcribe: (document: AxcutDocument, assetId: string) => Promise<AxcutTranscript>,
+): Promise<{ document: AxcutDocument; captionCount: number; transcribed: boolean }> {
+ const assetId = document.project.primaryAssetId ?? document.assets[0]?.id;
+ if (!assetId) throw new Error("Project file does not reference any recorded media");
+ const existing = findAssetTranscript(document, assetId);
+ const transcript = existing ?? (await transcribe(document, assetId));
+ if (!transcriptHasSpeech(transcript)) {
+ throw new Error("No speech detected in the project's audio");
+ }
+ const transcribed = existing ? document : applyTranscriptionResult(document, transcript).document;
+ // The v1 caption annotations this command used to write: the Captions pane offers to
+ // clear them, and next to real captions they would show every line twice.
+ const cleared: AxcutDocument = {
+ ...transcribed,
+ annotations: transcribed.annotations.filter((a) => a.annotationSource !== "auto-caption"),
+ };
+ const aspect = resolveAspectRatioValue(cleared, getEditorSettings(cleared).aspectRatio);
+ const next = patchCaptionSettings(
+ cleared,
+ {
+ enabled: true,
+ minWordsPerLine: options.minWordsPerLine,
+ maxWordsPerLine: options.maxWordsPerLine,
+ },
+ aspect,
+ );
+ const captionCount = deriveCaptionCues(
+ next,
+ getCaptionSettings(next, aspect),
+ getCaptionTranslations(next),
+ ).length;
+ return { document: next, captionCount, transcribed: !existing };
+}
diff --git a/src/cli/loadCliProject.ts b/src/cli/loadCliProject.ts
new file mode 100644
index 000000000..395514348
--- /dev/null
+++ b/src/cli/loadCliProject.ts
@@ -0,0 +1,166 @@
+// What `openscreen export` and `openscreen captions` read from a project file, in the
+// hidden runner window: either shape the file was saved in, as one AxcutDocument. The
+// renderer half of `exportProject.ts`, since probing a video needs a DOM.
+
+import {
+ type EditorProjectData,
+ normalizeProjectEditor,
+ resolveProjectMedia,
+ toFileUrl,
+ validateProjectData,
+} from "@/components/video-editor/projectPersistence";
+import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate";
+import { applyProbedDuration } from "@/lib/ai-edition/document/timeline";
+import { isAxcutDocumentFile } from "@/lib/ai-edition/schema";
+import { nativeBridgeClient } from "@/native";
+import { type ExportProject, loadDocumentProject } from "./exportProject";
+
+export function probeVideoDimensions(
+ url: string,
+): Promise<{ width: number; height: number; durationMs: number }> {
+ return new Promise((resolve, reject) => {
+ const video = document.createElement("video");
+ video.preload = "metadata";
+ video.muted = true;
+ const cleanup = () => {
+ clearTimeout(timer);
+ video.removeAttribute("src");
+ video.load();
+ };
+ // A stalled load fires neither event; without a deadline the CLI hangs.
+ const timer = setTimeout(() => {
+ cleanup();
+ reject(new Error(`Timed out reading video metadata: ${url}`));
+ }, 30_000);
+ video.onloadedmetadata = () => {
+ const width = video.videoWidth;
+ const height = video.videoHeight;
+ const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0;
+ cleanup();
+ resolve({ width, height, durationMs });
+ };
+ video.onerror = () => {
+ cleanup();
+ reject(new Error(`Failed to load video metadata: ${url}`));
+ };
+ video.src = url;
+ });
+}
+
+/**
+ * A legacy v2 project (`{ version, media, editor }`), as written before the editor saved the
+ * document itself: migrated to an AxcutDocument, with what the migration cannot know probed.
+ */
+async function loadLegacyProject(project: EditorProjectData): Promise<ExportProject> {
+ const media = resolveProjectMedia(project);
+ if (!media) {
+ throw new Error("Project file does not reference any recorded media");
+ }
+ // Prefer the main process's approved session paths: they carry the
+ // packed-project sibling fallback when the stored absolute paths are stale.
+ try {
+ const sessionResult = await window.electronAPI.getCurrentRecordingSession();
+ const session = sessionResult?.session;
+ if (session?.screenVideoPath) {
+ media.screenVideoPath = session.screenVideoPath;
+ if (media.webcamVideoPath && session.webcamVideoPath) {
+ media.webcamVideoPath = session.webcamVideoPath;
+ }
+ }
+ } catch {
+ // Fall back to the paths stored in the project file.
+ }
+ const editor = normalizeProjectEditor(project.editor ?? {});
+
+ const probed = await probeVideoDimensions(toFileUrl(media.screenVideoPath));
+
+ // Migrate the .openscreen project onto the AxcutDocument the native
+ // compositor consumes. The migration is pure and carries zooms, annotations,
+ // trims and the legacy editor settings; the clip's duration is unknown until
+ // probed, so applyProbedDuration must run or the export is a single frame.
+ let axcutDocument = migrateProjectDataToAxcutDocument({
+ ...project,
+ media,
+ editor,
+ });
+ const primaryAssetId = axcutDocument.project.primaryAssetId ?? axcutDocument.assets[0]?.id;
+ if (!primaryAssetId) {
+ throw new Error("Project migration produced no media asset");
+ }
+ if (probed.durationMs > 0) {
+ axcutDocument = applyProbedDuration(axcutDocument, primaryAssetId, probed.durationMs / 1000);
+ }
+ // The screen's dimensions, for the same reason as the camera's below: the pure migration
+ // cannot know them, and the scene's screen box, the output frame and Auto all read them off
+ // `asset.video`. Without them the CLI laid every recording out as 1920x1080 while sizing the
+ // export from the probe, two answers to one question.
+ if (probed.width > 0 && probed.height > 0) {
+ axcutDocument = {
+ ...axcutDocument,
+ assets: axcutDocument.assets.map((asset) =>
+ asset.id === primaryAssetId
+ ? {
+ ...asset,
+ video: {
+ codec: "unknown",
+ fps: 0,
+ ...asset.video,
+ width: probed.width,
+ height: probed.height,
+ },
+ }
+ : asset,
+ ),
+ };
+ }
+
+ // The camera's dimensions decide the PiP's layout box, and this is the one caller the
+ // document cannot answer for: there is no editor session here to have probed and saved
+ // them, so without this the CLI lays the box out from a hardcoded 4:3 and a 16:9 camera
+ // exports framed differently from the same project opened in the app.
+ //
+ // Failure-tolerant on purpose, unlike the screen probe above which is allowed to reject:
+ // a camera file that has gone missing should cost the export its camera, not the export.
+ if (media.webcamVideoPath) {
+ const camera = await probeVideoDimensions(toFileUrl(media.webcamVideoPath)).catch(() => null);
+ if (camera) {
+ axcutDocument = {
+ ...axcutDocument,
+ assets: axcutDocument.assets.map((asset) =>
+ asset.cameraTrack
+ ? {
+ ...asset,
+ cameraTrack: {
+ ...asset.cameraTrack,
+ width: camera.width,
+ height: camera.height,
+ },
+ }
+ : asset,
+ ),
+ };
+ }
+ }
+ return {
+ document: axcutDocument,
+ editor,
+ screenVideoPath: media.screenVideoPath,
+ durationMs: probed.durationMs,
+ sourceDims: { width: probed.width, height: probed.height },
+ };
+}
+
+/** The project at `projectPath`, whichever shape it was saved in. */
+export async function loadCliProject(projectPath: string): Promise<ExportProject> {
+ const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(projectPath);
+ if (!loaded.success || loaded.project === undefined) {
+ throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file");
+ }
+ if (isAxcutDocumentFile(loaded.project)) {
+ return loadDocumentProject(loaded.project, (videoPath) =>
+ probeVideoDimensions(toFileUrl(videoPath)),
+ );
+ }
+ if (validateProjectData(loaded.project)) return loadLegacyProject(loaded.project);
+ throw new Error("Project file is not a valid .openscreen project");
+}
diff --git a/src/cli/vendor/leadingSilence.ts b/src/cli/vendor/leadingSilence.ts
deleted file mode 100644
index 1f6b1b846..000000000
--- a/src/cli/vendor/leadingSilence.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-// Vendored for the CLI: @/lib/captioning/leadingSilence was deleted in the 1.8 line. The CLI's captions command still trims leading silence before transcription, so the module lives on here verbatim.
-
-/** Caption path is always mono 16 kHz after `extractMono16kFromVideoUrl`. */
-import type { TrimRegion } from "@/components/video-editor/types";
-
-const SAMPLE_RATE = 16_000;
-
-/** Window length for peak detection (~50 ms). */
-const WINDOW_SAMPLES = 800;
-
-/** Coarse hop so long intros scan quickly (~50 ms steps). */
-const HOP_SAMPLES = 800;
-
-/** Max |sample| in a window below this counts as silence (float PCM ~[-1, 1]). */
-const PEAK_THRESHOLD = 0.012;
-
-/** Keep a little audio before the first peak so word onsets are not clipped. */
-const PRE_ROLL_SEC = 0.12;
-
-/** Do not scan more than this much audio for leading silence (performance + pathological files). */
-const MAX_LEADING_SCAN_SEC = 15 * 60;
-
-/**
- * Drops quiet audio at the beginning so Whisper is not fed a long silent prefix (which can skew
- * the first phrase and wastes work). Returned `trimSec` must be added back to every segment time.
- */
-export function trimLeadingSilenceMono16k(samples: Float32Array): {
- samples: Float32Array;
- trimSec: number;
-} {
- if (samples.length < WINDOW_SAMPLES) {
- return { samples, trimSec: 0 };
- }
-
- const maxIndex = Math.min(
- samples.length - WINDOW_SAMPLES,
- Math.floor(MAX_LEADING_SCAN_SEC * SAMPLE_RATE),
- );
-
- let firstSpeechSample = -1;
- for (let i = 0; i <= maxIndex; i += HOP_SAMPLES) {
- let peak = 0;
- for (let j = 0; j < WINDOW_SAMPLES; j++) {
- peak = Math.max(peak, Math.abs(samples[i + j]!));
- }
- if (peak > PEAK_THRESHOLD) {
- firstSpeechSample = i;
- break;
- }
- }
-
- if (firstSpeechSample <= 0) {
- return { samples, trimSec: 0 };
- }
-
- const preRollSamples = Math.round(PRE_ROLL_SEC * SAMPLE_RATE);
- const start = Math.max(0, firstSpeechSample - preRollSamples);
- return {
- samples: samples.subarray(start),
- trimSec: start / SAMPLE_RATE,
- };
-}
-
-/**
- * When audio is trimmed from the front, Whisper times are relative to the shortened buffer.
- * Shift trim regions by the same offset so `segmentOverlapsTrim` still uses consistent coordinates.
- */
-export function shiftTrimRegionsMsForCaptionBuffer(
- regions: TrimRegion[],
- trimMs: number,
-): TrimRegion[] {
- if (trimMs <= 0) return regions;
- return regions
- .map((r) => ({
- ...r,
- startMs: Math.max(0, r.startMs - trimMs),
- endMs: Math.max(0, r.endMs - trimMs),
- }))
- .filter((r) => r.endMs > r.startMs);
-}
diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts
index fa400a872..df1e6e87d 100644
--- a/src/lib/cliContracts.ts
+++ b/src/lib/cliContracts.ts
@@ -132,6 +132,6 @@ export interface CliDoneResult {
projectData?: unknown;
/** Sources: enumeration payload printed by the main process. */
sources?: CliSourcesResult;
- /** Captions: number of caption annotations generated. */
+ /** Captions: how many captions the transcript now shows. */
captionCount?: number;
}