Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(cli): openscreen captions writes a transcript and turns captions on
EtienneLescot7aefe3afbab2
diff --git a/electron/cli/args.ts b/electron/cli/args.tsindex 9b621b660..6c1323010 100644--- a/electron/cli/args.ts+++ b/electron/cli/args.ts@@ -84,7 +84,7 @@ Usage:openscreen record [options] Record the screen headlesslyopenscreen sources [--json] [-o <file>] List displays, windows, microphones and camerasopenscreen pack <project.openscreen> --out <dir> Copy project + media into one portable folder- openscreen captions <project.openscreen> Add auto-captions (on-device Whisper) to a project+ openscreen captions <project.openscreen> Transcribe (on-device Whisper) and turn captions on[--min-words <1-12>] [--max-words <1-12>]openscreen info <project.openscreen> [--json] Inspect a project fileopenscreen mcp Serve the agent tools over MCP on stdiodiff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsxindex a87f729e2..046637440 100644--- a/src/cli/CliCaptionsRunner.tsx+++ b/src/cli/CliCaptionsRunner.tsx@@ -1,149 +1,58 @@-// Hidden-window runner for `openscreen captions`: transcribes the project's-// audio with the on-device Whisper worker and writes the resulting caption-// annotations back into the project. Mirrors VideoEditor.generateAutoCaptions.+// Hidden-window runner for `openscreen captions`: transcribes the project's primary media+// with the editor's own on-device Whisper path (`transcribeAsset`) and turns its captions+// on (`captionProject.ts`). The main process writes the project back.-import { useEffect, useRef, useState } from "react";-import {- normalizeProjectEditor,- resolveProjectMedia,- toFileUrl,- validateProjectData,-} from "@/components/video-editor/projectPersistence";-import type { AnnotationRegion, TrimRegion } from "@/components/video-editor/types";-import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k";-import { type SttRendererStatus, transcribeMono16kToSegments } from "@/lib/captioning/transcribe";+import { useEffect, useRef } from "react";+import { type TranscribeStatus, transcribeAsset } from "@/lib/ai-edition/document/transcribe";+import { classifyTranscriptionError } from "@/lib/ai-edition/transcription/status";import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts";-import { nativeBridgeClient } from "@/native";-import { captionSegmentsToAnnotationRegions } from "./captionAnnotations";-import {- shiftTrimRegionsMsForCaptionBuffer,- trimLeadingSilenceMono16k,-} from "./vendor/leadingSilence";--/** Highest trailing number across existing region ids, so new ids never collide. */-function nextNumericIdFrom(regions: { id: string }[]): number {- let max = 0;- for (const region of regions) {- const match = /(\d+)$/.exec(region.id);- if (match) max = Math.max(max, Number(match[1]));- }- return max + 1;-}+import { captionProject } from "./captionProject";+import { loadCliProject } from "./loadCliProject";async function runCaptions(request: CliCaptionsRequest): Promise<CliDoneResult> {- const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath);- if (!loaded.success || loaded.project === undefined) {- throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file");- }- if (!validateProjectData(loaded.project)) {- throw new Error("Project file is not a valid .openscreen project");- }- const project = loaded.project;- const media = resolveProjectMedia(project);- if (!media) {- throw new Error("Project file does not reference any recorded media");- }- const editor = normalizeProjectEditor(project.editor ?? {});- const trimRegions: TrimRegion[] = editor.trimRegions;-- window.electronAPI.cliLog("info", "Extracting audio…");- const videoUrl = toFileUrl(media.screenVideoPath);- const { samples, durationSec } = await extractMono16kFromVideoUrl(videoUrl);- if (!Number.isFinite(durationSec) || durationSec <= 0 || samples.length < 800) {- throw new Error("The project's video has no usable audio track to transcribe");- }-- const { samples: speechSamples, trimSec } = trimLeadingSilenceMono16k(samples);- if (speechSamples.length < 800) {- throw new Error("No speech detected in the project's audio");- }-- const trimMs = Math.round(trimSec * 1000);- const trimRegionsForTranscribe = shiftTrimRegionsMsForCaptionBuffer(trimRegions, trimMs);-- // `onStatus` now fires once per transcribed chunk, not once per phase, so log- // only on a phase change — otherwise a long transcription spams the CLI with- // one identical line per chunk.- let loggedPhase: SttRendererStatus["phase"] | null = null;- const transcribeOptions = {- onStatus: ({ phase }: SttRendererStatus) => {- if (phase === loggedPhase) return;- loggedPhase = phase;- window.electronAPI.cliLog(- "info",- phase === "model" ? "Loading caption model…" : "Transcribing…",- );- },+ const { document } = await loadCliProject(request.projectPath);++ // `onStatus` fires once per transcribed chunk, so log only on a phase change: a long+ // transcription would otherwise print one identical line per chunk.+ let loggedPhase: TranscribeStatus["phase"] | null = null;+ const onStatus = ({ phase }: TranscribeStatus) => {+ if (phase === loggedPhase) return;+ loggedPhase = phase;+ window.electronAPI.cliLog(+ "info",+ phase === "extracting-audio"+ ? "Extracting audio…"+ : phase === "loading-model"+ ? "Loading the speech model…"+ : "Transcribing…",+ );};- let { segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(speechSamples, {- trimRegions: trimRegionsForTranscribe,- ...transcribeOptions,- });- let transcribedFromTrimmedBuffer = true;-- // Leading-silence trimming can return empty even when the full source has- // speech. Retry once against the untrimmed buffer before giving up.- if (segmentsRaw.length === 0 && trimSec > 0) {- ({ segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(samples, {- trimRegions,- ...transcribeOptions,- }));- transcribedFromTrimmedBuffer = false;- }-- const segments =- transcribedFromTrimmedBuffer && trimSec > 0- ? segmentsRaw.map((segment) => ({- ...segment,- startSec: segment.startSec + trimSec,- endSec: segment.endSec + trimSec,- }))- : segmentsRaw;-- // Re-running the command replaces earlier auto-captions instead of stacking- // duplicates; manually added annotations are preserved.- const manualAnnotations: AnnotationRegion[] = editor.annotationRegions.filter(- (annotation) => annotation.annotationSource !== "auto-caption",+ const result = await captionProject(+ document,+ {+ minWordsPerLine: request.minWordsPerCaption,+ maxWordsPerLine: request.maxWordsPerCaption,+ },+ (doc, assetId) =>+ transcribeAsset(doc, assetId, { onStatus }).catch((error: unknown) => {+ throw new Error(`Transcription failed: ${classifyTranscriptionError(error).message}`);+ }),);- const startNumericId = nextNumericIdFrom([...editor.annotationRegions, ...editor.zoomRegions]);- const startZIndex = manualAnnotations.reduce((max, a) => Math.max(max, a.zIndex + 1), 1);-- let regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, {- minWordsPerCaption: request.minWordsPerCaption,- maxWordsPerCaption: request.maxWordsPerCaption,- timestampGranularity: granularity,- });- if (regions.length === 0 && segments.length > 0) {- regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, {- minWordsPerCaption: 1,- maxWordsPerCaption: Number.MAX_SAFE_INTEGER,- timestampGranularity: granularity,- });- }- if (regions.length === 0) {- throw new Error("Transcription produced no caption segments");+ if (!result.transcribed) {+ window.electronAPI.cliLog("info", "Using the transcript the project already has.");}- const updatedProject = {- ...project,- editor: {- ...editor,- annotationRegions: [...manualAnnotations, ...regions],- },- };-return {success: true,projectPath: request.projectPath,- projectData: updatedProject,- captionCount: regions.length,+ projectData: result.document,+ captionCount: result.captionCount,};}export function CliCaptionsRunner() {const startedRef = useRef(false);- const [status] = useState("Generating captions…");useEffect(() => {if (startedRef.current) return;@@ -166,7 +75,7 @@ export function CliCaptionsRunner() {return (<div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm">- {status}+ Generating captions…</div>);}diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsxindex e206e1b83..685f80f14 100644--- a/src/cli/CliExportRunner.tsx+++ b/src/cli/CliExportRunner.tsx@@ -6,22 +6,14 @@// builds the clip list + scene JSON and relays progress.import { useEffect, useRef, useState } from "react";-import {- type EditorProjectData,- normalizeProjectEditor,- resolveProjectMedia,- toFileUrl,- validateProjectData,-} from "@/components/video-editor/projectPersistence";+import { toFileUrl } from "@/components/video-editor/projectPersistence";import { prepareClickSound } from "@/lib/ai-edition/clickSound";-import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate";import {collectEffectiveClipDims,pickExtremeDims,resolveAspectRatioValue,} from "@/lib/ai-edition/document/outputFormat";-import { applyProbedDuration } from "@/lib/ai-edition/document/timeline";-import { type AxcutDocument, isAxcutDocumentFile } from "@/lib/ai-edition/schema";+import type { AxcutDocument } from "@/lib/ai-edition/schema";import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings";import {appendAutoZoomSuggestions,@@ -38,38 +30,7 @@ import { exportGifNative, exportMultiNative, nativeBridgeClient } from "@/nativeimport type { CompositorClipInput } from "@/native/contracts";import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription";import { type ExportProject, loadDocumentProject } from "./exportProject";--function probeVideoDimensions(- url: string,-): Promise<{ width: number; height: number; durationMs: number }> {- return new Promise((resolve, reject) => {- const video = document.createElement("video");- video.preload = "metadata";- video.muted = true;- const cleanup = () => {- clearTimeout(timer);- video.removeAttribute("src");- video.load();- };- // A stalled load fires neither event; without a deadline the CLI hangs.- const timer = setTimeout(() => {- cleanup();- reject(new Error(`Timed out reading video metadata: ${url}`));- }, 30_000);- video.onloadedmetadata = () => {- const width = video.videoWidth;- const height = video.videoHeight;- const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0;- cleanup();- resolve({ width, height, durationMs });- };- video.onerror = () => {- cleanup();- reject(new Error(`Failed to load video metadata: ${url}`));- };- video.src = url;- });-}+import { loadCliProject, probeVideoDimensions } from "./loadCliProject";function replaceExtension(filePath: string, newExtension: string): string {return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension;@@ -115,126 +76,18 @@ function gifOutputDims(return { width: even(sourceDims.width), height: even(sourceDims.height) };}-/**- * A legacy v2 project (`{ version, media, editor }`), as written before the editor saved the- * document itself: migrated to an AxcutDocument, with what the migration cannot know probed.- */-async function loadLegacyProject(project: EditorProjectData): Promise<ExportProject> {- const media = resolveProjectMedia(project);- if (!media) {- throw new Error("Project file does not reference any recorded media");- }- // Prefer the main process's approved session paths: they carry the- // packed-project sibling fallback when the stored absolute paths are stale.- try {- const sessionResult = await window.electronAPI.getCurrentRecordingSession();- const session = sessionResult?.session;- if (session?.screenVideoPath) {- media.screenVideoPath = session.screenVideoPath;- if (media.webcamVideoPath && session.webcamVideoPath) {- media.webcamVideoPath = session.webcamVideoPath;- }- }- } catch {- // Fall back to the paths stored in the project file.- }- const editor = normalizeProjectEditor(project.editor ?? {});-- const probed = await probeVideoDimensions(toFileUrl(media.screenVideoPath));-- // Migrate the .openscreen project onto the AxcutDocument the native- // compositor consumes. The migration is pure and carries zooms, annotations,- // trims and the legacy editor settings; the clip's duration is unknown until- // probed, so applyProbedDuration must run or the export is a single frame.- let axcutDocument = migrateProjectDataToAxcutDocument({- ...project,- media,- editor,- });- const primaryAssetId = axcutDocument.project.primaryAssetId ?? axcutDocument.assets[0]?.id;- if (!primaryAssetId) {- throw new Error("Project migration produced no media asset");- }- if (probed.durationMs > 0) {- axcutDocument = applyProbedDuration(axcutDocument, primaryAssetId, probed.durationMs / 1000);- }- // The screen's dimensions, for the same reason as the camera's below: the pure migration- // cannot know them, and the scene's screen box, the output frame and Auto all read them off- // `asset.video`. Without them the CLI laid every recording out as 1920x1080 while sizing the- // export from the probe, two answers to one question.- if (probed.width > 0 && probed.height > 0) {- axcutDocument = {- ...axcutDocument,- assets: axcutDocument.assets.map((asset) =>- asset.id === primaryAssetId- ? {- ...asset,- video: {- codec: "unknown",- fps: 0,- ...asset.video,- width: probed.width,- height: probed.height,- },- }- : asset,- ),- };- }-- // The camera's dimensions decide the PiP's layout box, and this is the one caller the- // document cannot answer for: there is no editor session here to have probed and saved- // them, so without this the CLI lays the box out from a hardcoded 4:3 and a 16:9 camera- // exports framed differently from the same project opened in the app.- //- // Failure-tolerant on purpose, unlike the screen probe above which is allowed to reject:- // a camera file that has gone missing should cost the export its camera, not the export.- if (media.webcamVideoPath) {- const camera = await probeVideoDimensions(toFileUrl(media.webcamVideoPath)).catch(() => null);- if (camera) {- axcutDocument = {- ...axcutDocument,- assets: axcutDocument.assets.map((asset) =>- asset.cameraTrack- ? {- ...asset,- cameraTrack: {- ...asset.cameraTrack,- width: camera.width,- height: camera.height,- },- }- : asset,- ),- };- }- }- return {- document: axcutDocument,- editor,- screenVideoPath: media.screenVideoPath,- durationMs: probed.durationMs,- sourceDims: { width: probed.width, height: probed.height },- };-}-/** The project to export: by id when the MCP server names one, else the file at its path. */async function loadExportProject(request: CliExportRequest): Promise<ExportProject> {- const probe = (videoPath: string) => probeVideoDimensions(toFileUrl(videoPath));if (request.projectId) {const stored = await nativeBridgeClient.aiEdition.get(request.projectId);if (!stored.success || !stored.document) {throw new Error(stored.error ?? `No project has the id "${request.projectId}"`);}- return loadDocumentProject(stored.document, probe);- }- const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath);- if (!loaded.success || loaded.project === undefined) {- throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file");+ return loadDocumentProject(stored.document, (videoPath) =>+ probeVideoDimensions(toFileUrl(videoPath)),+ );}- if (isAxcutDocumentFile(loaded.project)) return loadDocumentProject(loaded.project, probe);- if (validateProjectData(loaded.project)) return loadLegacyProject(loaded.project);- throw new Error("Project file is not a valid .openscreen project");+ return loadCliProject(request.projectPath);}async function runExport(request: CliExportRequest): Promise<CliDoneResult> {diff --git a/src/cli/captionAnnotations.test.ts b/src/cli/captionAnnotations.test.tsdeleted file mode 100644index 79f336a69..000000000--- a/src/cli/captionAnnotations.test.ts+++ /dev/null@@ -1,53 +0,0 @@-import { describe, expect, it } from "vitest";-import { DEFAULT_CAPTION_SETTINGS } from "@/lib/ai-edition/captions/settings";-import type { CaptionSegment } from "@/lib/captioning/transcribe";-import { captionSegmentsToAnnotationRegions } from "./captionAnnotations";--const words = (...texts: string[]): CaptionSegment[] =>- texts.map((text, i) => ({ text, startSec: i * 0.5, endSec: i * 0.5 + 0.4 }));--describe("captionSegmentsToAnnotationRegions", () => {- it("numbers ids and z-indexes from the given start, with non-empty spans", () => {- const regions = captionSegmentsToAnnotationRegions(words("one", "two", "three", "four"), 7, 3);-- expect(regions.length).toBeGreaterThan(0);- expect(regions.map((r) => r.id)).toEqual(regions.map((_, i) => `annotation-${7 + i}`));- expect(regions.map((r) => r.zIndex)).toEqual(regions.map((_, i) => 3 + i));- for (const region of regions) {- expect(region.endMs).toBeGreaterThan(region.startMs);- expect(region.annotationSource).toBe("auto-caption");- expect(region.content.trim()).not.toBe("");- }- });-- it("groups phrase-granularity segments one line at a time", () => {- const regions = captionSegmentsToAnnotationRegions(- [- { text: "hello there", startSec: 0, endSec: 1 },- { text: "second line", startSec: 1.2, endSec: 2 },- ],- 1,- 1,- { timestampGranularity: "phrase" },- );-- expect(regions.map((r) => r.content)).toEqual(["hello there", "second line"]);- });-- it("looks like the editor's captions, not a faint annotation", () => {- const [region] = captionSegmentsToAnnotationRegions(words("one", "two"), 1, 1);- expect(region.style).toMatchObject({- fontSize: DEFAULT_CAPTION_SETTINGS.fontSize,- fontWeight: DEFAULT_CAPTION_SETTINGS.fontWeight,- color: DEFAULT_CAPTION_SETTINGS.color,- backgroundColor: "rgba(0, 0, 0, 0.55)",- });- // The editor's caption box: 16:9 safe column, bottom edge 1.5% off the frame's, room for- // a caption that wraps. The text is bottom-anchored in it (see sceneDescription).- expect(region.position.x).toBe(16);- expect(region.size.width).toBe(68);- expect(region.position.y + region.size.height).toBeCloseTo(98.5);- // Three 48 px lines of 1.5 em plus the plate padding: a wrapped caption is not clipped.- expect(region.size.height).toBeCloseTo(((48 * (3 * 1.5 + 0.2)) / 1080) * 100);- });-});diff --git a/src/cli/captionAnnotations.ts b/src/cli/captionAnnotations.tsdeleted file mode 100644index 91e3baa96..000000000--- a/src/cli/captionAnnotations.ts+++ /dev/null@@ -1,84 +0,0 @@-// CLI-only: the v4 editor renders captions natively, but `openscreen captions`-// still writes caption *annotations* into .openscreen projects. Only the-// annotation conversion lives here — the segment grouping/dedupe helpers come-// from the live captioning module so the CLI can't drift from the editor.--import type { AnnotationRegion, AnnotationTextStyle } from "@/components/video-editor/types";-import {- captionBackgroundCss,- captionBoxRect,- DEFAULT_CAPTION_SETTINGS,-} from "@/lib/ai-edition/captions/settings";-import {- type CaptionSegmentLayoutOptions,- dedupeAdjacentCaptionRepeats,- finalizeCaptionSegmentsForPlayback,- groupPhraseCaptionSegmentsIntoLines,- groupTimedCaptionWordsIntoLines,-} from "@/lib/captioning/annotationsFromCaptions";-import type { CaptionSegment } from "@/lib/captioning/transcribe";--// The look and place of the editor's own captions (`DEFAULT_CAPTION_SETTINGS`): bold 48 px-// white on a 55% black plate, in the landscape safe column, just off the bottom edge. The CLI-// used to write 24 px regular text on no plate, which read as faint over any bright frame.-// `.openscreen` projects carry no transcript, so the CLI cannot write the editor's transcript--// driven captions; matching their appearance is what keeps the two from looking different.-//-// Placed with the editor's own caption box: room for three lines, pinned by its BOTTOM edge-// 1.5% off the frame's. The scene description bottom-anchors `auto-caption` text in it, so-// the plate ends at the inset whether the caption holds one line or wraps to three.-const CAPTION_BOX = captionBoxRect(DEFAULT_CAPTION_SETTINGS, 16 / 9);--const CAPTION_POSITION = { x: CAPTION_BOX.x, y: CAPTION_BOX.y };--const CAPTION_SIZE = { width: CAPTION_BOX.width, height: CAPTION_BOX.height };--const CAPTION_STYLE: AnnotationTextStyle = {- color: DEFAULT_CAPTION_SETTINGS.color,- backgroundColor: captionBackgroundCss(DEFAULT_CAPTION_SETTINGS),- fontSize: DEFAULT_CAPTION_SETTINGS.fontSize,- fontFamily: DEFAULT_CAPTION_SETTINGS.fontFamily,- fontWeight: DEFAULT_CAPTION_SETTINGS.fontWeight,- fontStyle: "normal",- textDecoration: "none",- textAlign: "center",-};--export function captionSegmentsToAnnotationRegions(- segments: CaptionSegment[],- startNumericId: number,- startZIndex: number,- layout?: CaptionSegmentLayoutOptions,-): AnnotationRegion[] {- // Don't echo-collapse raw word tokens before grouping: repeated words ("I … I") share a- // normalized key and would merge spans while keeping only the first token's text.- const minW = layout?.minWordsPerCaption ?? DEFAULT_CAPTION_SETTINGS.minWordsPerLine;- const maxW = layout?.maxWordsPerCaption ?? DEFAULT_CAPTION_SETTINGS.maxWordsPerLine;- const granularity = layout?.timestampGranularity ?? "word";-- const grouped =- granularity === "phrase"- ? groupPhraseCaptionSegmentsIntoLines(segments, minW, maxW)- : groupTimedCaptionWordsIntoLines(segments, minW, maxW);-- const finalized = finalizeCaptionSegmentsForPlayback(dedupeAdjacentCaptionRepeats(grouped));-- let nid = startNumericId;- let z = startZIndex;- return finalized.map((seg) => {- const startMs = Math.round(seg.startSec * 1000);- const endMs = Math.max(Math.round(seg.endSec * 1000), startMs + 1);- return {- id: `annotation-${nid++}`,- startMs,- endMs,- type: "text",- content: seg.text,- annotationSource: "auto-caption",- position: { ...CAPTION_POSITION },- size: { ...CAPTION_SIZE },- style: { ...CAPTION_STYLE },- zIndex: z++,- };- });-}diff --git a/src/cli/captionProject.test.ts b/src/cli/captionProject.test.tsnew file mode 100644index 000000000..a7f09f032--- /dev/null+++ b/src/cli/captionProject.test.ts@@ -0,0 +1,108 @@+import { describe, expect, it, vi } from "vitest";+import { getCaptionSettings } from "@/lib/ai-edition/captions";+import {+ type AxcutDocument,+ type AxcutTranscript,+ createEmptyDocument,+ documentSchema,+} from "@/lib/ai-edition/schema";+import { captionProject } from "./captionProject";++function project(extra: Partial<AxcutDocument> = {}): AxcutDocument {+ const base = createEmptyDocument({+ title: "CLI",+ projectId: "proj_1",+ createdAt: "2026-01-01T00:00:00.000Z",+ });+ return documentSchema.parse({+ ...base,+ project: { ...base.project, primaryAssetId: "asset_1" },+ assets: [+ { id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 10 },+ ],+ timeline: {+ ...base.timeline,+ clips: [+ {+ id: "clip_1",+ assetId: "asset_1",+ sourceStartSec: 0,+ sourceEndSec: 10,+ timelineStartSec: 0,+ timelineEndSec: 10,+ wordRefs: [],+ origin: "user",+ reason: "",+ },+ ],+ },+ ...extra,+ });+}++function transcript(text: string): AxcutTranscript {+ const words = text.split(" ");+ return {+ assetId: "asset_1",+ language: "en",+ segments: words.map((word, i) => ({+ id: `seg_${i + 1}`,+ kind: "speech" as const,+ startSec: i,+ endSec: i + 1,+ text: word,+ wordIds: [`word_${i + 1}`],+ })),+ words: words.map((word, i) => ({+ id: `word_${i + 1}`,+ segmentId: `seg_${i + 1}`,+ startSec: i,+ endSec: i + 1,+ text: word,+ })),+ };+}++const LINE = { minWordsPerLine: 1, maxWordsPerLine: 2 };++describe("captionProject", () => {+ it("transcribes the primary media and turns the editor's captions on", async () => {+ const transcribe = vi.fn(async () => transcript("one two three four"));+ const result = await captionProject(project(), LINE, transcribe);++ expect(transcribe).toHaveBeenCalledWith(expect.anything(), "asset_1");+ expect(result.transcribed).toBe(true);+ expect(result.document.transcripts).toHaveLength(1);+ expect(getCaptionSettings(result.document)).toMatchObject({+ enabled: true,+ minWordsPerLine: 1,+ maxWordsPerLine: 2,+ });+ expect(result.captionCount).toBe(2);+ });++ it("reuses a transcript the project has, and clears the old caption annotations", async () => {+ const withLegacy = {+ ...project({ transcripts: [transcript("hello there")] }),+ annotations: [+ { id: "annotation-1", annotationSource: "auto-caption" },+ ] as unknown as AxcutDocument["annotations"],+ };+ const transcribe = vi.fn();+ const result = await captionProject(withLegacy, LINE, transcribe);++ expect(transcribe).not.toHaveBeenCalled();+ expect(result.transcribed).toBe(false);+ expect(result.document.annotations).toEqual([]);+ });++ it("fails on media with no speech, writing nothing", async () => {+ const silent: AxcutTranscript = {+ assetId: "asset_1",+ language: "auto",+ segments: [],+ words: [],+ };+ await expect(captionProject(project(), LINE, async () => silent)).rejects.toThrow(/No speech/);+ });+});diff --git a/src/cli/captionProject.ts b/src/cli/captionProject.tsnew file mode 100644index 000000000..44ac48042--- /dev/null+++ b/src/cli/captionProject.ts@@ -0,0 +1,59 @@+// What `openscreen captions` does to a project, kept out of the runner window so it is+// testable: the editor's own captions, not annotations. The primary media gets a+// transcript (the one it has, or a fresh one), then the captions are turned on with the+// requested line length, as the Captions pane would. They are the transcript drawn at+// render time, so `openscreen export` burns them in and the editor shows them editable.++import {+ deriveCaptionCues,+ getCaptionSettings,+ getCaptionTranslations,+ patchCaptionSettings,+} from "@/lib/ai-edition/captions";+import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat";+import { applyTranscriptionResult } from "@/lib/ai-edition/document/transcript";+import type { AxcutDocument, AxcutTranscript } from "@/lib/ai-edition/schema";+import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings";+import { findAssetTranscript, transcriptHasSpeech } from "@/lib/ai-edition/transcription/status";++export interface CaptionProjectOptions {+ minWordsPerLine: number;+ maxWordsPerLine: number;+}++export async function captionProject(+ document: AxcutDocument,+ options: CaptionProjectOptions,+ transcribe: (document: AxcutDocument, assetId: string) => Promise<AxcutTranscript>,+): Promise<{ document: AxcutDocument; captionCount: number; transcribed: boolean }> {+ const assetId = document.project.primaryAssetId ?? document.assets[0]?.id;+ if (!assetId) throw new Error("Project file does not reference any recorded media");+ const existing = findAssetTranscript(document, assetId);+ const transcript = existing ?? (await transcribe(document, assetId));+ if (!transcriptHasSpeech(transcript)) {+ throw new Error("No speech detected in the project's audio");+ }+ const transcribed = existing ? document : applyTranscriptionResult(document, transcript).document;+ // The v1 caption annotations this command used to write: the Captions pane offers to+ // clear them, and next to real captions they would show every line twice.+ const cleared: AxcutDocument = {+ ...transcribed,+ annotations: transcribed.annotations.filter((a) => a.annotationSource !== "auto-caption"),+ };+ const aspect = resolveAspectRatioValue(cleared, getEditorSettings(cleared).aspectRatio);+ const next = patchCaptionSettings(+ cleared,+ {+ enabled: true,+ minWordsPerLine: options.minWordsPerLine,+ maxWordsPerLine: options.maxWordsPerLine,+ },+ aspect,+ );+ const captionCount = deriveCaptionCues(+ next,+ getCaptionSettings(next, aspect),+ getCaptionTranslations(next),+ ).length;+ return { document: next, captionCount, transcribed: !existing };+}diff --git a/src/cli/loadCliProject.ts b/src/cli/loadCliProject.tsnew file mode 100644index 000000000..395514348--- /dev/null+++ b/src/cli/loadCliProject.ts@@ -0,0 +1,166 @@+// What `openscreen export` and `openscreen captions` read from a project file, in the+// hidden runner window: either shape the file was saved in, as one AxcutDocument. The+// renderer half of `exportProject.ts`, since probing a video needs a DOM.++import {+ type EditorProjectData,+ normalizeProjectEditor,+ resolveProjectMedia,+ toFileUrl,+ validateProjectData,+} from "@/components/video-editor/projectPersistence";+import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate";+import { applyProbedDuration } from "@/lib/ai-edition/document/timeline";+import { isAxcutDocumentFile } from "@/lib/ai-edition/schema";+import { nativeBridgeClient } from "@/native";+import { type ExportProject, loadDocumentProject } from "./exportProject";++export function probeVideoDimensions(+ url: string,+): Promise<{ width: number; height: number; durationMs: number }> {+ return new Promise((resolve, reject) => {+ const video = document.createElement("video");+ video.preload = "metadata";+ video.muted = true;+ const cleanup = () => {+ clearTimeout(timer);+ video.removeAttribute("src");+ video.load();+ };+ // A stalled load fires neither event; without a deadline the CLI hangs.+ const timer = setTimeout(() => {+ cleanup();+ reject(new Error(`Timed out reading video metadata: ${url}`));+ }, 30_000);+ video.onloadedmetadata = () => {+ const width = video.videoWidth;+ const height = video.videoHeight;+ const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0;+ cleanup();+ resolve({ width, height, durationMs });+ };+ video.onerror = () => {+ cleanup();+ reject(new Error(`Failed to load video metadata: ${url}`));+ };+ video.src = url;+ });+}++/**+ * A legacy v2 project (`{ version, media, editor }`), as written before the editor saved the+ * document itself: migrated to an AxcutDocument, with what the migration cannot know probed.+ */+async function loadLegacyProject(project: EditorProjectData): Promise<ExportProject> {+ const media = resolveProjectMedia(project);+ if (!media) {+ throw new Error("Project file does not reference any recorded media");+ }+ // Prefer the main process's approved session paths: they carry the+ // packed-project sibling fallback when the stored absolute paths are stale.+ try {+ const sessionResult = await window.electronAPI.getCurrentRecordingSession();+ const session = sessionResult?.session;+ if (session?.screenVideoPath) {+ media.screenVideoPath = session.screenVideoPath;+ if (media.webcamVideoPath && session.webcamVideoPath) {+ media.webcamVideoPath = session.webcamVideoPath;+ }+ }+ } catch {+ // Fall back to the paths stored in the project file.+ }+ const editor = normalizeProjectEditor(project.editor ?? {});++ const probed = await probeVideoDimensions(toFileUrl(media.screenVideoPath));++ // Migrate the .openscreen project onto the AxcutDocument the native+ // compositor consumes. The migration is pure and carries zooms, annotations,+ // trims and the legacy editor settings; the clip's duration is unknown until+ // probed, so applyProbedDuration must run or the export is a single frame.+ let axcutDocument = migrateProjectDataToAxcutDocument({+ ...project,+ media,+ editor,+ });+ const primaryAssetId = axcutDocument.project.primaryAssetId ?? axcutDocument.assets[0]?.id;+ if (!primaryAssetId) {+ throw new Error("Project migration produced no media asset");+ }+ if (probed.durationMs > 0) {+ axcutDocument = applyProbedDuration(axcutDocument, primaryAssetId, probed.durationMs / 1000);+ }+ // The screen's dimensions, for the same reason as the camera's below: the pure migration+ // cannot know them, and the scene's screen box, the output frame and Auto all read them off+ // `asset.video`. Without them the CLI laid every recording out as 1920x1080 while sizing the+ // export from the probe, two answers to one question.+ if (probed.width > 0 && probed.height > 0) {+ axcutDocument = {+ ...axcutDocument,+ assets: axcutDocument.assets.map((asset) =>+ asset.id === primaryAssetId+ ? {+ ...asset,+ video: {+ codec: "unknown",+ fps: 0,+ ...asset.video,+ width: probed.width,+ height: probed.height,+ },+ }+ : asset,+ ),+ };+ }++ // The camera's dimensions decide the PiP's layout box, and this is the one caller the+ // document cannot answer for: there is no editor session here to have probed and saved+ // them, so without this the CLI lays the box out from a hardcoded 4:3 and a 16:9 camera+ // exports framed differently from the same project opened in the app.+ //+ // Failure-tolerant on purpose, unlike the screen probe above which is allowed to reject:+ // a camera file that has gone missing should cost the export its camera, not the export.+ if (media.webcamVideoPath) {+ const camera = await probeVideoDimensions(toFileUrl(media.webcamVideoPath)).catch(() => null);+ if (camera) {+ axcutDocument = {+ ...axcutDocument,+ assets: axcutDocument.assets.map((asset) =>+ asset.cameraTrack+ ? {+ ...asset,+ cameraTrack: {+ ...asset.cameraTrack,+ width: camera.width,+ height: camera.height,+ },+ }+ : asset,+ ),+ };+ }+ }+ return {+ document: axcutDocument,+ editor,+ screenVideoPath: media.screenVideoPath,+ durationMs: probed.durationMs,+ sourceDims: { width: probed.width, height: probed.height },+ };+}++/** The project at `projectPath`, whichever shape it was saved in. */+export async function loadCliProject(projectPath: string): Promise<ExportProject> {+ const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(projectPath);+ if (!loaded.success || loaded.project === undefined) {+ throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file");+ }+ if (isAxcutDocumentFile(loaded.project)) {+ return loadDocumentProject(loaded.project, (videoPath) =>+ probeVideoDimensions(toFileUrl(videoPath)),+ );+ }+ if (validateProjectData(loaded.project)) return loadLegacyProject(loaded.project);+ throw new Error("Project file is not a valid .openscreen project");+}diff --git a/src/cli/vendor/leadingSilence.ts b/src/cli/vendor/leadingSilence.tsdeleted file mode 100644index 1f6b1b846..000000000--- a/src/cli/vendor/leadingSilence.ts+++ /dev/null@@ -1,80 +0,0 @@-// Vendored for the CLI: @/lib/captioning/leadingSilence was deleted in the 1.8 line. The CLI's captions command still trims leading silence before transcription, so the module lives on here verbatim.--/** Caption path is always mono 16 kHz after `extractMono16kFromVideoUrl`. */-import type { TrimRegion } from "@/components/video-editor/types";--const SAMPLE_RATE = 16_000;--/** Window length for peak detection (~50 ms). */-const WINDOW_SAMPLES = 800;--/** Coarse hop so long intros scan quickly (~50 ms steps). */-const HOP_SAMPLES = 800;--/** Max |sample| in a window below this counts as silence (float PCM ~[-1, 1]). */-const PEAK_THRESHOLD = 0.012;--/** Keep a little audio before the first peak so word onsets are not clipped. */-const PRE_ROLL_SEC = 0.12;--/** Do not scan more than this much audio for leading silence (performance + pathological files). */-const MAX_LEADING_SCAN_SEC = 15 * 60;--/**- * Drops quiet audio at the beginning so Whisper is not fed a long silent prefix (which can skew- * the first phrase and wastes work). Returned `trimSec` must be added back to every segment time.- */-export function trimLeadingSilenceMono16k(samples: Float32Array): {- samples: Float32Array;- trimSec: number;-} {- if (samples.length < WINDOW_SAMPLES) {- return { samples, trimSec: 0 };- }-- const maxIndex = Math.min(- samples.length - WINDOW_SAMPLES,- Math.floor(MAX_LEADING_SCAN_SEC * SAMPLE_RATE),- );-- let firstSpeechSample = -1;- for (let i = 0; i <= maxIndex; i += HOP_SAMPLES) {- let peak = 0;- for (let j = 0; j < WINDOW_SAMPLES; j++) {- peak = Math.max(peak, Math.abs(samples[i + j]!));- }- if (peak > PEAK_THRESHOLD) {- firstSpeechSample = i;- break;- }- }-- if (firstSpeechSample <= 0) {- return { samples, trimSec: 0 };- }-- const preRollSamples = Math.round(PRE_ROLL_SEC * SAMPLE_RATE);- const start = Math.max(0, firstSpeechSample - preRollSamples);- return {- samples: samples.subarray(start),- trimSec: start / SAMPLE_RATE,- };-}--/**- * When audio is trimmed from the front, Whisper times are relative to the shortened buffer.- * Shift trim regions by the same offset so `segmentOverlapsTrim` still uses consistent coordinates.- */-export function shiftTrimRegionsMsForCaptionBuffer(- regions: TrimRegion[],- trimMs: number,-): TrimRegion[] {- if (trimMs <= 0) return regions;- return regions- .map((r) => ({- ...r,- startMs: Math.max(0, r.startMs - trimMs),- endMs: Math.max(0, r.endMs - trimMs),- }))- .filter((r) => r.endMs > r.startMs);-}diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.tsindex fa400a872..df1e6e87d 100644--- a/src/lib/cliContracts.ts+++ b/src/lib/cliContracts.ts@@ -132,6 +132,6 @@ export interface CliDoneResult {projectData?: unknown;/** Sources: enumeration payload printed by the main process. */sources?: CliSourcesResult;- /** Captions: number of caption annotations generated. */+ /** Captions: how many captions the transcript now shows. */captionCount?: number;}