Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(mcp): transcribe an asset over MCP with the on-device Whisper
EtienneLescot024642616bf3
diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.tsindex 786f651b5..9e1d68825 100644--- a/electron/cli/cliMain.ts+++ b/electron/cli/cliMain.ts@@ -11,7 +11,7 @@ import { isDiagnosticModeEnabled } from "../diagnostics/main-log-buffer";import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers";import { openVideoInHiddenWindow } from "../mcp/hidden-video";import { serveMcpOverStdio } from "../mcp/stdio-server";-import { registerSttIpc } from "../stt";+import { registerSttIpc, shutdownStt } from "../stt";import { createRunnerWindow } from "../windows";import { CLI_USAGE, type CliCommand } from "./args";import { runInfoCommand, runPackCommand } from "./projectCommands";@@ -392,7 +392,8 @@ export function runCli(command: CliCommand): void {userDataDir,runners,openVideo: openVideoInHiddenWindow,- onClose: () => app.exit(0),+ // The Whisper helper is a process of its own: it must not outlive this one.+ onClose: () => void shutdownStt().finally(() => app.exit(0)),});milestone("mcp server listening on stdio");return;@@ -496,7 +497,7 @@ export function runCli(command: CliCommand): void {printSources(output, result.sources);} else if (command.kind === "captions") {output.info(- `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`,+ `Captions on: ${result.captionCount ?? 0} caption(s) from the transcript → ${result.projectPath}`,);} else if (command.kind === "export") {output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`);@@ -512,7 +513,8 @@ export function runCli(command: CliCommand): void {}// Give the renderer a beat to resolve the invoke before exiting.- setTimeout(() => app.exit(result.success ? 0 : 1), 50);+ // and stop the Whisper helper `captions` started, which would outlive this process.+ setTimeout(() => void shutdownStt().finally(() => app.exit(result.success ? 0 : 1)), 50);}).catch((error) => {output.error(error instanceof Error ? (error.stack ?? error.message) : String(error));diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.tsindex 05bb98aa2..8a3fa45d3 100644--- a/electron/ipc/handlers.ts+++ b/electron/ipc/handlers.ts@@ -157,6 +157,7 @@ import {shouldEnumerateRecordingSources,shouldPersistSelectedSource,} from "../recording-source-settings";+import { getSttManager } from "../stt";import { createRunnerWindow } from "../windows";import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes";import { registerNativeBridgeHandlers } from "./nativeBridge";@@ -5061,6 +5062,8 @@ export function registerIpcHandlers(editsAllowed: () => mcpSettings.getSettings().allowEdits,cursor: agentCursorTelemetryReader,cameraSegmentation: () => loudnessService.probeSegmentation(),+ // The editor's own engine, in this process: `openscreen mcp` transcribes without the app.+ transcribe: (request) => getSttManager().transcribe(request),version: app.getVersion(),// importMedia: the editor's own probe, and the grant a picked file gets, so the editor// can play a file a client imported into the project it has open.diff --git a/electron/mcp/app-channel.ts b/electron/mcp/app-channel.tsindex 09d7f6be4..9da5585cf 100644--- a/electron/mcp/app-channel.ts+++ b/electron/mcp/app-channel.ts@@ -44,6 +44,7 @@ import {LIST_PROJECTS_TOOL,type McpToolDeps,type McpToolRunner,+ TRANSCRIBE_TOOL,} from "./openscreen-mcp-server";import { PROJECT_EDIT_TOOLS } from "./project-tools";@@ -52,6 +53,8 @@ const DISCOVERY_FILE = "mcp-app.json";const CONNECT_TIMEOUT_MS = 3_000;// Above the editor's own 30 s for a snapshot plus 30 s for an apply.const CALL_TIMEOUT_MS = 90_000;+// Whisper over a whole recording: above the editor's own hour for it.+const TRANSCRIBE_TIMEOUT_MS = 65 * 60_000;const MAX_TOKEN_LINE_BYTES = 256;const MAX_ANSWER_BYTES = 256 * 1024 * 1024;@@ -260,7 +263,9 @@ export async function connectToApp(/** A call that changes nothing, so running it again here after a lost answer is safe. */function isReadOnly(name: string): boolean {if (name === LIST_PROJECTS_TOOL) return true;- if (name === "createCheckpoint" || name === "restoreCheckpoint") return false;+ if (name === "createCheckpoint" || name === "restoreCheckpoint" || name === TRANSCRIBE_TOOL) {+ return false;+ }return !isMutatingTool(name) && !PROJECT_EDIT_TOOLS.has(name);}@@ -297,7 +302,7 @@ export function routeToApp(options: {return (await connection.client.callTool({ name, arguments: (args ?? {}) as Record<string, unknown> },undefined,- { timeout: CALL_TIMEOUT_MS },+ { timeout: name === TRANSCRIBE_TOOL ? TRANSCRIBE_TIMEOUT_MS : CALL_TIMEOUT_MS },)) as CallToolResult;} catch {// Closed or timed out: a hung app must not hold every later call too.diff --git a/electron/mcp/editor-document-host.test.ts b/electron/mcp/editor-document-host.test.tsindex 5b11cd19a..4777b6440 100644--- a/electron/mcp/editor-document-host.test.ts+++ b/electron/mcp/editor-document-host.test.ts@@ -114,6 +114,28 @@ describe("EditorDocumentHost", () => {expect(second.sent).toHaveLength(0);});+ it("waits past the usual timeout for a transcription, and passes its outcome through", async () => {+ vi.useFakeTimers();+ const { host, fromRenderer } = setup();+ const editor = new FakeWebContents();+ fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true);+ const pending = host.transcribe("p", "asset_1", "fr");+ await vi.advanceTimersByTimeAsync(5 * 60_000);+ expect(editor.sent[0]).toMatchObject({ op: "transcribe", projectId: "p", language: "fr" });+ await answer(editor, fromRenderer, { status: "transcribed", language: "fr", words: 4 });+ await expect(pending).resolves.toEqual({ status: "transcribed", language: "fr", words: 4 });+ });++ it("answers a transcription at once when its editor goes away", async () => {+ const { host, fromRenderer } = setup();+ const editor = new FakeWebContents();+ fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true);+ const pending = host.transcribe("p", "asset_1", undefined);+ await vi.waitFor(() => expect(editor.sent).toHaveLength(1));+ editor.destroy();+ await expect(pending).resolves.toBe("no-editor");+ });+it("reports an unanswered apply as a timeout, not as a failure", async () => {vi.useFakeTimers();const { host, fromRenderer } = setup();diff --git a/electron/mcp/editor-document-host.ts b/electron/mcp/editor-document-host.tsindex 55d6d6156..4a3be27dd 100644--- a/electron/mcp/editor-document-host.ts+++ b/electron/mcp/editor-document-host.ts@@ -19,12 +19,16 @@ import {type AiEditionMcpHostRequest,type AiEditionMcpHostResponse,type AiEditionMcpHostSnapshot,+ type AiEditionMcpTranscription,} from "../../src/native/contracts";import type { McpApplyResult, McpDocumentHost } from "./openscreen-mcp-server";// Long enough for a save of a large project to land, short enough that a hung// renderer does not hold the client's call open indefinitely.const REQUEST_TIMEOUT_MS = 30_000;+// A transcription runs Whisper over the whole recording, on the CPU at worst. The editor+// going away answers sooner: every pending request is dropped then.+const TRANSCRIBE_TIMEOUT_MS = 60 * 60_000;const TIMED_OUT = Symbol("timed-out");@@ -35,7 +39,8 @@ interface Pending {type RequestBody =| { op: "snapshot" }- | { op: "apply"; document: unknown; expectedRevision: number };+ | { op: "apply"; document: unknown; expectedRevision: number }+ | { op: "transcribe"; projectId: string; assetId: string; language?: string };export class EditorDocumentHost implements McpDocumentHost {private editor: WebContents | null = null;@@ -46,10 +51,10 @@ export class EditorDocumentHost implements McpDocumentHost {if (active === true) {this.editor = event.sender;event.sender.once("destroyed", () => {- if (this.editor === event.sender) this.editor = null;+ if (this.editor === event.sender) this.leave();});} else if (this.editor === event.sender) {- this.editor = null;+ this.leave();}});ipcMain.on(AI_EDITION_MCP_RESPONSE_CHANNEL, (event: IpcMainEvent, message: unknown) => {@@ -64,7 +69,17 @@ export class EditorDocumentHost implements McpDocumentHost {});}- private request(body: RequestBody): Promise<unknown> {+ /** The editor stopped answering: what it was asked will never be answered. */+ private leave(): void {+ this.editor = null;+ for (const [requestId, pending] of this.pending) {+ clearTimeout(pending.timer);+ this.pending.delete(requestId);+ pending.resolve(undefined);+ }+ }++ private request(body: RequestBody, timeoutMs = REQUEST_TIMEOUT_MS): Promise<unknown> {const editor = this.editor;if (!editor || editor.isDestroyed()) return Promise.resolve(undefined);const requestId = randomUUID();@@ -72,7 +87,7 @@ export class EditorDocumentHost implements McpDocumentHost {const timer = setTimeout(() => {this.pending.delete(requestId);resolve(TIMED_OUT);- }, REQUEST_TIMEOUT_MS);+ }, timeoutMs);this.pending.set(requestId, { resolve, timer });try {editor.send(AI_EDITION_MCP_REQUEST_CHANNEL, {@@ -104,4 +119,18 @@ export class EditorDocumentHost implements McpDocumentHost {if (typeof result !== "string") return "no-editor";return result as McpApplyResult;}++ async transcribe(+ projectId: string,+ assetId: string,+ language: string | undefined,+ ): Promise<AiEditionMcpTranscription | "no-editor" | "timeout"> {+ const result = await this.request(+ { op: "transcribe", projectId, assetId, ...(language ? { language } : {}) },+ TRANSCRIBE_TIMEOUT_MS,+ );+ if (result === TIMED_OUT) return "timeout";+ if (!result || typeof result !== "object" || !("status" in result)) return "no-editor";+ return result as AiEditionMcpTranscription;+ }}diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.tsindex dd4688a1f..59adac7f7 100644--- a/electron/mcp/openscreen-mcp-server.ts+++ b/electron/mcp/openscreen-mcp-server.ts@@ -49,8 +49,20 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";import { z } from "zod";+import {+ applyTranscriptionResult,+ withTranscript,+} from "../../src/lib/ai-edition/document/transcript";import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema";-import type { SegmentationSupport } from "../../src/native/contracts";+import {+ segmentsFromStt,+ transcriptFromSegments,+} from "../../src/lib/ai-edition/transcription/fromStt";+import {+ classifyTranscriptionError,+ withTranscriptionFailure,+} from "../../src/lib/ai-edition/transcription/status";+import type { AiEditionMcpTranscription, SegmentationSupport } from "../../src/native/contracts";import {type AgentToolExecution,consentRequired,@@ -70,6 +82,7 @@ import {type ProjectSummary,type ProjectVersion,} from "../ai-edition/document-service";+import type { SttTranscribeRequest, SttTranscribeResponse } from "../stt/transcriptionContract";import type { MediaTools } from "./media-tools";import {IMPORT_MEDIA_TOOL,@@ -107,6 +120,16 @@ export interface McpDocumentHost {/** `null` when no editor window is open or it has no project loaded. */snapshot(): Promise<McpDocumentSnapshot | null>;apply(document: AxcutDocument, expectedRevision: number): Promise<McpApplyResult>;+ /**+ * The editor's own transcription of an asset of the project it holds (`projectId`): a+ * job of its own already under way is waited for rather than run twice. Absent where+ * there is no editor to ask.+ */+ transcribe?(+ projectId: string,+ assetId: string,+ language: string | undefined,+ ): Promise<AiEditionMcpTranscription | "no-editor" | "timeout">;}/** Every project on disk — the subset of `DocumentService` the server needs. */@@ -147,10 +170,34 @@ export interface McpToolDeps extends ImportMediaDeps {cursor?: CursorTelemetryReader;/** Whether this machine can cut out the camera background; absent where nothing can ask. */cameraSegmentation?: () => SegmentationSupport;+ /** The main process's on-device Whisper (`SttManager.transcribe`). Absent: no transcribe tool. */+ transcribe?(request: SttTranscribeRequest): Promise<SttTranscribeResponse>;version: string;}export const LIST_PROJECTS_TOOL = "listProjects";+export const TRANSCRIBE_TOOL = "transcribe";++const transcribeArgs = z.object({+ assetId: z+ .string()+ .min(1)+ .optional()+ .describe("The asset to transcribe, from getCurrentDocument. Default: the primary asset."),+ language: z+ .string()+ .regex(/^(auto|[a-z]{2,3})$/, 'an ISO 639-1 code such as "en", or "auto"')+ .optional()+ .describe(+ 'Transcribe again in this language (ISO 639-1, such as "en" or "fr"), or "auto" to let Whisper detect it. Omit it to keep a transcript the asset already has.',+ ),+});++const TRANSCRIBE_DESCRIPTION =+ "Transcribe one asset's speech with OpenScreen's on-device Whisper, as the editor does when it opens a project, and store the transcript in the project: getTranscript, getTranscriptWords, setWordText, removeWords and the captions all read it. A take from stopRecording has none until this runs. Without `language`, an asset that already has a transcript keeps it, and a transcription the editor is already running is waited for rather than run twice; with `language`, the asset is transcribed again in it, and the word corrections that still match are carried over. The project's vocabulary (setVocabulary) primes Whisper. It runs on this computer and can take minutes on a long recording, more the first time, when the speech model downloads. Media with no audio fails with failure.kind `no-audio`, which the asset then remembers (assets[].transcriptionFailure in getCurrentDocument). The result gives the language and the word count; it needs no checkpoint, and restoreCheckpoint keeps a transcript made after its checkpoint.";++const TRANSCRIBE_NOTE =+ "Speech: transcribe makes an asset's transcript with OpenScreen's on-device Whisper. Everything that reads speech needs it (getTranscript, getTranscriptWords, removeWords, the captions); a fresh take has none until you call it.";/** The in-app server's edit switch, as AI settings label it (`mcpServer.editsLabel`). */export const MCP_EDITS_SETTING = "'MCP edits' under MCP server in AI settings";@@ -234,6 +281,8 @@ export function mcpInstructions(options: {/** The media tools' preamble. Both servers register them; only the app's channel for* `openscreen mcp` (app-channel.ts), which no agent reads, goes without. */media?: string;+ /** The transcribe tool is served. */+ transcribe?: boolean;}): string {const { transport, editsAllowed, media } = options;const preamble = [@@ -245,6 +294,7 @@ export function mcpInstructions(options: {`startRecording, exportVideo and createProject are refused too while ${MCP_EDITS_SETTING} is off.`,]: []),+ ...(options.transcribe ? [TRANSCRIBE_NOTE] : []),"",].join("\n");return (@@ -396,7 +446,7 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner {return {ok: true,resultJson: JSON.stringify({ ok: true, restored: id }),- document: checkpoint.document,+ document: keepTranscriptsMadeSince(checkpoint.document, document),sidecars: [...checkpoint.sidecars].map(([path, content]) => ({ path, content })),};}@@ -486,8 +536,9 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner {if ((await deps.projects.projectOpenInEditor()) === projectId) {return textResult(new ProjectOpenInEditorError(projectId).message, true);}- // A restore IS the way back, so it needs no checkpoint of its own.- if (name !== "restoreCheckpoint" && !hasCheckpoint(projectId)) {+ // A restore IS the way back, so it needs no checkpoint of its own; a transcript is+ // no edit to take back (restoreCheckpoint keeps it), so it needs none either.+ if (name !== "restoreCheckpoint" && name !== TRANSCRIBE_TOOL && !hasCheckpoint(projectId)) {return textResult(CHECKPOINT_REQUIRED_MESSAGES[deps.transport], true);}// Refused if anything read the project since (the editor opening it would@@ -556,7 +607,139 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner {return runOnFile(projectId, read, name, args, call);}- return (name, args, options = {}) => {+ /** Whisper runs one transcription at a time; this runner's wait their turn here. */+ let whisperQueue: Promise<unknown> = Promise.resolve();++ /** Lands a change computed outside the queue, as an edit of the project as it is by then. */+ function landEdit(projectId: string, editsAllowed: boolean, edit: RunnerEdit) {+ return runner(TRANSCRIBE_TOOL, { projectId }, { editsAllowed, edit });+ }++ /**+ * The transcribe tool. Whisper takes minutes, so it runs OUTSIDE the queue: other calls+ * go on meanwhile, and only the landing of the transcript is queued, as an edit of the+ * project as it is by then. The project open in the editor is transcribed by the editor's+ * own job instead, so the two never run the same transcription side by side.+ */+ async function transcribe(rawArgs: unknown, editsAllowed: boolean): Promise<CallToolResult> {+ const { projectId, args } = splitProjectId(rawArgs);+ const parsed = transcribeArgs.safeParse(args ?? {});+ if (!parsed.success) return textResult(JSON.stringify({ error: parsed.error.message }), true);+ if (!editsAllowed) {+ return textResult(consentRequired(TRANSCRIBE_TOOL, args, MCP_EDITS_SETTING).resultJson, true);+ }+ const { language } = parsed.data;+ const snapshot = await deps.host.snapshot();+ const openId = openProjectId(snapshot);+ const targetId = projectId ?? openId;+ if (!targetId) return textResult(NO_PROJECT_MESSAGES[deps.transport], true);+ const onEditor = openId === targetId;+ let document: AxcutDocument;+ if (onEditor) {+ const read = documentSchema.safeParse(snapshot?.document);+ if (!read.success) {+ return textResult("The project open in the editor could not be read.", true);+ }+ document = read.data;+ } else {+ const read = await deps.projects.getProjectForUpdate(targetId).catch((error) => {+ if (error instanceof DocumentNotFoundError) return null;+ throw error;+ });+ if (!read) {+ return textResult(+ `No project has the id "${targetId}". Call listProjects for the valid ids.`,+ true,+ );+ }+ document = read.document;+ }+ const assetId =+ parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id;+ const asset = document.assets.find((a) => a.id === assetId);+ if (!asset) {+ return textResult(+ JSON.stringify({+ error: `Unknown asset: ${assetId ?? "(none)"}. The project has: ${document.assets.map((a) => a.id).join(", ") || "no assets"}.`,+ }),+ true,+ );+ }+ if (onEditor && deps.host.transcribe) {+ return editorTranscription(+ await deps.host.transcribe(targetId, asset.id, language),+ asset.id,+ );+ }+ const existing = document.transcripts.find((t) => t.assetId === asset.id);+ if (language === undefined && existing) {+ return textResult(JSON.stringify({ ...transcriptSummary(existing), unchanged: true }));+ }+ // An editor this process cannot reach would save over the transcript: say so now,+ // not after minutes of Whisper.+ if ((await deps.projects.projectOpenInEditor()) === targetId) {+ return textResult(new ProjectOpenInEditorError(targetId).message, true);+ }+ const whisper = deps.transcribe;+ if (!whisper) return textResult("Transcription is not available here.", true);+ const started = whisperQueue.then(() =>+ whisper({+ sourcePath: asset.originalPath,+ language: language === "auto" ? undefined : language,+ prompt: document.project.vocabulary,+ }),+ );+ whisperQueue = started.catch(() => undefined);+ let response: SttTranscribeResponse;+ try {+ response = await started;+ } catch (error) {+ const failure = classifyTranscriptionError(error);+ // Remembered on the asset as the editor remembers it: best effort, the answer stands.+ await landEdit(targetId, editsAllowed, async (current) => {+ const next = withTranscriptionFailure(current, asset.id, failure);+ return { ok: true, resultJson: "", ...(next === current ? {} : { document: next }) };+ });+ return textResult(+ JSON.stringify({+ error: `The transcription failed: ${failure.message}`,+ failure: { kind: failure.kind, message: failure.message },+ }),+ true,+ );+ }+ const transcript = transcriptFromSegments(+ asset.id,+ segmentsFromStt(response).segments,+ response.detectedLanguage ?? language,+ );+ return landEdit(targetId, editsAllowed, async (current) => {+ if (!current.assets.some((a) => a.id === asset.id)) {+ return {+ ok: false,+ resultJson: JSON.stringify({+ error: `Asset ${asset.id} was removed from the project while it was transcribed.`,+ }),+ };+ }+ const landed = applyTranscriptionResult(current, transcript);+ return {+ ok: true,+ document: landed.document,+ resultJson: JSON.stringify({+ ...transcriptSummary(transcript),+ ...(landed.dropped > 0 ? { correctionsLost: landed.dropped } : {}),+ }),+ };+ });+ }++ const runner: McpToolRunner = (name, args, options = {}) => {+ if (name === TRANSCRIBE_TOOL && !options.edit) {+ return transcribe(args, options.editsAllowed ?? deps.editsAllowed()).catch((error) =>+ textResult(`Tool failed: ${errorMessage(error)}`, true),+ );+ }// The setting is read when the call's turn comes: it may change while it waits.const runNow = async () => {const editsAllowed = options.editsAllowed ?? deps.editsAllowed();@@ -573,6 +756,71 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner {queue = next.catch(() => undefined);return next.catch((error) => textResult(`Tool failed: ${errorMessage(error)}`, true));};+ return runner;+}++/** What a transcription made, in the terms a caller acts on. */+function transcriptSummary(transcript: AxcutDocument["transcripts"][number]) {+ return {+ assetId: transcript.assetId,+ language: transcript.language,+ words: transcript.words.length,+ ...(transcript.words.length === 0 ? { note: "No speech was found in this media." } : {}),+ };+}++/** The editor's answer about a transcription it ran or waited for, as this tool's result. */+function editorTranscription(+ outcome: AiEditionMcpTranscription | "no-editor" | "timeout",+ assetId: string,+): CallToolResult {+ if (outcome === "timeout") {+ return textResult(+ "OpenScreen did not finish this transcription in time; it may still be running in the editor. Call getCurrentDocument later: assets[].hasTranscript says when it has landed.",+ true,+ );+ }+ if (outcome === "no-editor" || outcome.status === "unavailable") {+ return textResult(+ "The editor closed the project before its transcription finished. Call transcribe again with its projectId.",+ true,+ );+ }+ if (outcome.status === "failed") {+ return textResult(+ JSON.stringify({+ error: `The transcription failed: ${outcome.failure.message}`,+ failure: outcome.failure,+ }),+ true,+ );+ }+ return textResult(+ JSON.stringify({+ assetId,+ language: outcome.language,+ words: outcome.words,+ ...(outcome.words === 0 ? { note: "No speech was found in this media." } : {}),+ }),+ );+}++/**+ * A transcript is the media read, not an edit to take back: a restore keeps the one an+ * asset was given since the checkpoint, where the checkpoint had none for it.+ */+function keepTranscriptsMadeSince(+ checkpoint: AxcutDocument,+ current: AxcutDocument,+): AxcutDocument {+ let restored = checkpoint;+ for (const transcript of current.transcripts) {+ const { assetId } = transcript;+ if (!restored.assets.some((a) => a.id === assetId)) continue;+ if (restored.transcripts.some((t) => t.assetId === assetId)) continue;+ restored = withTranscript(restored, transcript);+ }+ return restored;}export function createOpenScreenMcpServer(@@ -588,6 +836,7 @@ export function createOpenScreenMcpServer(transport: deps.transport,editsAllowed: deps.editsAllowed(),media: media?.preamble,+ transcribe: deps.transcribe !== undefined,}),},);@@ -632,6 +881,17 @@ export function createOpenScreenMcpServer((args: unknown) => runTool(tool.name, args),);}+ if (deps.transcribe) {+ server.registerTool(+ TRANSCRIBE_TOOL,+ {+ description: TRANSCRIBE_DESCRIPTION,+ inputSchema: transcribeArgs.extend({ projectId: projectIdArg }),+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },+ },+ (args: unknown) => runTool(TRANSCRIBE_TOOL, args),+ );+ }for (const tool of checkpointTools(media !== undefined)) {server.registerTool(tool.name,diff --git a/electron/mcp/transcribe-tool.test.ts b/electron/mcp/transcribe-tool.test.tsnew file mode 100644index 000000000..32bc4670e--- /dev/null+++ b/electron/mcp/transcribe-tool.test.ts@@ -0,0 +1,216 @@+// The transcribe tool through the runner both servers share: a project on file is+// transcribed by the injected Whisper and lands like an edit; the project open in the+// editor is handed to the editor's own job instead.++import { mkdtempSync, rmSync } from "node:fs";+import os from "node:os";+import path from "node:path";+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ type AxcutDocument,+ createEmptyDocument,+ documentSchema,+} from "../../src/lib/ai-edition/schema";+import { DocumentService } from "../ai-edition/document-service";+import type { SttTranscribeRequest, SttTranscribeResponse } from "../stt/transcriptionContract";+import {+ createToolRunner,+ type McpApplyResult,+ type McpDocumentHost,+ mcpInstructions,+ TRANSCRIBE_TOOL,+} from "./openscreen-mcp-server";++function fixtureDocument(projectId: string): AxcutDocument {+ const base = createEmptyDocument({+ title: projectId,+ projectId,+ createdAt: "2026-01-01T00:00:00.000Z",+ });+ return documentSchema.parse({+ ...base,+ project: { ...base.project, primaryAssetId: "asset_1", vocabulary: "OpenScreen" },+ assets: [+ { id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 30 },+ ],+ });+}++const SPEECH: SttTranscribeResponse = {+ segments: [{ text: "Hello world", startSec: 0, endSec: 1 }],+ wordSegments: [+ { word: "Hello", startSec: 0, endSec: 0.5 },+ { word: "world", startSec: 0.5, endSec: 1 },+ ],+ detectedLanguage: "en",+ backend: "whispercpp-cpu",+};++/** No editor holds a project unless a test opens one. */+class Host implements McpDocumentHost {+ document: AxcutDocument | null = null;+ transcribe = vi.fn(async () => ({ status: "transcribed" as const, language: "fr", words: 3 }));+ async snapshot() {+ return this.document ? { document: structuredClone(this.document), revision: 1 } : null;+ }+ async apply(document: AxcutDocument): Promise<McpApplyResult> {+ this.document = document;+ return "applied";+ }+}++let dir: string;+let projects: DocumentService;++beforeEach(() => {+ dir = mkdtempSync(path.join(os.tmpdir(), "openscreen-mcp-transcribe-"));+ projects = new DocumentService(path.join(dir, "projects"), dir);+});++afterEach(() => {+ rmSync(dir, { recursive: true, force: true });+});++function runnerWith(whisper: (request: SttTranscribeRequest) => Promise<SttTranscribeResponse>) {+ const host = new Host();+ const transcribe = vi.fn(whisper);+ const run = createToolRunner({+ transport: "stdio",+ host,+ projects,+ editsAllowed: () => true,+ transcribe,+ version: "0.0.0",+ });+ return { run, host, transcribe };+}++function payload(result: CallToolResult): Record<string, unknown> {+ const part = result.content[0];+ return JSON.parse(part?.type === "text" ? part.text : "{}");+}++describe("transcribe, on a project's file", () => {+ it("transcribes with the project's vocabulary and lands the words, with no checkpoint", async () => {+ await projects.saveProject(fixtureDocument("proj_1"));+ const { run, transcribe } = runnerWith(async () => SPEECH);++ const result = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" });++ expect(result.isError).toBeUndefined();+ expect(payload(result)).toMatchObject({ assetId: "asset_1", language: "en", words: 2 });+ expect(transcribe).toHaveBeenCalledWith({+ sourcePath: "/tmp/rec.mp4",+ language: undefined,+ prompt: "OpenScreen",+ });+ const saved = await projects.getProject("proj_1");+ expect(saved.transcripts[0]?.words.map((w) => w.text)).toEqual(["Hello", "world"]);+ });++ it("keeps a transcript without a language, and redoes it with one", async () => {+ await projects.saveProject(fixtureDocument("proj_1"));+ const { run, transcribe } = runnerWith(async () => SPEECH);+ await run(TRANSCRIBE_TOOL, { projectId: "proj_1" });++ const kept = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" });+ expect(payload(kept)).toMatchObject({ unchanged: true, words: 2 });+ expect(transcribe).toHaveBeenCalledTimes(1);++ await run(TRANSCRIBE_TOOL, { projectId: "proj_1", language: "fr" });+ expect(transcribe).toHaveBeenLastCalledWith(expect.objectContaining({ language: "fr" }));+ await run(TRANSCRIBE_TOOL, { projectId: "proj_1", language: "auto" });+ expect(transcribe).toHaveBeenLastCalledWith(expect.objectContaining({ language: undefined }));+ });++ it("reports a silent recording as the editor does, and remembers it on the asset", async () => {+ await projects.saveProject(fixtureDocument("proj_1"));+ const { run } = runnerWith(async () => {+ throw new Error("No decodable audio in /tmp/rec.mp4");+ });++ const result = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" });++ expect(result.isError).toBe(true);+ expect(payload(result).failure).toMatchObject({ kind: "no-audio" });+ const saved = await projects.getProject("proj_1");+ expect(saved.assets[0]?.transcriptionFailure?.kind).toBe("no-audio");+ });++ it("refuses an unknown asset, and while MCP edits are off", async () => {+ await projects.saveProject(fixtureDocument("proj_1"));+ const { run, transcribe } = runnerWith(async () => SPEECH);+ const unknown = await run(TRANSCRIBE_TOOL, { projectId: "proj_1", assetId: "nope" });+ expect(String(payload(unknown).error)).toContain("asset_1");++ const off = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" }, { editsAllowed: false });+ expect(payload(off).code).toBe("consent_required");+ expect(transcribe).not.toHaveBeenCalled();+ });++ it("lets other calls run while Whisper works", async () => {+ await projects.saveProject(fixtureDocument("proj_1"));+ const whisper: { finish: ((response: SttTranscribeResponse) => void) | null } = {+ finish: null,+ };+ const { run } = runnerWith(+ () =>+ new Promise((resolve) => {+ whisper.finish = resolve;+ }),+ );+ const pending = run(TRANSCRIBE_TOOL, { projectId: "proj_1" });+ await vi.waitFor(() => expect(whisper.finish).not.toBeNull());+ // Whisper is still working, and a read goes through all the same.+ const read = await run("getCurrentDocument", { projectId: "proj_1" });+ expect(read.isError).toBeUndefined();+ whisper.finish?.(SPEECH);+ expect((await pending).isError).toBeUndefined();+ });++ it("survives a restore to a checkpoint made before it", async () => {+ await projects.saveProject(fixtureDocument("proj_1"));+ const { run } = runnerWith(async () => SPEECH);+ const checkpoint = payload(await run("createCheckpoint", { projectId: "proj_1" }));+ await run(TRANSCRIBE_TOOL, { projectId: "proj_1" });++ await run("restoreCheckpoint", { projectId: "proj_1", checkpointId: checkpoint.checkpointId });++ expect((await projects.getProject("proj_1")).transcripts).toHaveLength(1);+ });+});++describe("the instructions", () => {+ it("name the transcribe tool only where it is served", () => {+ expect(mcpInstructions({ transport: "stdio", editsAllowed: true, transcribe: true })).toContain(+ "transcribe makes an asset's transcript",+ );+ expect(mcpInstructions({ transport: "stdio", editsAllowed: true })).not.toContain(+ "transcribe makes",+ );+ });+});++describe("transcribe, on the project open in the editor", () => {+ it("hands it to the editor's own job rather than running Whisper beside it", async () => {+ const { run, host, transcribe } = runnerWith(async () => SPEECH);+ host.document = fixtureDocument("proj_1");++ const result = await run(TRANSCRIBE_TOOL, { language: "fr" });++ expect(host.transcribe).toHaveBeenCalledWith("proj_1", "asset_1", "fr");+ expect(transcribe).not.toHaveBeenCalled();+ expect(payload(result)).toMatchObject({ assetId: "asset_1", language: "fr", words: 3 });+ });++ it("says so when the editor closes the project mid-run", async () => {+ const { run, host } = runnerWith(async () => SPEECH);+ host.document = fixtureDocument("proj_1");+ host.transcribe.mockResolvedValueOnce("no-editor" as never);++ const result = await run(TRANSCRIBE_TOOL, {});++ expect(result.isError).toBe(true);+ });+});diff --git a/src/components/ai-edition/v4/MediaStage.test.ts b/src/components/ai-edition/v4/MediaStage.test.tsindex 51d9c789e..390b3751f 100644--- a/src/components/ai-edition/v4/MediaStage.test.ts+++ b/src/components/ai-edition/v4/MediaStage.test.ts@@ -1,15 +1,11 @@import { describe, expect, it, vi } from "vitest";+import { VOCABULARY_MAX_CHARS, withVocabulary } from "@/lib/ai-edition/document/transcript";import {type AxcutTranscript,createEmptyDocument,parseDocumentFile,} from "@/lib/ai-edition/schema";-import {- addSelectedAssetToTimeline,- detectedLanguageForTranscript,- VOCABULARY_MAX_CHARS,- withVocabulary,-} from "./MediaStage";+import { addSelectedAssetToTimeline, detectedLanguageForTranscript } from "./MediaStage";function transcript(language: string, text = ""): AxcutTranscript {return {diff --git a/src/components/ai-edition/v4/MediaStage.tsx b/src/components/ai-edition/v4/MediaStage.tsxindex 7923ab04a..9e93158cc 100644--- a/src/components/ai-edition/v4/MediaStage.tsx+++ b/src/components/ai-edition/v4/MediaStage.tsx@@ -3,12 +3,8 @@ import { useMemo, useRef, useState } from "react";import { toast } from "sonner";import { Tooltip } from "@/components/ui/tooltip";import { useI18n, useScopedT } from "@/contexts/I18nContext";-import type {- AxcutAsset,- AxcutDocument,- AxcutTranscript,- TranscriptLanguageCode,-} from "@/lib/ai-edition/schema";+import { VOCABULARY_MAX_CHARS, withVocabulary } from "@/lib/ai-edition/document/transcript";+import type { AxcutAsset, AxcutTranscript, TranscriptLanguageCode } from "@/lib/ai-edition/schema";import { useProjectStore } from "@/lib/ai-edition/store/projectStore";import {useAssetTranscriptions,@@ -53,19 +49,6 @@ export function detectedLanguageForTranscript(transcript: AxcutTranscript | nullreturn transcript.language;}-/**- * Whisper reads at most the last 223 tokens of its prompt; 600 characters of- * names and terms stays under that for Latin scripts (issue #1023).- */-export const VOCABULARY_MAX_CHARS = 600;--/** The document with its names and terms set, or null when nothing changed. */-export function withVocabulary(document: AxcutDocument, text: string): AxcutDocument | null {- const vocabulary = text.trim().slice(0, VOCABULARY_MAX_CHARS) || undefined;- if (vocabulary === document.project.vocabulary) return null;- return { ...document, project: { ...document.project, vocabulary } };-}-export async function addSelectedAssetToTimeline(selected: Pick<AxcutAsset, "id" | "label" | "originalPath"> | null,onAddToTimeline: (assetId: string) => Promise<void>,diff --git a/src/lib/ai-edition/document/transcribe.ts b/src/lib/ai-edition/document/transcribe.tsindex cffddf51d..4f3f1c701 100644--- a/src/lib/ai-edition/document/transcribe.ts+++ b/src/lib/ai-edition/document/transcribe.ts@@ -13,7 +13,8 @@ import {} from "@/lib/captioning";import type { SttRendererStatus } from "@/lib/captioning/transcribe";import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract";-import type { AxcutDocument, AxcutTranscript, AxcutTranscriptSegment, AxcutWord } from "../schema";+import type { AxcutDocument, AxcutTranscript } from "../schema";+import { transcriptFromSegments } from "../transcription/fromStt";/*** What the caller can show while a transcription runs. `completedSec` /@@ -119,54 +120,13 @@ export async function transcribeAsset(});});- const segments: AxcutTranscriptSegment[] = [];- const words: AxcutWord[] = [];-- for (let segIndex = 0; segIndex < result.segments.length; segIndex++) {- const seg = result.segments[segIndex];- const segId = `seg_${segIndex + 1}`;- const wordIds: string[] = [];-- const tokens = seg.text.trim().split(/\s+/).filter(Boolean);- if (tokens.length === 0) continue;-- // Never end before the start: the schema rejects it, and one such word made the whole- // project unreadable (see `raiseInvertedTranscriptEnds`, which repairs old files).- const segEndSec = Math.max(seg.endSec, seg.startSec);- const wordDuration = (segEndSec - seg.startSec) / tokens.length;- for (let w = 0; w < tokens.length; w++) {- const wordId = `word_${words.length + 1}`;- const startSec = seg.startSec + w * wordDuration;- const endSec = startSec + wordDuration;- words.push({- id: wordId,- segmentId: segId,- startSec,- endSec,- text: tokens[w],- });- wordIds.push(wordId);- }-- segments.push({- id: segId,- kind: "speech" as const,- startSec: seg.startSec,- endSec: segEndSec,- text: seg.text,- wordIds,- });- }-- return {+ // Prefer the model-reported language (covers both forced picks and auto-detect),+ // then the input option.+ return transcriptFromSegments(assetId,- // Prefer the model-reported language (covers both forced picks and- // auto-detect); fall back to the input option, then "auto" when- // nothing was detected (very rare on tiny.en — usually a no-audio run).- language: result.detectedLanguage ?? options.language ?? "auto",- segments,- words,- };+ result.segments,+ result.detectedLanguage ?? options.language,+ );}// `withTranscript` used to live here. It moved to `document/transcript.ts`, next todiff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.tsindex 1bfdb7ccf..b702d416d 100644--- a/src/lib/ai-edition/document/transcript.ts+++ b/src/lib/ai-edition/document/transcript.ts@@ -257,3 +257,46 @@ export function carryOverWordEdits(return { transcript, carried, dropped: edits.length - carried };}++/**+ * A finished transcription landing on the document: it replaces the asset's transcript,+ * carrying the user's word corrections over (`carryOverWordEdits`), and clears the failure+ * the asset remembered from an earlier run. What could not be carried is counted in+ * `dropped`. The editor's queue and the MCP `transcribe` tool both land a run through here.+ */+export function applyTranscriptionResult(+ document: AxcutDocument,+ transcript: AxcutTranscript,+): { document: AxcutDocument; dropped: number } {+ const merged = carryOverWordEdits(+ document.transcripts.find((t) => t.assetId === transcript.assetId),+ transcript,+ );+ return {+ document: withTranscript(+ {+ ...document,+ assets: document.assets.map((a) =>+ a.id === transcript.assetId && a.transcriptionFailure+ ? { ...a, transcriptionFailure: null }+ : a,+ ),+ },+ merged.transcript,+ ),+ dropped: merged.dropped,+ };+}++/**+ * Whisper reads at most the last 223 tokens of its prompt; 600 characters of+ * names and terms stays under that for Latin scripts (issue #1023).+ */+export const VOCABULARY_MAX_CHARS = 600;++/** The document with its names and terms set, or null when nothing changed. */+export function withVocabulary(document: AxcutDocument, text: string): AxcutDocument | null {+ const vocabulary = text.trim().slice(0, VOCABULARY_MAX_CHARS) || undefined;+ if (vocabulary === document.project.vocabulary) return null;+ return { ...document, project: { ...document.project, vocabulary } };+}diff --git a/src/lib/ai-edition/store/mcpDocumentHost.ts b/src/lib/ai-edition/store/mcpDocumentHost.tsindex d1c82ff16..d99956b38 100644--- a/src/lib/ai-edition/store/mcpDocumentHost.ts+++ b/src/lib/ai-edition/store/mcpDocumentHost.ts@@ -1,7 +1,13 @@import { useEffect } from "react";-import type { AiEditionMcpHostRequest, AiEditionMcpHostResponse } from "@/native/contracts";+import type {+ AiEditionMcpHostRequest,+ AiEditionMcpHostResponse,+ AiEditionMcpTranscription,+} from "@/native/contracts";+import { findAssetTranscript } from "../transcription/status";import { applyAgentDocumentIfCurrent } from "./agentDocumentApply";import { useProjectStore } from "./projectStore";+import { useTranscriptionStore } from "./transcriptionStore";/*** The editor's side of the MCP server (electron/mcp/): hand it the live document@@ -16,6 +22,7 @@ export async function answerMcpHostRequest(if (request.op === "snapshot") {return document ? { document, revision } : null;}+ if (request.op === "transcribe") return transcribeForMcp(request);if (!document) return "no-live-document";// The revision counter restarts when a project is closed, so a quick close// and reopen of ANOTHER project can land on the revision the call was given.@@ -29,6 +36,33 @@ export async function answerMcpHostRequest(}}+/**+ * The MCP `transcribe` tool on the project this editor holds: through the editor's own+ * queue (`ensureTranscript`), so a job it already runs is waited for, never doubled.+ */+async function transcribeForMcp(+ request: Extract<AiEditionMcpHostRequest, { op: "transcribe" }>,+): Promise<AiEditionMcpTranscription> {+ const held = () => {+ const document = useProjectStore.getState().document;+ return document?.project.id === request.projectId ? document : null;+ };+ if (!held()) return { status: "unavailable" };+ await useTranscriptionStore.getState().ensureTranscript(request.assetId, request.language);+ const job = useTranscriptionStore.getState().jobs[request.assetId];+ const transcript = findAssetTranscript(held(), request.assetId);+ // A transcript the asset kept answers a call that asked for no language, whatever an+ // earlier job left behind.+ if (job?.status === "failed" && job.failure && (request.language || !transcript)) {+ return {+ status: "failed",+ failure: { kind: job.failure.kind, message: job.failure.message },+ };+ }+ if (!transcript) return { status: "unavailable" };+ return { status: "transcribed", language: transcript.language, words: transcript.words.length };+}+/** Makes this editor window the one the MCP server reads and writes, while mounted. */export function useMcpDocumentHost(): void {useEffect(() => window.electronAPI?.onAiEditionMcpRequest?.(answerMcpHostRequest), []);diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.tsindex f58291e41..78e5766a0 100644--- a/src/lib/ai-edition/store/transcriptionStore.test.ts+++ b/src/lib/ai-edition/store/transcriptionStore.test.ts@@ -2,6 +2,7 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { toastText } from "@/i18n/toastText";import type { AxcutDocument, AxcutTranscript } from "../schema";+import { answerMcpHostRequest } from "./mcpDocumentHost";import { useProjectStore } from "./projectStore";import { useTranscriptionStore, whenTranscriptionIdle } from "./transcriptionStore";@@ -452,6 +453,70 @@ describe("useTranscriptionStore", () => {expect(useProjectStore.getState().document?.transcripts).toHaveLength(1);});+ it("ensureTranscript waits for the editor's own run, and the MCP host reports it", async () => {+ const firstRun = deferred();+ transcribeMocks.transcribeAsset.mockImplementation(+ async (_doc: AxcutDocument, assetId: string) => {+ await firstRun.promise;+ return transcriptFor(assetId);+ },+ );+ loadDocument(makeDoc(["asset_1"]));+ useTranscriptionStore.getState().sync(useProjectStore.getState().document);+ await Promise.resolve();+ expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("running");++ const answered = answerMcpHostRequest({+ requestId: "r",+ op: "transcribe",+ projectId: "proj_1",+ assetId: "asset_1",+ });+ firstRun.release();++ await expect(answered).resolves.toEqual({ status: "transcribed", language: "en", words: 1 });+ expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1);+ });++ it("ensureTranscript keeps a transcript unless a language is asked for", async () => {+ transcribeMocks.transcribeAsset.mockImplementation(+ async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId),+ );+ loadDocument({ ...makeDoc(["asset_1"]), transcripts: [transcriptFor("asset_1")] });++ await useTranscriptionStore.getState().ensureTranscript("asset_1");+ expect(transcribeMocks.transcribeAsset).not.toHaveBeenCalled();++ await useTranscriptionStore.getState().ensureTranscript("asset_1", "fr");+ expect(transcribeMocks.transcribeAsset).toHaveBeenCalledWith(+ expect.anything(),+ "asset_1",+ expect.objectContaining({ language: "fr" }),+ );+ });++ it("the MCP host reports the failure of the run it asked for", async () => {+ transcribeMocks.transcribeAsset.mockRejectedValue(new Error("No decodable audio in x"));+ loadDocument(makeDoc(["asset_1"]));++ await expect(+ answerMcpHostRequest({+ requestId: "r",+ op: "transcribe",+ projectId: "proj_1",+ assetId: "asset_1",+ }),+ ).resolves.toMatchObject({ status: "failed", failure: { kind: "no-audio" } });+ await expect(+ answerMcpHostRequest({+ requestId: "r",+ op: "transcribe",+ projectId: "another",+ assetId: "asset_1",+ }),+ ).resolves.toEqual({ status: "unavailable" });+ });+it("still honours a manual request without the auto pass", async () => {// biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridgedelete (window as any).electronAPI;diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.tsindex 316f30791..341bd72f8 100644--- a/src/lib/ai-edition/store/transcriptionStore.ts+++ b/src/lib/ai-edition/store/transcriptionStore.ts@@ -27,7 +27,7 @@ import { toast } from "sonner";import { create } from "zustand";import { toastText as translateToast } from "@/i18n/toastText";import { transcribeAsset } from "../document/transcribe";-import { carryOverWordEdits, withTranscript } from "../document/transcript";+import { applyTranscriptionResult } from "../document/transcript";import type { AxcutDocument } from "../schema";import {type AssetTranscriptionView,@@ -44,6 +44,7 @@ import {type TranscriptionProgress,transcriptionFailureHintKey,transcriptRelevantAssetIds,+ withTranscriptionFailure,} from "../transcription/status";import { useProjectStore } from "./projectStore";@@ -87,6 +88,14 @@ interface TranscriptionState {* leave that button unable to transcribe the talking clip next to it.*/requestTimelineTranscripts: () => Promise<void>;+ /**+ * What a caller outside the editor asks for (the MCP `transcribe` tool): a transcript of+ * this asset, without a second run beside the editor's own. Without `language`, an asset+ * that has one is left alone and a job already queued or running is waited for; with+ * it, the job is waited for only when it runs in that language, else a run is asked for.+ * Resolves once the run settles.+ */+ ensureTranscript: (assetId: string, language?: string) => Promise<void>;reset: () => void;}@@ -189,6 +198,17 @@ export const useTranscriptionStore = create<TranscriptionState>((set, get) => ({).then(() => undefined);},+ ensureTranscript(assetId, language) {+ const document = useProjectStore.getState().document;+ if (document && document.project.id !== get().projectId) get().sync(document);+ const job = get().jobs[assetId];+ if (job && job.status !== "failed" && (language === undefined || language === job.language)) {+ return waitForSettle(assetId);+ }+ if (language === undefined && findAssetTranscript(document, assetId)) return Promise.resolve();+ return get().request(assetId, language);+ },+reset() {abortActiveRun();const pending = Object.keys(get().jobs);@@ -318,8 +338,7 @@ async function persistPermanentFailure(assetId: string,failure: TranscriptionFailure,): Promise<void> {- const kind = failure.kind;- if (!isPermanentFailure(kind)) return;+ if (!isPermanentFailure(failure.kind)) return;const project = useProjectStore.getState();const doc = project.document;if (!doc || doc.project.id !== projectId) return;@@ -328,21 +347,7 @@ async function persistPermanentFailure(// false rather than throwing, and a note on the asset is not worth a second// message on top of the one the user already got.const persisted = await project.saveDocument(- {- ...doc,- assets: doc.assets.map((a) =>- a.id === assetId- ? {- ...a,- transcriptionFailure: {- kind,- message: failure.message,- at: new Date().toISOString(),- },- }- : a,- ),- },+ withTranscriptionFailure(doc, assetId, failure),// Bookkeeping, not an edit — it must not become an undo step.{ history: false },);@@ -416,33 +421,17 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise<void> {// hand would go with it. Carry those corrections onto the new words first —// strictly, so nothing is invented (see `carryOverWordEdits`). What could not be// carried is lost; telling the user so is the UI's job, and there is no surface- // for it yet.- const merged = carryOverWordEdits(- current.transcripts.find((t) => t.assetId === assetId),- transcript,- );- if (merged.dropped > 0) {+ // for it yet. The same save clears the verdict remembered on the asset.+ const landed = applyTranscriptionResult(current, transcript);+ if (landed.dropped > 0) {console.warn(- `[transcription] ${merged.dropped} word correction(s) on asset ${assetId} could not be carried over to the new transcript.`,+ `[transcription] ${landed.dropped} word correction(s) on asset ${assetId} could not be carried over to the new transcript.`,);}- // One save: the transcript, and (on a successful retry) the removal of- // the verdict remembered on the asset.// `history: false`: a transcript landing from a background job is not an edit// the user made, and making it the target of the next Ctrl+Z would both surprise// them and throw the transcript away.- await useProjectStore.getState().saveDocument(- withTranscript(- {- ...current,- assets: current.assets.map((a) =>- a.id === assetId && a.transcriptionFailure ? { ...a, transcriptionFailure: null } : a,- ),- },- merged.transcript,- ),- { history: false },- );+ await useProjectStore.getState().saveDocument(landed.document, { history: false });dropJob(assetId, runId);if (job.manual) toast.success(toastText("mediaStage.transcriptReady"));} catch (error) {diff --git a/src/lib/ai-edition/transcription/fromStt.test.ts b/src/lib/ai-edition/transcription/fromStt.test.tsnew file mode 100644index 000000000..85f6dc03d--- /dev/null+++ b/src/lib/ai-edition/transcription/fromStt.test.ts@@ -0,0 +1,50 @@+import { describe, expect, it } from "vitest";+import { segmentsFromStt, transcriptFromSegments } from "./fromStt";++describe("segmentsFromStt", () => {+ it("takes the engine's words, one segment each", () => {+ const { segments, granularity } = segmentsFromStt({+ segments: [{ text: "Hi you", startSec: 0, endSec: 1 }],+ wordSegments: [+ { word: "Hi", startSec: 0, endSec: 0.4 },+ { word: "you", startSec: 0.4, endSec: 1 },+ ],+ });+ expect(granularity).toBe("word");+ expect(segments.map((s) => s.text)).toEqual(["Hi", "you"]);+ });++ it("falls back to the phrases when whisper kept no word", () => {+ const { segments, granularity } = segmentsFromStt({+ segments: [{ text: "Hi you", startSec: 0, endSec: 1 }],+ wordSegments: [],+ });+ expect(granularity).toBe("phrase");+ expect(segments).toEqual([{ text: "Hi you", startSec: 0, endSec: 1 }]);+ });+});++describe("transcriptFromSegments", () => {+ it("spreads each segment's tokens over its span, keeping ids by position", () => {+ const transcript = transcriptFromSegments(+ "asset_1",+ [+ { text: " ", startSec: 0, endSec: 1 },+ { text: "two words", startSec: 2, endSec: 3 },+ ],+ "fr",+ );+ expect(transcript.language).toBe("fr");+ expect(transcript.segments.map((s) => s.id)).toEqual(["seg_2"]);+ expect(transcript.words).toEqual([+ { id: "word_1", segmentId: "seg_2", startSec: 2, endSec: 2.5, text: "two" },+ { id: "word_2", segmentId: "seg_2", startSec: 2.5, endSec: 3, text: "words" },+ ]);+ });++ it("never ends a word before it starts, and says auto when no language came back", () => {+ const transcript = transcriptFromSegments("a", [{ text: "x", startSec: 2, endSec: 1 }], null);+ expect(transcript.language).toBe("auto");+ expect(transcript.words[0]).toMatchObject({ startSec: 2, endSec: 2 });+ });+});diff --git a/src/lib/ai-edition/transcription/fromStt.ts b/src/lib/ai-edition/transcription/fromStt.tsnew file mode 100644index 000000000..6466c027c--- /dev/null+++ b/src/lib/ai-edition/transcription/fromStt.ts@@ -0,0 +1,80 @@+// What the speech engine answers, turned into the document's transcript. Pure, so the+// editor's queue (through the renderer) and the MCP `transcribe` tool (in the main+// process, `electron/mcp/`) build the very same words from the very same answer.++import type { SttTranscribeResponse } from "../../../../electron/stt/transcriptionContract";+import type { CaptionSegment, CaptionTimestampGranularity } from "../../captioning/transcribe";+import type { AxcutTranscript, AxcutTranscriptSegment, AxcutWord } from "../schema";++/**+ * The engine's words as timed segments, one per word. When whisper dropped every word+ * (OOV-heavy speech), its phrase spans instead, so there is still something to edit.+ */+export function segmentsFromStt(+ response: Pick<SttTranscribeResponse, "segments" | "wordSegments">,+): { segments: CaptionSegment[]; granularity: CaptionTimestampGranularity } {+ const words = response.wordSegments ?? [];+ if (words.length > 0) {+ return {+ segments: words.map((w) => ({ startSec: w.startSec, endSec: w.endSec, text: w.word })),+ granularity: "word",+ };+ }+ return {+ segments: (response.segments ?? []).map((s) => ({+ startSec: s.startSec,+ endSec: s.endSec,+ text: s.text,+ })),+ granularity: "phrase",+ };+}++/**+ * Timed segments as an asset's transcript: each segment's tokens become its words,+ * spread evenly over its span.+ *+ * `language` is what the engine settled on, or failing that what was asked for, then+ * "auto" (very rare: usually a run with no audio).+ */+export function transcriptFromSegments(+ assetId: string,+ segments: readonly CaptionSegment[],+ language: string | null | undefined,+): AxcutTranscript {+ const transcriptSegments: AxcutTranscriptSegment[] = [];+ const words: AxcutWord[] = [];++ for (const [index, seg] of segments.entries()) {+ const tokens = seg.text.trim().split(/\s+/).filter(Boolean);+ if (tokens.length === 0) continue;+ const segId = `seg_${index + 1}`;+ const wordIds: string[] = [];+ // Never end before the start: the schema rejects it, and one such word made the whole+ // project unreadable (see `raiseInvertedTranscriptEnds`, which repairs old files).+ const segEndSec = Math.max(seg.endSec, seg.startSec);+ const wordDuration = (segEndSec - seg.startSec) / tokens.length;+ for (let w = 0; w < tokens.length; w++) {+ const wordId = `word_${words.length + 1}`;+ const startSec = seg.startSec + w * wordDuration;+ words.push({+ id: wordId,+ segmentId: segId,+ startSec,+ endSec: startSec + wordDuration,+ text: tokens[w],+ });+ wordIds.push(wordId);+ }+ transcriptSegments.push({+ id: segId,+ kind: "speech" as const,+ startSec: seg.startSec,+ endSec: segEndSec,+ text: seg.text,+ wordIds,+ });+ }++ return { assetId, language: language || "auto", segments: transcriptSegments, words };+}diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.tsindex fe28e88ed..ea317e60e 100644--- a/src/lib/ai-edition/transcription/status.ts+++ b/src/lib/ai-edition/transcription/status.ts@@ -126,6 +126,27 @@ export function isPermanentFailure(kind: TranscriptionFailureKind): kind is Pers/** The failure kinds `assetSchema.transcriptionFailure` accepts. */export type PersistableFailureKind = Exclude<TranscriptionFailureKind, "error">;+/**+ * The document with a deterministic failure remembered on its asset, so the next open+ * shows "no audio" at once instead of re-extracting the audio to find out again. A+ * transient failure is not remembered: the document comes back unchanged.+ */+export function withTranscriptionFailure(+ document: AxcutDocument,+ assetId: string,+ failure: TranscriptionFailure,+ at = new Date().toISOString(),+): AxcutDocument {+ const kind = failure.kind;+ if (!isPermanentFailure(kind)) return document;+ return {+ ...document,+ assets: document.assets.map((a) =>+ a.id === assetId ? { ...a, transcriptionFailure: { kind, message: failure.message, at } } : a,+ ),+ };+}+/*** Map an exception out of `transcribeAsset` onto a failure the UI can explain.* The deterministic cases come from whichever layer decoded the audio, and therediff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.tsindex 132c1cea1..806b66914 100644--- a/src/lib/captioning/transcribe.ts+++ b/src/lib/captioning/transcribe.ts@@ -1,5 +1,6 @@import type { TrimRegion } from "@/components/video-editor/types";import type { SttWordSegment } from "../../../electron/stt/transcriptionContract";+import { segmentsFromStt } from "../ai-edition/transcription/fromStt";export interface CaptionSegment {startSec: number;@@ -138,30 +139,7 @@ function runTranscription(// quiet music intros / room tone. VAD or nothing.return api.transcribe({ ...payload, language: forcedLanguage, prompt: options?.prompt })- .then((result) => {- const words = result.wordSegments ?? [];- let segments: CaptionSegment[];- let granularity: CaptionTimestampGranularity;- if (words.length > 0) {- segments = words.map((w) => ({- startSec: w.startSec,- endSec: w.endSec,- text: w.word,- }));- granularity = "word";- } else {- // ponytail: whisper dropped every word for a segment (e.g. OOV- // heavy); fall back to raw phrase spans so the user still gets- // captions to edit.- segments = (result.segments ?? []).map((s) => ({- startSec: s.startSec,- endSec: s.endSec,- text: s.text,- }));- granularity = "phrase";- }- return { segments, granularity, detectedLanguage: result.detectedLanguage };- })+ .then((result) => ({ ...segmentsFromStt(result), detectedLanguage: result.detectedLanguage })).catch((error: unknown) => {// A run the caller cancelled surfaces as an abort, not as an engine// failure: the store drops it silently instead of toasting the userdiff --git a/src/native/contracts.ts b/src/native/contracts.tsindex 24b193cf2..7576b2cc7 100644--- a/src/native/contracts.ts+++ b/src/native/contracts.ts@@ -339,7 +339,24 @@ export interface AiEditionMcpStatus {*/export type AiEditionMcpHostRequest =| { requestId: string; op: "snapshot" }- | { requestId: string; op: "apply"; document: unknown; expectedRevision: number };+ | { requestId: string; op: "apply"; document: unknown; expectedRevision: number }+ | {+ requestId: string;+ op: "transcribe";+ projectId: string;+ assetId: string;+ /** Absent: keep a transcript the asset has. */+ language?: string;+ };++/**+ * How the editor's own transcription of an asset ended, for the MCP `transcribe` tool.+ * `unavailable`: the editor no longer holds that project, or the run was dropped.+ */+export type AiEditionMcpTranscription =+ | { status: "transcribed"; language: string; words: number }+ | { status: "failed"; failure: { kind: string; message: string } }+ | { status: "unavailable" };export interface AiEditionMcpHostSnapshot {document: unknown;@@ -350,7 +367,7 @@ export type AiEditionMcpApplyResult = "applied" | "conflict" | "save-failed" | "export interface AiEditionMcpHostResponse {requestId: string;- result: AiEditionMcpHostSnapshot | null | AiEditionMcpApplyResult;+ result: AiEditionMcpHostSnapshot | null | AiEditionMcpApplyResult | AiEditionMcpTranscription;}/** Editor → main: `true` when an editor starts answering MCP requests, `false` when it stops. */