Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(mcp): read, style and translate the captions through the agent tools
EtienneLescot1e0ea14caebd
diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.tsindex 26fd98cb5..9cfc65449 100644--- a/electron/ai-edition/agent-tools.test.ts+++ b/electron/ai-edition/agent-tools.test.ts@@ -216,6 +216,10 @@ describe("the mutating-tool table", () => {"setTrim","setWordText","setZoom",+ "setCaptionSettings",+ "putCaptionTranslation",+ "removeCaptionTranslation",+ "setVocabulary",].sort(),);expect(isMutatingTool("getCurrentDocument")).toBe(false);diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.tsindex 2df9ab491..3a60dd38d 100644--- a/electron/ai-edition/agent-tools.ts+++ b/electron/ai-edition/agent-tools.ts@@ -87,6 +87,7 @@ import { SETTING_BOUNDS } from "../../src/lib/projectDefaults";import type { SegmentationSupport } from "../../src/native/contracts";import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription";import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils";+import { CAPTION_TOOL_NAMES, executeCaptionTool, MUTATING_CAPTION_TOOLS } from "./caption-tools";import { editorSettingsSnapshot, executeSetEditorSettings } from "./editor-settings-tool";import { measureTextInShippedFonts } from "./text-measure";@@ -1068,6 +1069,7 @@ export const OPENSCREEN_TOOL_NAMES = ["removeClip","setAspectRatio","setEditorSettings",+ ...CAPTION_TOOL_NAMES,] as const;/**@@ -1145,6 +1147,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set(["removeClip","setAspectRatio","setEditorSettings",+ ...MUTATING_CAPTION_TOOLS,]);export function isMutatingTool(name: string): boolean {@@ -1323,6 +1326,17 @@ export function documentSnapshotForModel(// probe threw. Defaulting that to `false` would put our failure back in// the answer as their fact, one layer lower down.hasCursorTelemetry: availability?.[a.id] ?? null,+ // Whether getTranscriptWords can read it; transcribe makes one when it is false.+ hasTranscript: document.transcripts.some((t) => t.assetId === a.id),+ // A verdict a past run left on the media ("no-audio", "unsupported-audio").+ ...(a.transcriptionFailure+ ? {+ transcriptionFailure: {+ kind: a.transcriptionFailure.kind,+ message: a.transcriptionFailure.message,+ },+ }+ : {}),})),// ponytail: `index`, `reason` and `origin` are here because without them a// clip cannot be DESIGNATED. "Put the demo first" is unanswerable when the@@ -1413,6 +1427,8 @@ export function documentSnapshotForModel(fadeOutMs: t.fadeOutMs,})),hasTranscript: document.transcripts.length > 0 || document.transcript !== null,+ // The names and terms Whisper is primed with on every transcription (setVocabulary).+ vocabulary: document.project.vocabulary ?? null,};}@@ -1783,6 +1799,14 @@ export function executeAgentTool(}switch (name) {+ case "getCaptions":+ case "setCaptionSettings":+ case "getCaptionTranslationUnits":+ case "putCaptionTranslation":+ case "removeCaptionTranslation":+ case "setVocabulary":+ return executeCaptionTool(document, name, args);+case "setAspectRatio": {// The format alone, kept for clients that know it: setEditorSettings does the work.const parsed = setAspectRatioArgs.safeParse(args);diff --git a/electron/ai-edition/caption-tools.test.ts b/electron/ai-edition/caption-tools.test.tsnew file mode 100644index 000000000..4e96e447a--- /dev/null+++ b/electron/ai-edition/caption-tools.test.ts@@ -0,0 +1,236 @@+// The caption tools end to end on a document: turn them on, style and place them,+// translate them as a client would, and check the export draws what was asked.++import { describe, expect, it } from "vitest";+import { CAPTION_STYLES, getCaptionSettings } from "../../src/lib/ai-edition/captions";+import {+ type AxcutDocument,+ createEmptyDocument,+ documentSchema,+} from "../../src/lib/ai-edition/schema";+import { buildSceneDescription } from "../../src/native/sceneDescription";+import { type AgentToolExecution, executeAgentTool } from "./agent-tools";++function fixtureDocument(): AxcutDocument {+ const base = createEmptyDocument({+ title: "Captions",+ projectId: "proj_1",+ createdAt: "2026-01-01T00:00:00.000Z",+ });+ const words = [+ ["w1", "Hello", 0, 0.4],+ ["w2", "there.", 0.4, 0.8],+ ["w3", "Second", 2, 2.4],+ ["w4", "phrase.", 2.4, 2.8],+ ] as const;+ return documentSchema.parse({+ ...base,+ project: { ...base.project, primaryAssetId: "asset_1" },+ assets: [+ { id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 10 },+ ],+ // One segment per word, as Whisper's transcripts are stored.+ transcripts: [+ {+ assetId: "asset_1",+ language: "en",+ segments: words.map(([id, text, startSec, endSec]) => ({+ id: `seg_${id}`,+ kind: "speech",+ startSec,+ endSec,+ text,+ wordIds: [id],+ })),+ words: words.map(([id, text, startSec, endSec]) => ({+ id,+ segmentId: `seg_${id}`,+ startSec,+ endSec,+ text,+ })),+ },+ ],+ timeline: {+ ...base.timeline,+ clips: [+ {+ id: "clip_1",+ assetId: "asset_1",+ sourceStartSec: 0,+ sourceEndSec: 10,+ timelineStartSec: 0,+ timelineEndSec: 10,+ wordRefs: [],+ origin: "user",+ reason: "",+ },+ ],+ },+ });+}++function call(document: AxcutDocument, name: string, args: unknown = {}) {+ const execution: AgentToolExecution = executeAgentTool(document, name, JSON.stringify(args));+ return { ...execution, result: JSON.parse(execution.resultJson) as Record<string, unknown> };+}++function captionTexts(document: AxcutDocument): string[] {+ return buildSceneDescription(document).annotations.map((a) => a.text?.content ?? "");+}++describe("setCaptionSettings", () => {+ it("turns the captions on, and the export draws the transcript", () => {+ const doc = fixtureDocument();+ expect(captionTexts(doc)).toEqual([]);+ const on = call(doc, "setCaptionSettings", { enabled: true });+ expect(on.ok).toBe(true);+ expect(on.document && captionTexts(on.document).join(" ")).toContain("Hello there.");+ });++ it("applies a style, then a plate, then the explicit fields over both", () => {+ const doc = fixtureDocument();+ const set = call(doc, "setCaptionSettings", {+ style: "bold",+ plate: "light",+ fontSize: 72,+ anchorV: "top",+ insetY: 8,+ minWordsPerLine: 3,+ maxWordsPerLine: 5,+ });+ const settings = getCaptionSettings(set.document);+ expect(settings.fontFamily).toBe(CAPTION_STYLES.bold.fontFamily);+ expect(settings.fontSize).toBe(72);+ expect(settings.backgroundColor).toBe("#ffffff");+ // The bold style's yellow would vanish on a light plate: the plate recoloured it.+ expect(settings.color).not.toBe(CAPTION_STYLES.bold.color);+ expect(settings).toMatchObject({ anchorV: "top", insetY: 8, minWordsPerLine: 3 });+ expect(set.result.plate).toBe("light");+ });++ it("refuses what the pane cannot express, and changes nothing", () => {+ const doc = fixtureDocument();+ expect(call(doc, "setCaptionSettings", {}).ok).toBe(false);+ expect(call(doc, "setCaptionSettings", { minWordsPerLine: 9, maxWordsPerLine: 3 }).ok).toBe(+ false,+ );+ expect(call(doc, "setCaptionSettings", { fontSize: 500 }).ok).toBe(false);+ expect(call(doc, "setCaptionSettings", { fontFamily: "Comic Sans" }).ok).toBe(false);+ const missing = call(doc, "setCaptionSettings", { language: "fr" });+ expect(missing.ok).toBe(false);+ expect(String(missing.result.error)).toContain("putCaptionTranslation");+ });++ it("reports the lane the captions really read", () => {+ const set = call(fixtureDocument(), "setCaptionSettings", { lane: "voiceover" });+ expect(set.result.resolvedLane).toBe("recording");+ expect(set.result.laneNote).toBeDefined();+ });+});++describe("getCaptions", () => {+ it("reads the settings back, warns when there is nothing to caption", () => {+ const doc = call(fixtureDocument(), "setCaptionSettings", { enabled: true }).document;+ if (!doc) throw new Error("not applied");+ const read = call(doc, "getCaptions").result;+ expect(read).toMatchObject({ style: "classic", plate: "dark", resolvedLane: "recording" });+ expect(read.warning).toBeUndefined();+ const silent = call({ ...doc, transcripts: [], transcript: null }, "getCaptions").result;+ expect(silent.warning).toBeDefined();+ });+});++describe("translating the captions", () => {+ it("reads phrase units, stores the client's translation, and the export shows it", () => {+ let doc = fixtureDocument();+ const units = call(doc, "getCaptionTranslationUnits", { language: "fr" }).result+ .assets as Array<{+ units: Array<{ id: string; text: string }>;+ }>;+ // Phrases, not words: one unit per sentence.+ expect(units[0]?.units.map((u) => u.text)).toEqual(["Hello there.", "Second phrase."]);+ const [first, second] = units[0]?.units ?? [];++ const put = call(doc, "putCaptionTranslation", {+ language: "fr",+ label: "Français",+ assetId: "asset_1",+ units: [{ id: first.id, text: "Bonjour." }],+ });+ expect(put.result).toMatchObject({+ written: 1,+ coverage: { translated: 1, total: 2 },+ shown: false,+ });+ doc = put.document ?? doc;++ // Only what is left to translate comes back.+ const left = call(doc, "getCaptionTranslationUnits", { language: "fr" }).result+ .assets as Array<{+ units: Array<{ id: string }>;+ }>;+ expect(left[0]?.units.map((u) => u.id)).toEqual([second.id]);++ doc = call(doc, "setCaptionSettings", { enabled: true, language: "fr" }).document ?? doc;+ expect(captionTexts(doc).join(" ")).toContain("Bonjour.");+ // The transcript itself is untouched.+ expect(doc.transcripts[0]?.words[0]?.text).toBe("Hello");++ const read = call(doc, "getCaptions").result.translations as Array<Record<string, unknown>>;+ expect(read).toEqual([+ expect.objectContaining({+ language: "fr",+ label: "Français",+ coverage: [{ assetId: "asset_1", translated: 1, total: 2 }],+ }),+ ]);+ });++ it("refuses unit ids it does not know, and nothing is written", () => {+ const put = call(fixtureDocument(), "putCaptionTranslation", {+ language: "fr",+ assetId: "asset_1",+ units: [{ id: "seg_w1", text: "Bonjour" }],+ });+ expect(put.ok).toBe(false);+ expect(put.document).toBeUndefined();+ const twice = call(fixtureDocument(), "putCaptionTranslation", {+ language: "fr",+ assetId: "asset_1",+ units: [+ { id: "u:seg_w1", text: "Bonjour." },+ { id: "u:seg_w1", text: "Salut." },+ ],+ });+ expect(twice.ok).toBe(false);+ });++ it("removing the language on screen puts the original back", () => {+ let doc = fixtureDocument();+ doc =+ call(doc, "putCaptionTranslation", {+ language: "fr",+ assetId: "asset_1",+ units: [{ id: "u:seg_w1", text: "Bonjour." }],+ }).document ?? doc;+ doc = call(doc, "setCaptionSettings", { language: "fr" }).document ?? doc;+ const removed = call(doc, "removeCaptionTranslation", { language: "fr" });+ expect(removed.result.shownLanguage).toBeNull();+ expect(call(removed.document ?? doc, "removeCaptionTranslation", { language: "fr" }).ok).toBe(+ false,+ );+ });+});++describe("setVocabulary", () => {+ it("stores the names Whisper is primed with, and getCurrentDocument reads them", () => {+ const set = call(fixtureDocument(), "setVocabulary", { vocabulary: " OpenScreen, Vulkan " });+ expect(set.document?.project.vocabulary).toBe("OpenScreen, Vulkan");+ const read = set.document && call(set.document, "getCurrentDocument").result;+ expect(read?.vocabulary).toBe("OpenScreen, Vulkan");+ expect(call(fixtureDocument(), "setVocabulary", { vocabulary: "x".repeat(601) }).ok).toBe(+ false,+ );+ });+});diff --git a/electron/ai-edition/caption-tools.ts b/electron/ai-edition/caption-tools.tsnew file mode 100644index 000000000..5053fa125--- /dev/null+++ b/electron/ai-edition/caption-tools.ts@@ -0,0 +1,378 @@+// The caption tools: how the captions look and where they sit, which language they show,+// and the translation layers behind that choice. Pure, like the rest of the executor+// (`agent-tools.ts` routes these names here): no IPC, no fs.+//+// The writes are the Captions pane's own (`useCaptions`): `patchCaptionSettings` at the+// document's output aspect, `putCaptionTranslation`, `removeCaptionTranslation`. The+// caller translates by itself: an MCP client and the in-app agent are both language+// models, so neither needs the app's own provider for it. The transcript is never touched.++import { z } from "zod";+import {+ CAPTION_FONT_SIZE_MAX,+ CAPTION_FONT_SIZE_MIN,+ CAPTION_INSET_X_MAX,+ CAPTION_INSET_Y_MAX,+ CAPTION_PLATE_OPACITY_MAX,+ CAPTION_PLATE_OPACITY_MIN,+ CAPTION_STYLES,+ type CaptionSettings,+ type CaptionSettingsPatch,+ type CaptionStyleId,+ captionPlateOf,+ captionPlatePatch,+ captionStyleOf,+ captionTranslationUnits,+ getCaptionSettings,+ getCaptionTranslations,+ patchCaptionSettings,+ putCaptionTranslation,+ removeCaptionTranslation,+ translationCoverage,+} from "../../src/lib/ai-edition/captions";+import { resolveCaptionLane } from "../../src/lib/ai-edition/captions/settings";+import {+ CAPTION_WORDS_PER_LINE_MAX,+ CAPTION_WORDS_PER_LINE_MIN,+} from "../../src/lib/ai-edition/captions/wordsPerLine";+import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat";+import { VOCABULARY_MAX_CHARS, withVocabulary } from "../../src/lib/ai-edition/document/transcript";+import type { AxcutDocument, AxcutTranscript } from "../../src/lib/ai-edition/schema";+import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings";+import { lanePlacements } from "../../src/lib/ai-edition/timeline/aggregated-transcript";+import { transcriptHasSpeech } from "../../src/lib/ai-edition/transcription/status";+import { TEXT_FONT_FAMILIES } from "../../src/lib/textFonts";+import type { AgentToolExecution } from "./agent-tools";++const STYLE_IDS = Object.keys(CAPTION_STYLES) as [CaptionStyleId, ...CaptionStyleId[]];+const hexColor = z.string().regex(/^#[0-9a-fA-F]{6}$/, "a #rrggbb colour");+const wordsPerLine = z+ .number()+ .int()+ .min(CAPTION_WORDS_PER_LINE_MIN)+ .max(CAPTION_WORDS_PER_LINE_MAX);+// A BCP 47 tag ("fr", "pt-BR"): it is the layer's storage key.+const languageCode = z+ .string()+ .regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "a language code such as fr or pt-BR");++export const getCaptionsArgs = z.object({});++export const setCaptionSettingsArgs = z.object({+ enabled: z.boolean().optional(),+ style: z.enum(STYLE_IDS).optional(),+ fontFamily: z.enum(TEXT_FONT_FAMILIES).optional(),+ fontWeight: z.enum(["normal", "bold"]).optional(),+ fontSize: z.number().min(CAPTION_FONT_SIZE_MIN).max(CAPTION_FONT_SIZE_MAX).optional(),+ color: hexColor.optional(),+ plate: z.enum(["none", "dark", "light"]).optional(),+ plateOpacity: z.number().min(CAPTION_PLATE_OPACITY_MIN).max(CAPTION_PLATE_OPACITY_MAX).optional(),+ anchorV: z.enum(["bottom", "top"]).optional(),+ anchorH: z.enum(["left", "center", "right"]).optional(),+ insetY: z.number().min(0).max(CAPTION_INSET_Y_MAX).optional(),+ insetX: z.number().min(0).max(CAPTION_INSET_X_MAX).optional(),+ minWordsPerLine: wordsPerLine.optional(),+ maxWordsPerLine: wordsPerLine.optional(),+ lane: z.enum(["recording", "voiceover"]).optional(),+ language: languageCode.nullable().optional(),+});++export const getCaptionTranslationUnitsArgs = z.object({+ language: languageCode,+ assetId: z.string().min(1).optional(),+ includeTranslated: z.boolean().optional(),+});++export const putCaptionTranslationArgs = z.object({+ language: languageCode,+ label: z.string().trim().min(1).max(40).optional(),+ assetId: z.string().min(1),+ units: z.array(z.object({ id: z.string().min(1), text: z.string().trim().min(1) })).min(1),+});++export const removeCaptionTranslationArgs = z.object({ language: languageCode });++export const setVocabularyArgs = z.object({ vocabulary: z.string().max(VOCABULARY_MAX_CHARS) });++export const CAPTION_TOOL_NAMES = [+ "getCaptions",+ "setCaptionSettings",+ "getCaptionTranslationUnits",+ "putCaptionTranslation",+ "removeCaptionTranslation",+ "setVocabulary",+] as const;++export const MUTATING_CAPTION_TOOLS = [+ "setCaptionSettings",+ "putCaptionTranslation",+ "removeCaptionTranslation",+ "setVocabulary",+] as const;++function failure(message: string): AgentToolExecution {+ return { ok: false, resultJson: JSON.stringify({ error: message }) };+}++/** The output aspect, as the Captions pane reads it: it sets the default insets. */+function aspectOf(document: AxcutDocument): number {+ return resolveAspectRatioValue(document, getEditorSettings(document).aspectRatio);+}++/** The transcripts the captions are read from: those of the media on the resolved lane. */+function captionedTranscripts(+ document: AxcutDocument,+ settings: CaptionSettings,+): AxcutTranscript[] {+ const placed = new Set(+ lanePlacements(+ resolveCaptionLane(document, settings),+ document.timeline.clips,+ document.audioTracks ?? [],+ ).map((placement) => placement.assetId),+ );+ return document.transcripts.filter((t) => placed.has(t.assetId));+}++/** Everything a caller needs to read the captions back, in the editor's own terms. */+function captionsReport(document: AxcutDocument): Record<string, unknown> {+ const settings = getCaptionSettings(document, aspectOf(document));+ const transcripts = captionedTranscripts(document, settings);+ const translations = getCaptionTranslations(document);+ const nothingToCaption = transcripts.every((t) => !transcriptHasSpeech(t));+ return {+ settings,+ style: captionStyleOf(settings) ?? "custom",+ plate: captionPlateOf(settings),+ resolvedLane: resolveCaptionLane(document, settings),+ captionedAssets: transcripts.map((t) => ({+ assetId: t.assetId,+ language: t.language,+ words: t.words.length,+ })),+ ...(settings.enabled && nothingToCaption+ ? {+ warning:+ "Captions are on, but no transcript with speech is on their lane, so nothing shows: the media needs a transcript first (assets[].hasTranscript in getCurrentDocument).",+ }+ : {}),+ translations: Object.values(translations).map((layer) => ({+ language: layer.language,+ label: layer.label,+ updatedAt: layer.updatedAt,+ coverage: transcripts.map((t) => ({+ assetId: t.assetId,+ ...translationCoverage(t, translations, layer.language),+ })),+ })),+ };+}++function hasAnyField(data: Record<string, unknown>): boolean {+ return Object.values(data).some((value) => value !== undefined);+}++export function executeCaptionTool(+ document: AxcutDocument,+ name: (typeof CAPTION_TOOL_NAMES)[number],+ args: unknown,+): AgentToolExecution {+ switch (name) {+ case "getCaptions":+ return {+ ok: true,+ resultJson: JSON.stringify({+ ...captionsReport(document),+ choices: {+ styles: STYLE_IDS,+ fonts: TEXT_FONT_FAMILIES,+ plates: ["none", "dark", "light"],+ },+ }),+ };++ case "setCaptionSettings": {+ const parsed = setCaptionSettingsArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ const { style, plate, plateOpacity, lane, ...fields } = parsed.data;+ if (!hasAnyField(parsed.data)) return failure("Pass at least one setting to change.");+ const aspect = aspectOf(document);+ const current = getCaptionSettings(document, aspect);+ if (fields.language) {+ const layers = Object.keys(getCaptionTranslations(document));+ if (!layers.includes(fields.language)) {+ return failure(+ `No "${fields.language}" translation to show. ${layers.length ? `This project has: ${layers.join(", ")}.` : "This project has none."} Write one with putCaptionTranslation first, or pass null for the transcript's own language.`,+ );+ }+ }+ // The pane's order: a style sets the look as a whole, a plate then recolours the+ // text that would vanish on it, and an explicit field wins over both.+ const patch: CaptionSettingsPatch = { ...(style ? CAPTION_STYLES[style] : {}) };+ if (plate) Object.assign(patch, captionPlatePatch({ ...current, ...patch }, plate));+ for (const [key, value] of Object.entries(fields)) {+ if (value !== undefined) Object.assign(patch, { [key]: value });+ }+ if (plateOpacity !== undefined) patch.backgroundOpacity = plateOpacity;+ if (lane !== undefined) patch.captionLane = lane;+ const min = patch.minWordsPerLine ?? current.minWordsPerLine;+ const max = patch.maxWordsPerLine ?? current.maxWordsPerLine;+ if (min > max) {+ return failure(+ `minWordsPerLine (${min}) cannot exceed maxWordsPerLine (${max}). Nothing was modified.`,+ );+ }+ const next = patchCaptionSettings(document, patch, aspect);+ const report = captionsReport(next);+ return {+ ok: true,+ document: next,+ resultJson: JSON.stringify({+ settings: report.settings,+ style: report.style,+ plate: report.plate,+ resolvedLane: report.resolvedLane,+ ...(lane && lane !== report.resolvedLane+ ? {+ laneNote: `The captions read the ${report.resolvedLane} lane: the project has no voiceover track to caption.`,+ }+ : {}),+ ...(report.warning ? { warning: report.warning } : {}),+ }),+ summary:+ fields.enabled === true+ ? "turned the captions on"+ : fields.enabled === false+ ? "turned the captions off"+ : "restyled the captions",+ };+ }++ case "getCaptionTranslationUnits": {+ const parsed = getCaptionTranslationUnitsArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ const { language, assetId, includeTranslated } = parsed.data;+ const settings = getCaptionSettings(document, aspectOf(document));+ const transcripts = assetId+ ? document.transcripts.filter((t) => t.assetId === assetId)+ : captionedTranscripts(document, settings);+ if (transcripts.length === 0) {+ return failure(+ assetId+ ? `Asset ${assetId} has no transcript to translate yet.`+ : "No transcript is on the captions' lane to translate yet.",+ );+ }+ const translations = getCaptionTranslations(document);+ return {+ ok: true,+ resultJson: JSON.stringify({+ language,+ assets: transcripts.map((t) => {+ const done = translations[language]?.byAsset[t.assetId] ?? {};+ const units = captionTranslationUnits(t)+ .filter((unit) => includeTranslated || !done[unit.id]?.trim())+ .map((unit) => ({+ id: unit.id,+ startSec: Math.round(unit.startSec * 1000) / 1000,+ endSec: Math.round(unit.endSec * 1000) / 1000,+ text: unit.text,+ ...(done[unit.id] ? { translation: done[unit.id] } : {}),+ }));+ return {+ assetId: t.assetId,+ sourceLanguage: t.language,+ coverage: translationCoverage(t, translations, language),+ units,+ };+ }),+ }),+ };+ }++ case "putCaptionTranslation": {+ const parsed = putCaptionTranslationArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ const { language, label, assetId, units } = parsed.data;+ const transcript = document.transcripts.find((t) => t.assetId === assetId);+ if (!transcript) return failure(`Asset ${assetId} has no transcript to translate.`);+ const ids = units.map((unit) => unit.id);+ const known = new Set(captionTranslationUnits(transcript).map((unit) => unit.id));+ const unknown = ids.filter((id) => !known.has(id));+ if (unknown.length > 0) {+ return failure(+ `Unknown unit id(s) for asset ${assetId}: ${unknown.slice(0, 5).join(", ")}${unknown.length > 5 ? ` and ${unknown.length - 5} more` : ""}. Take the ids from getCaptionTranslationUnits. Nothing was modified.`,+ );+ }+ if (new Set(ids).size !== ids.length) {+ return failure("A unit id is given twice. Nothing was modified.");+ }+ const next = putCaptionTranslation(document, {+ language,+ label: label ?? getCaptionTranslations(document)[language]?.label ?? language,+ assetId,+ segments: Object.fromEntries(units.map((unit) => [unit.id, unit.text])),+ });+ const shown = getCaptionSettings(next, aspectOf(next)).language === language;+ return {+ ok: true,+ document: next,+ resultJson: JSON.stringify({+ language,+ assetId,+ written: ids.length,+ coverage: translationCoverage(transcript, getCaptionTranslations(next), language),+ shown,+ ...(shown+ ? {}+ : {+ next: "The captions still show another language: setCaptionSettings with this language shows it.",+ }),+ }),+ summary: `translated ${ids.length} caption unit${ids.length === 1 ? "" : "s"} into ${language}`,+ };+ }++ case "removeCaptionTranslation": {+ const parsed = removeCaptionTranslationArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ const { language } = parsed.data;+ const layers = Object.keys(getCaptionTranslations(document));+ if (!layers.includes(language)) {+ return failure(+ `No "${language}" translation to remove. ${layers.length ? `This project has: ${layers.join(", ")}.` : "This project has none."}`,+ );+ }+ // As the pane deletes one: captions showing it fall back to the transcript's language.+ const aspect = aspectOf(document);+ const cleared = removeCaptionTranslation(document, language);+ const next =+ getCaptionSettings(cleared, aspect).language === language+ ? patchCaptionSettings(cleared, { language: null }, aspect)+ : cleared;+ return {+ ok: true,+ document: next,+ resultJson: JSON.stringify({+ removed: language,+ shownLanguage: getCaptionSettings(next, aspect).language,+ }),+ summary: `removed the ${language} caption translation`,+ };+ }++ case "setVocabulary": {+ const parsed = setVocabularyArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ const next = withVocabulary(document, parsed.data.vocabulary);+ const vocabulary = (next ?? document).project.vocabulary ?? "";+ const resultJson = JSON.stringify({+ vocabulary,+ note: "Whisper reads it on the next transcription. An existing transcript is unchanged: transcribe with a language redoes it.",+ });+ return next+ ? { ok: true, document: next, resultJson, summary: "set the project vocabulary" }+ : { ok: true, resultJson };+ }+ }+}diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.tsindex 30d9734a7..8fa62ea51 100644--- a/electron/ai-edition/deep-agent/service.test.ts+++ b/electron/ai-edition/deep-agent/service.test.ts@@ -92,6 +92,17 @@ const ARGS: Record<string, unknown> = {removeClip: { clipId: "clip_1" },setAspectRatio: { aspectRatio: "16:9" },setEditorSettings: { padding: 30 },+ getCaptions: {},+ setCaptionSettings: { enabled: true, style: "bold" },+ getCaptionTranslationUnits: { language: "fr" },+ putCaptionTranslation: {+ language: "fr",+ assetId: "asset_1",+ units: [{ id: "u:seg_1", text: "Bonjour" }],+ },+ // The fixture has no translation layer, so this one is refused.+ removeCaptionTranslation: { language: "fr" },+ setVocabulary: { vocabulary: "OpenScreen, Whisper" },};/** ponytail: the fixture starts from a FIXED instant. `createEmptyDocument` reads@@ -378,6 +389,8 @@ describe("one description of the tools, not two", () => {"getTranscript","getTranscriptWords","getCursorTrack",+ "getCaptions",+ "getCaptionTranslationUnits",]);});});diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.tsindex e003a7460..44571c2dd 100644--- a/electron/ai-edition/deep-agent/service.ts+++ b/electron/ai-edition/deep-agent/service.ts@@ -20,7 +20,20 @@import { anthropicPromptCachingMiddleware, createAgent, tool } from "langchain";import { z } from "zod";import type { CursorTelemetryPoint } from "../../../src/components/video-editor/types";+import {+ CAPTION_FONT_SIZE_MAX,+ CAPTION_FONT_SIZE_MIN,+ CAPTION_INSET_X_MAX,+ CAPTION_INSET_Y_MAX,+ CAPTION_PLATE_OPACITY_MAX,+ CAPTION_PLATE_OPACITY_MIN,+} from "../../../src/lib/ai-edition/captions";+import {+ CAPTION_WORDS_PER_LINE_MAX,+ CAPTION_WORDS_PER_LINE_MIN,+} from "../../../src/lib/ai-edition/captions/wordsPerLine";import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks";+import { VOCABULARY_MAX_CHARS } from "../../../src/lib/ai-edition/document/transcript";import type { AxcutDocument } from "../../../src/lib/ai-edition/schema";import { collectAutoZoomSuggestionsForDocument } from "../../../src/lib/ai-edition/timeline/apply-auto-zooms";// ponytail: the legend is DERIVED from the depth→scale table, not retyped. Both@@ -72,6 +85,14 @@ import {setWordTextArgs,setZoomArgs,} from "../agent-tools";+import {+ getCaptionsArgs,+ getCaptionTranslationUnitsArgs,+ putCaptionTranslationArgs,+ removeCaptionTranslationArgs,+ setCaptionSettingsArgs,+ setVocabularyArgs,+} from "../caption-tools";import { setEditorSettingsArgs } from "../editor-settings-tool";import {createOpenScreenChatModel,@@ -113,7 +134,7 @@ const consentPromptBlock = (setting: string) =>"","PROJECT EDITS ARE CURRENTLY DISABLED by the user, who asked to be consulted before the timeline changes.","- Read freely: getCurrentDocument and getTranscript work as usual.",- "- Do NOT call any tool that writes (removeWords, addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.",+ "- Do NOT call any tool that writes (removeWords, addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects and settings, putCaptionTranslation, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.","- Instead: say precisely what you would change — which tool, which times, which ids — and ask the user to confirm. Be specific enough that they can say yes to it.",`- Never state or imply that an edit was applied. If the user confirms and you are still refused, tell them to turn on ${setting} first.`,].join("\n");@@ -145,6 +166,7 @@ const BASE_SYSTEM_PROMPT = ["- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.","- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.","- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.",+ "- Captions are the transcript drawn on the video, never typed in: setCaptionSettings turns them on, styles and places them, getCaptions reads them back. To caption in another language, read the phrases with getCaptionTranslationUnits, translate them yourself, store them with putCaptionTranslation, then show that language with setCaptionSettings. A translation never rewrites the transcript. For text the speaker did not say, use addAnnotation.","- Deleting is a first-class action, not a workaround: removeTrim, removeModifier, removeClip. Never fake a deletion by re-adding an element or zeroing it out (span 0, speed 1×) — that leaves it in the document and misreports what you did.","If nothing in the list does what was asked, say so; do not approximate it with a bigger tool.","",@@ -244,6 +266,16 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {"Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.",removeClip:"Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.",+ getCaptions:+ "Read the captions: whether they show (enabled), how they look (font, weight, size, colour, plate and its opacity), where they sit (anchorV/insetY, anchorH/insetX), how many words show at once, the lane they read (resolvedLane: the recording's speech, or the voiceover tracks'), and the language they show (null: the transcript's own words). captionedAssets lists the transcripts they are drawn from, and `warning` says when they are on with nothing to show. translations lists each language layer with its coverage per asset (translated of total phrase units). choices lists what setCaptionSettings accepts for style, fontFamily and plate.",+ setCaptionSettings: `Turn the captions on or off, and set how they look and where they sit, as the editor's Captions pane does. Only the fields you pass change. \`style\` applies a named look first (font, weight, size, colour and plate together); \`plate\` then picks what is behind the text (none, dark, light) and recolours text that would vanish on it; any other field you pass wins over both. fontSize is px at a 1080-high frame (${CAPTION_FONT_SIZE_MIN}–${CAPTION_FONT_SIZE_MAX}), color a #rrggbb hex, plateOpacity ${CAPTION_PLATE_OPACITY_MIN}–${CAPTION_PLATE_OPACITY_MAX}. anchorV pins the caption block to the bottom or top edge, and insetY is its distance from that edge in % of the frame height (0–${CAPTION_INSET_Y_MAX}); anchorH pins it left, centre or right, and insetX (0–${CAPTION_INSET_X_MAX} % of the frame width) is ignored when it is centred. minWordsPerLine and maxWordsPerLine (${CAPTION_WORDS_PER_LINE_MIN}–${CAPTION_WORDS_PER_LINE_MAX}) bound how many words show at once. lane picks whose speech is captioned: the recording's, or the voiceover tracks' (it falls back to the recording when there is none, and resolvedLane says so). language shows a translation layer written with putCaptionTranslation; null shows the transcript's own words. Captions are drawn from the transcript, so they need one; enabled captions are burnt into the export.`,+ getCaptionTranslationUnits:+ "Read what to translate for one caption language: per asset the captions read (or the one assetId names), the transcript grouped into phrase units, each with its id, its span in the asset's source seconds and its text. A unit is a sentence or a run of speech between pauses, never a single word, so translate each one as a whole. By default only the units with no text yet in that language, so a translation sent in batches resumes where it stopped; includeTranslated returns the done ones too, with their translation. Translate them yourself, then store them with putCaptionTranslation.",+ putCaptionTranslation:+ "Store your translation of caption units for one asset in a language layer: `units` lists each unit as { id, text }: its id from getCaptionTranslationUnits and your translation of it. It merges into the layer, which the first call creates, so send long transcripts in batches. language is the layer's code (fr, pt-BR); label is the name the editor's language picker shows, such as Français (default: the code). Unknown unit ids refuse the whole call. The transcript is never changed: this is a separate layer the captions can show. It does not switch the captions to it: setCaptionSettings with this language does.",+ removeCaptionTranslation:+ "Delete one caption language layer. The transcript is untouched, and captions that showed this language go back to the transcript's own.",+ setVocabulary: `Set the project's vocabulary: the names and technical terms Whisper is primed with on every transcription of this project, so it spells them right (at most ${VOCABULARY_MAX_CHARS} characters, read as a plain list such as "OpenScreen, Vulkan, whisper.cpp"). An empty string clears it. It changes no existing transcript: transcribe the asset again for that. getCurrentDocument reports it as vocabulary.`,};// ponytail: mutable document holder so a write-tool that updates the snapshot@@ -479,6 +511,12 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray<readonly [string, z.ZodObject]> = [["removeClip", removeClipArgs],["setAspectRatio", setAspectRatioArgs],["setEditorSettings", setEditorSettingsArgs],+ ["getCaptions", getCaptionsArgs],+ ["setCaptionSettings", setCaptionSettingsArgs],+ ["getCaptionTranslationUnits", getCaptionTranslationUnitsArgs],+ ["putCaptionTranslation", putCaptionTranslationArgs],+ ["removeCaptionTranslation", removeCaptionTranslationArgs],+ ["setVocabulary", setVocabularyArgs],];/**diff --git a/src/components/ai-edition/CaptionsPane.tsx b/src/components/ai-edition/CaptionsPane.tsxindex ba8710b7d..d34aed06b 100644--- a/src/components/ai-edition/CaptionsPane.tsx+++ b/src/components/ai-edition/CaptionsPane.tsx@@ -20,6 +20,8 @@ import type {CaptionStyleId,} from "@/lib/ai-edition/captions";import {+ CAPTION_FONT_SIZE_MAX,+ CAPTION_FONT_SIZE_MIN,CAPTION_INSET_X_MAX,CAPTION_INSET_Y_MAX,CAPTION_PLATE_OPACITY_MAX,@@ -526,8 +528,8 @@ export function CaptionsPane({ onClose }: { onClose?: () => void } = {}) {<SliderCelllabel={t("captions.fontSize")}value={settings.fontSize}- min={16}- max={140}+ min={CAPTION_FONT_SIZE_MIN}+ max={CAPTION_FONT_SIZE_MAX}defaultValue={DEFAULT_CAPTION_SETTINGS.fontSize}suffix="px"disabled={disabled}diff --git a/src/lib/ai-edition/captions/index.ts b/src/lib/ai-edition/captions/index.tsindex 2db6a320c..9a3fc8ee9 100644--- a/src/lib/ai-edition/captions/index.ts+++ b/src/lib/ai-edition/captions/index.ts@@ -17,6 +17,8 @@ export type {CaptionStyleId,} from "./settings";export {+ CAPTION_FONT_SIZE_MAX,+ CAPTION_FONT_SIZE_MIN,CAPTION_INSET_X_MAX,CAPTION_INSET_Y_MAX,CAPTION_PLATE_OPACITY_MAX,diff --git a/src/lib/ai-edition/captions/settings.ts b/src/lib/ai-edition/captions/settings.tsindex 3ab6fcb91..d3f5731b1 100644--- a/src/lib/ai-edition/captions/settings.ts+++ b/src/lib/ai-edition/captions/settings.ts@@ -126,6 +126,10 @@ export const DEFAULT_CAPTION_SETTINGS: CaptionSettings = {maxWordsPerLine: 7,};+/** The font sizes the Captions pane offers, in px at a 1080-high frame. */+export const CAPTION_FONT_SIZE_MIN = 16;+export const CAPTION_FONT_SIZE_MAX = 140;+/*** Plate opacity bounds. The on/off state belongs to the plate choice alone: an opacity* that reached 0 was a second way to switch the plate off, and one past 90% hid the plate's