Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(mcp): insertClip, duplicateClip, addAutoZooms and per-clip crop
EtienneLescotb20d7cbc7b81
diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.tsindex f9811fa81..af629ae70 100644--- a/electron/ai-edition/agent-tools.test.ts+++ b/electron/ai-edition/agent-tools.test.ts@@ -8,7 +8,12 @@ import {} from "../../src/lib/ai-edition/schema";import { ZOOM_DEPTH_SCALES } from "../../src/lib/ai-edition/timeline/zoom-scale";import { zoomScaleLimit } from "../../src/native/sceneDescription";-import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES } from "./agent-tools";+import {+ type AgentToolOptions,+ executeAgentTool,+ isMutatingTool,+ MUTATING_TOOL_NAMES,+} from "./agent-tools";import { measureTextInShippedFonts } from "./text-measure";/** ponytail: every fixture in this file starts from the SAME instant.@@ -192,6 +197,9 @@ describe("the mutating-tool table", () => {"addTrims","addZoom","addZooms",+ "addAutoZooms",+ "duplicateClip",+ "insertClip","moveClip","removeClip","removeModifier",@@ -2491,7 +2499,7 @@ describe("addAudio / setAudio", () => {it("refuses a video asset, pointing at the tool that does place footage", () => {const result = place(withAudioAsset(), { assetId: "asset_1", startSec: 0, endSec: 4 });expect(result.ok).toBe(false);- expect(result.resultJson).toContain("replaceTimeline");+ expect(result.resultJson).toContain("insertClip");});it("setAudio re-levels and re-lanes the track it names", () => {@@ -3475,3 +3483,216 @@ describe("region tools reach the inspector's settings (#1180)", () => {).toBe(false);});});++describe("clip structure (#1180)", () => {+ /** Two recordings, one not probed yet, and a music file; one clip of the first, cut once. */+ function twoTakes(): AxcutDocument {+ const base = createEmptyDocument({+ title: "Takes",+ projectId: "proj_takes",+ createdAt: FIXTURE_CREATED_AT,+ });+ return documentSchema.parse({+ ...base,+ project: { ...base.project, primaryAssetId: "asset_1" },+ assets: [+ {+ id: "asset_1",+ kind: "video",+ label: "Take 1",+ originalPath: "C:/v/1.mp4",+ durationSec: 20,+ },+ {+ id: "asset_2",+ kind: "video",+ label: "Take 2",+ originalPath: "C:/v/2.mp4",+ durationSec: 8,+ },+ { id: "asset_3", kind: "video", label: "Unprobed", originalPath: "C:/v/3.mp4" },+ {+ id: "audio_1",+ kind: "audio",+ label: "Music",+ originalPath: "C:/v/m.mp3",+ durationSec: 30,+ },+ ],+ timeline: {+ ...base.timeline,+ clips: [+ {+ id: "clip_1",+ assetId: "asset_1",+ sourceStartSec: 0,+ sourceEndSec: 10,+ timelineStartSec: 0,+ timelineEndSec: 10,+ wordRefs: [],+ origin: "user",+ reason: "",+ },+ ],+ trimRanges: [+ {+ id: "trim_1",+ assetId: "asset_1",+ clipId: "clip_1",+ startSec: 2,+ endSec: 3,+ reason: "",+ origin: "user",+ },+ ],+ },+ });+ }++ const run = (document: AxcutDocument, name: string, args: unknown) =>+ executeAgentTool(document, name, JSON.stringify(args));++ it("insertClip places another asset's footage before the clip it names", () => {+ const result = run(twoTakes(), "insertClip", { assetId: "asset_2", beforeClipId: "clip_1" });+ expect(result.ok).toBe(true);+ const clips = (result.document as AxcutDocument).timeline.clips;+ const { clipId } = JSON.parse(result.resultJson);+ expect(clips.map((c) => c.id)).toEqual([clipId, "clip_1"]);+ expect(clips[0]).toMatchObject({ assetId: "asset_2", sourceStartSec: 0, sourceEndSec: 8 });+ // Back to back: the clip after it moved along the ruler.+ expect(clips[1]).toMatchObject({ timelineStartSec: 8, timelineEndSec: 18 });+ });++ it("insertClip folds footage that continues the clip before it into that clip", () => {+ const result = run(twoTakes(), "insertClip", {+ assetId: "asset_1",+ sourceStartSec: 10,+ sourceEndSec: 15,+ });+ expect(JSON.parse(result.resultJson)).toMatchObject({+ clipId: null,+ joinedIntoNeighbour: true,+ });+ expect((result.document as AxcutDocument).timeline.clips).toMatchObject([+ { id: "clip_1", sourceStartSec: 0, sourceEndSec: 15 },+ ]);+ });++ it("insertClip refuses what is not footage, and says what to use", () => {+ expect(run(twoTakes(), "insertClip", { assetId: "audio_1" }).resultJson).toContain("addAudio");+ const unknown = run(twoTakes(), "insertClip", { assetId: "nope" });+ expect(unknown.ok).toBe(false);+ expect(unknown.resultJson).toContain("asset_2 (Take 2)");+ const nowhere = run(twoTakes(), "insertClip", { assetId: "asset_2", beforeClipId: "clip_x" });+ expect(nowhere.ok).toBe(false);+ });++ it("insertClip asks for an end when the asset's length is unknown, and clamps to a known one", () => {+ const unprobed = run(twoTakes(), "insertClip", { assetId: "asset_3" });+ expect(unprobed.ok).toBe(false);+ expect(unprobed.resultJson).toContain("sourceEndSec");+ expect(run(twoTakes(), "insertClip", { assetId: "asset_3", sourceEndSec: 4 }).ok).toBe(true);+ const clamped = run(twoTakes(), "insertClip", { assetId: "asset_2", sourceEndSec: 99 });+ expect(JSON.parse(clamped.resultJson).sourceEndSec).toBe(8);+ expect(run(twoTakes(), "insertClip", { assetId: "asset_2", sourceStartSec: 9 }).ok).toBe(false);+ });++ it("duplicateClip puts the copy right after the original, with its own copy of the cuts", () => {+ const result = run(twoTakes(), "duplicateClip", { clipId: "clip_1" });+ const payload = JSON.parse(result.resultJson);+ const document = result.document as AxcutDocument;+ expect(document.timeline.clips.map((c) => c.id)).toEqual(["clip_1", payload.clipId]);+ expect(payload.copiedTrimIds).toHaveLength(1);+ expect(document.timeline.trimRanges.map((t) => t.clipId)).toEqual(["clip_1", payload.clipId]);+ expect(run(twoTakes(), "duplicateClip", { clipId: "clip_x" }).ok).toBe(false);+ });++ it("setClipRange crops a clip without touching its range, and the snapshot reads it back", () => {+ const crop = { x: 0.1, y: 0.2, width: 0.5, height: 0.6 };+ const result = run(twoTakes(), "setClipRange", { clipId: "clip_1", cropRegion: crop });+ const document = result.document as AxcutDocument;+ expect(document.timeline.clips[0]).toMatchObject({+ sourceStartSec: 0,+ sourceEndSec: 10,+ cropRegion: crop,+ });+ const snapshot = JSON.parse(run(document, "getCurrentDocument", {}).resultJson);+ expect(snapshot.clips[0].cropRegion).toEqual(crop);+ // Absent, not null, on a clip that shows the whole frame.+ const plain = JSON.parse(run(twoTakes(), "getCurrentDocument", {}).resultJson);+ expect(plain.clips[0]).not.toHaveProperty("cropRegion");+ });++ it("setClipRange clears a crop with null or with the whole frame", () => {+ const cropped = run(twoTakes(), "setClipRange", {+ clipId: "clip_1",+ cropRegion: { x: 0, y: 0, width: 0.5, height: 0.5 },+ }).document as AxcutDocument;+ for (const cropRegion of [null, { x: 0, y: 0, width: 1, height: 1 }]) {+ const cleared = run(cropped, "setClipRange", { clipId: "clip_1", cropRegion });+ expect((cleared.document as AxcutDocument).timeline.clips[0].cropRegion).toBeUndefined();+ }+ });++ it("setClipRange refuses a crop outside the frame, and a call that changes nothing", () => {+ const outside = run(twoTakes(), "setClipRange", {+ clipId: "clip_1",+ cropRegion: { x: 0.6, y: 0, width: 0.6, height: 1 },+ });+ expect(outside.ok).toBe(false);+ expect(outside.resultJson).toContain("inside the frame");+ expect(run(twoTakes(), "setClipRange", { clipId: "clip_1" }).ok).toBe(false);+ });++ it("setClipRange moves one edge and keeps the other", () => {+ const result = run(twoTakes(), "setClipRange", { clipId: "clip_1", sourceEndSec: 6 });+ expect((result.document as AxcutDocument).timeline.clips[0]).toMatchObject({+ sourceStartSec: 0,+ sourceEndSec: 6,+ });+ });++ describe("addAutoZooms", () => {+ const suggestion = {+ span: { start: 3500, end: 5500 },+ focus: { cx: 0.4, cy: 0.6 },+ depth: 3 as const,+ };+ const withPlan = (autoZooms: AgentToolOptions["autoZooms"]) =>+ executeAgentTool(twoTakes(), "addAutoZooms", "{}", { autoZooms });++ it("adds the planned zooms, as the wand does", () => {+ const result = withPlan({ suggestions: [suggestion], telemetry: { asset_1: "ok" } });+ expect(result.ok).toBe(true);+ expect(JSON.parse(result.resultJson)).toMatchObject({+ added: 1,+ zooms: [{ startSec: 3.5, endSec: 5.5, focus: { cx: 0.4, cy: 0.6 } }],+ telemetry: { asset_1: "ok" },+ });+ expect((result.document as AxcutDocument).zoomRanges).toMatchObject([+ { focusMode: "auto", depth: 3, clipId: "clip_1" },+ ]);+ });++ it("answers added 0 without writing when no click calls for a zoom", () => {+ const result = withPlan({ suggestions: [], telemetry: { asset_1: "no-sidecar" } });+ expect(result.ok).toBe(true);+ expect(result.document).toBeUndefined();+ expect(JSON.parse(result.resultJson)).toMatchObject({+ added: 0,+ telemetry: { asset_1: "no-sidecar" },+ });+ });++ it("refuses, as a limit of its own, when the clicks could not be read", () => {+ for (const plan of [+ undefined,+ { suggestions: [], telemetry: { asset_1: "unavailable" as const } },+ ]) {+ const result = withPlan(plan);+ expect(result.ok).toBe(false);+ expect(result.resultJson).toContain("could not be read");+ }+ });+ });+});diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.tsindex e6bf21585..a2420cb9f 100644--- a/electron/ai-edition/agent-tools.ts+++ b/electron/ai-edition/agent-tools.ts@@ -32,12 +32,15 @@ import {import { createId } from "../../src/lib/ai-edition/document/ids";import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat";import {+ duplicateClip,+ insertAssetClip,moveClip,planTimelineReplacement,type RegionKind,removeClip,removeRegion,replaceTimeline,+ setClipCrop,setClipSourceRange,} from "../../src/lib/ai-edition/document/timeline";import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript";@@ -45,9 +48,11 @@ import {type AxcutAnnotationRegion,type AxcutDocument,blurDataSchema,+ clipCropRegionSchema,figureDataSchema,} from "../../src/lib/ai-edition/schema";import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings";+import { appendAutoZoomSuggestions } from "../../src/lib/ai-edition/timeline/apply-auto-zooms";import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera";import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts";import {@@ -72,6 +77,7 @@ import {ZOOM_DEPTH_LEGEND,zoomLevelFor,} from "../../src/lib/ai-edition/timeline/zoom-scale";+import type { AutoZoomSuggestion } from "../../src/lib/ai-edition/timeline/zoom-suggestions";import {chainedPans,transitionCutsMs,@@ -664,12 +670,38 @@ export const setTrimArgs = z.object({endSec: secondsSchema,});+/** A crop the frame can hold: a non-empty rectangle inside it, in 0–1 fractions. */+const cropRegionArgs = clipCropRegionSchema.refine(+ (c) => c.width > 0 && c.height > 0 && c.x + c.width <= 1 + 1e-6 && c.y + c.height <= 1 + 1e-6,+ "cropRegion must be a non-empty rectangle inside the frame: width and height above 0, x + width and y + height at most 1.",+);+export const setClipRangeArgs = z.object({clipId: z.string().min(1),- sourceStartSec: secondsSchema,- sourceEndSec: secondsSchema,+ sourceStartSec: secondsSchema.optional(),+ sourceEndSec: secondsSchema.optional(),+ cropRegion: cropRegionArgs+ .nullable()+ .optional()+ .describe(+ "The part of the source frame this clip shows, as 0–1 fractions of it ({x, y, width, height}); null shows the whole frame again.",+ ),+});++/** `beforeClipId`, never an index, for the reason `moveClipArgs` gives. */+export const insertClipArgs = z.object({+ assetId: z.string().min(1),+ beforeClipId: z.string().min(1).nullish(),+ sourceStartSec: secondsSchema.default(0),+ sourceEndSec: secondsSchema.optional(),});+export const duplicateClipArgs = z.object({+ clipId: z.string().min(1),+});++export const addAutoZoomsArgs = z.object({});+export const setAspectRatioArgs = z.object({aspectRatio: z.string()@@ -1016,9 +1048,12 @@ export const OPENSCREEN_TOOL_NAMES = ["setTrim","setClipRange","moveClip",+ "insertClip",+ "duplicateClip","replaceTimeline","addZoom","addZooms",+ "addAutoZooms","setZoom","addSpeed","setSpeed",@@ -1091,8 +1126,11 @@ export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set(["setTrim","setClipRange","moveClip",+ "insertClip",+ "duplicateClip","replaceTimeline","addZoom",+ "addAutoZooms","setZoom","addSpeed","setSpeed",@@ -1301,6 +1339,8 @@ export function documentSnapshotForModel(sourceEndSec: c.sourceEndSec ?? null,timelineStartSec: c.timelineStartSec,timelineEndSec: c.timelineEndSec,+ // Only when set: absent is the whole frame.+ ...(c.cropRegion ? { cropRegion: c.cropRegion } : {}),})),trimRanges: document.timeline.trimRanges.map((s) => ({id: s.id,@@ -1522,6 +1562,16 @@ export interface AgentToolOptions {/** Whether this machine can cut out the camera background, asked only by the two tools that* report it. Absent where nothing can ask the compositor: reported as not checked. */cameraSegmentation?: () => SegmentationSupport;+ /** `addAutoZooms`' plan, made by the async wrapper from the recorded clicks. Absent means+ * no cursor reader is wired here, which the tool reports as "could not look". */+ autoZooms?: AutoZoomPlan;+}++/** What the timeline wand would add, and what each recording's cursor read gave. */+export interface AutoZoomPlan {+ suggestions: AutoZoomSuggestion[];+ /** assetId → the read's verdict, for every video asset with a clip on the timeline. */+ telemetry: Record<string, CursorTelemetryLoad["status"]>;}/**@@ -2284,31 +2334,211 @@ export function executeAgentTool(case "setClipRange": {const parsed = setClipRangeArgs.safeParse(args);if (!parsed.success) return failure(parsed.error.message);- const { clipId } = parsed.data;- if (!document.timeline.clips.some((c) => c.id === clipId)) {- return failure(`Unknown clip: ${clipId}`);+ const { clipId, cropRegion } = parsed.data;+ const clip = document.timeline.clips.find((c) => c.id === clipId);+ if (!clip) return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`);+ const ranged =+ parsed.data.sourceStartSec !== undefined || parsed.data.sourceEndSec !== undefined;+ if (!ranged && cropRegion === undefined) {+ return failure(+ "Pass sourceStartSec / sourceEndSec, cropRegion, or both: nothing to change.",+ );+ }+ const start = parsed.data.sourceStartSec ?? clip.sourceStartSec;+ const end =+ parsed.data.sourceEndSec ??+ clip.sourceEndSec ??+ document.assets.find((a) => a.id === clip.assetId)?.durationSec;+ if (end === undefined) {+ return failure(`Clip ${clipId} has no known end yet: pass sourceEndSec too.`);}- const sourceStartSec = Math.min(parsed.data.sourceStartSec, parsed.data.sourceEndSec);- const sourceEndSec = Math.max(parsed.data.sourceStartSec, parsed.data.sourceEndSec);- // One shared mutator with the modale + op dispatcher: recomputes the clip's- // width from the new source window AND clamps/drops the anchored pills the- // trim removed. Hand-rolling it here is exactly what left this façade orphaning- // stale pills the other two didn't.- const next = setClipSourceRange(document, clipId, sourceStartSec, sourceEndSec);+ const sourceStartSec = Math.min(start, end);+ const sourceEndSec = Math.max(start, end);+ // The clip's Edit modal, as one document: the range through the one shared mutator+ // (which recomputes the clip's width and clamps / drops the anchored pills the new+ // window no longer covers), then the crop. `null` clears it, and so does the whole+ // frame: an untouched clip does not store the identity region.+ const wholeFrame =+ cropRegion?.x === 0 &&+ cropRegion.y === 0 &&+ cropRegion.width === 1 &&+ cropRegion.height === 1;+ const afterRange = ranged+ ? setClipSourceRange(document, clipId, sourceStartSec, sourceEndSec)+ : document;+ const next =+ cropRegion === undefined+ ? afterRange+ : setClipCrop(afterRange, clipId, wholeFrame ? null : cropRegion);const dropped = droppedByEdit(document, next);const casualties = dropped.droppedModifierIds.length + dropped.droppedTrimIds.length;+ const after = next.timeline.clips.find((c) => c.id === clipId);return {ok: true,document: next,- resultJson: JSON.stringify({ clipId, sourceStartSec, sourceEndSec, ...dropped }),+ resultJson: JSON.stringify({+ clipId,+ sourceStartSec,+ sourceEndSec,+ cropRegion: after?.cropRegion ?? null,+ // Its media now continues the clip before it, with the same framing: one clip.+ ...(after+ ? {}+ : { joinedIntoNeighbour: true, clipOrder: next.timeline.clips.map((c) => c.id) }),+ ...dropped,+ }),summary:- `trimmed clip to ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)}` ++ (ranged+ ? `trimmed clip to ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)}`+ : `cropped clip ${clipId}`) ++ (ranged && cropRegion !== undefined ? ", cropped" : "") +(casualties > 0? ` — dropped ${[...dropped.droppedModifierIds, ...dropped.droppedTrimIds].join(", ")}`: ""),};}+ case "insertClip": {+ const parsed = insertClipArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ const { assetId } = parsed.data;+ const asset = document.assets.find((a) => a.id === assetId);+ const footage = document.assets.filter(+ (a) => a.kind === "video" && !isGeneratedAssetId(a.id),+ );+ const roster = footage.length+ ? ` Footage in this project: ${footage.map((a) => `${a.id} (${a.label})`).join(", ")}.`+ : " This project has no footage to place.";+ if (!asset) return failure(`Unknown asset: ${assetId}.${roster}`);+ if (asset.kind === "audio") {+ return failure(+ `Asset ${assetId} is audio, not footage: lay it over the recording with addAudio.`,+ );+ }+ if (isGeneratedAssetId(assetId)) {+ return failure(`Asset ${assetId} is generated media, not footage you can place.${roster}`);+ }+ const beforeClipId = parsed.data.beforeClipId ?? null;+ const clips = document.timeline.clips;+ const index =+ beforeClipId === null ? clips.length : clips.findIndex((c) => c.id === beforeClipId);+ if (index < 0) return failure(`Unknown clip: ${beforeClipId}. ${clipRoster(document)}`);+ const durationSec = asset.durationSec;+ const requestedEnd = parsed.data.sourceEndSec ?? durationSec;+ if (requestedEnd === undefined) {+ return failure(+ `The length of ${assetId} is not known yet: pass sourceEndSec, in its source seconds, to say where the clip ends.`,+ );+ }+ const bound = (sec: number) => (durationSec === undefined ? sec : Math.min(sec, durationSec));+ const sourceStartSec = bound(parsed.data.sourceStartSec);+ const sourceEndSec = bound(requestedEnd);+ if (sourceEndSec <= sourceStartSec) {+ return failure(+ `The clip would be empty: ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)} of ${assetId}` ++ (durationSec === undefined ? "." : `, which lasts ${formatSec(durationSec)}.`),+ );+ }+ const clipId = createId("clip");+ // The media panel's drop, through the same mutator.+ const next = insertAssetClip(document, index, {+ id: clipId,+ assetId,+ sourceStartSec,+ sourceEndSec,+ origin: "agent",+ reason: "",+ });+ const placed = next.timeline.clips.find((c) => c.id === clipId);+ return {+ ok: true,+ document: next,+ resultJson: JSON.stringify({+ clipId: placed ? clipId : null,+ assetId,+ sourceStartSec,+ sourceEndSec,+ ...(placed+ ? { timelineStartSec: placed.timelineStartSec, timelineEndSec: placed.timelineEndSec }+ : // Its media continues the clip before it, with the same framing: one clip.+ { joinedIntoNeighbour: true }),+ clipOrder: next.timeline.clips.map((c) => c.id),+ }),+ summary:+ `inserted ${asset.label} ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)} ` ++ (beforeClipId ? `before ${beforeClipId}` : "at the end"),+ };+ }++ case "duplicateClip": {+ const parsed = duplicateClipArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ const { clipId } = parsed.data;+ const index = document.timeline.clips.findIndex((c) => c.id === clipId);+ if (index < 0) return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`);+ // The editor's Duplicate: the copy right after the original, with its own copy of+ // the original's cuts and anchored effects.+ const next = duplicateClip(document, clipId, "agent", "");+ const copy = next.timeline.clips[index + 1];+ return {+ ok: true,+ document: next,+ resultJson: JSON.stringify({+ clipId: copy.id,+ of: clipId,+ timelineStartSec: copy.timelineStartSec,+ timelineEndSec: copy.timelineEndSec,+ clipOrder: next.timeline.clips.map((c) => c.id),+ copiedTrimIds: next.timeline.trimRanges+ .filter((t) => t.clipId === copy.id)+ .map((t) => t.id),+ }),+ summary: `duplicated ${clipId}`,+ };+ }++ case "addAutoZooms": {+ const parsed = addAutoZoomsArgs.safeParse(args);+ if (!parsed.success) return failure(parsed.error.message);+ if (document.timeline.clips.length === 0) {+ return failure("The timeline has no clips, so there is nothing to zoom on.");+ }+ const plan = options?.autoZooms;+ const reads = Object.values(plan?.telemetry ?? {});+ if (!plan || (reads.length > 0 && reads.every((status) => status === "unavailable"))) {+ return failure(+ "The recorded clicks could not be read from here, so no zoom was planned. This says nothing about whether the recording has any: report the limit as yours.",+ );+ }+ // The timeline wand's write: zooms on the clicks, clear of the zooms already there.+ const next = appendAutoZoomSuggestions(document, plan.suggestions);+ const before = new Set(document.zoomRanges.map((z) => z.id));+ const added = coalesceForAgent(next.zoomRanges).filter((z) => !before.has(z.id));+ return {+ ok: true,+ ...(added.length ? { document: next } : {}),+ resultJson: JSON.stringify({+ added: added.length,+ zooms: added.map((z) => ({+ id: z.id,+ startSec: roundSec(z.startMs),+ endSec: roundSec(z.endMs),+ renderedScale: effectiveZoomScale(z),+ focus: z.focus,+ })),+ telemetry: plan.telemetry,+ ...(added.length+ ? cutTransitionsReport(document, next)+ : {+ note: "No click on the timeline calls for a new zoom: none was recorded, each one is already under a zoom, or they fall in the first seconds or on the stop.",+ }),+ }),+ ...(added.length+ ? { summary: `added ${added.length} automatic zoom${added.length === 1 ? "" : "s"}` }+ : {}),+ };+ }+case "moveClip": {const parsed = moveClipArgs.safeParse(args);if (!parsed.success) return failure(parsed.error.message);@@ -2980,7 +3210,7 @@ export function executeAgentTool(}if (asset.kind !== "audio") {return failure(- `Asset ${assetId} is video, not audio. addAudio plays an imported audio file over the recording; to place footage use replaceTimeline.`,+ `Asset ${assetId} is video, not audio. addAudio plays an imported audio file over the recording; to place footage on the timeline use insertClip.`,);}const durationSec = asset.durationSec ?? 0;diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.tsindex a718552ff..30d9734a7 100644--- a/electron/ai-edition/deep-agent/service.test.ts+++ b/electron/ai-edition/deep-agent/service.test.ts@@ -38,7 +38,9 @@ import {anthropicCachingMiddleware,buildSystemPrompt,buildTools,+ type CursorTelemetryReader,type OpenScreenAgentSink,+ runDocumentTool,SYSTEM_PROMPT,TOOL_DESCRIPTIONS,} from "./service";@@ -67,9 +69,13 @@ const ARGS: Record<string, unknown> = {setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 },setClipRange: { clipId: "clip_1", sourceStartSec: 0, sourceEndSec: 10 },moveClip: { clipId: "clip_1", beforeClipId: null },+ insertClip: { assetId: "asset_1", sourceStartSec: 2, sourceEndSec: 4 },+ duplicateClip: { clipId: "clip_1" },replaceTimeline: { intervals: [{ startSec: 0, endSec: 10 }] },addZoom: { startSec: 1, endSec: 2 },addZooms: { regions: [{ startSec: 1, endSec: 2 }] },+ // No cursor reader in these tests: the refusal that says it could not look.+ addAutoZooms: {},setZoom: { zoomId: "zoom_nope" },addSpeed: { startSec: 1, endSec: 2 },setSpeed: { speedId: "speed_nope" },@@ -707,3 +713,43 @@ describe("what the descriptions say about a zoom's focus", () => {expect(SYSTEM_PROMPT).not.toMatch(/cursorAnchor/);});});++describe("addAutoZooms reads the clicks through the runtime's cursor reader", () => {+ const click = (timeMs: number, interactionType?: string) => ({+ timeMs,+ cx: 0.4,+ cy: 0.6,+ ...(interactionType ? { interactionType } : {}),+ });++ it("plans on the recorded clicks, as the timeline wand does", async () => {+ const reads: string[] = [];+ const cursor: CursorTelemetryReader = {+ read: async ({ assetId, originalPath }) => {+ reads.push(`${assetId}:${originalPath}`);+ return {+ status: "ok",+ assetId,+ samples: [click(3700), click(4000, "click"), click(4300, "mouseup"), click(20000)],+ };+ },+ };+ const result = await runDocumentTool(fixtureDocument(), "addAutoZooms", {}, true, { cursor });+ expect(reads).toEqual(["asset_1:/tmp/rec.mp4"]);+ expect(result.ok).toBe(true);+ const payload = JSON.parse(result.resultJson);+ expect(payload).toMatchObject({ added: 1, telemetry: { asset_1: "ok" } });+ expect(result.document?.zoomRanges).toMatchObject([{ focusMode: "auto" }]);+ });++ it("reports a reader that throws as a read it could not make", async () => {+ const cursor: CursorTelemetryReader = {+ read: async () => {+ throw new Error("disk gone");+ },+ };+ const result = await runDocumentTool(fixtureDocument(), "addAutoZooms", {}, true, { cursor });+ expect(result.ok).toBe(false);+ expect(result.resultJson).toContain("could not be read");+ });+});diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.tsindex fa610515f..408a2aa4d 100644--- a/electron/ai-edition/deep-agent/service.ts+++ b/electron/ai-edition/deep-agent/service.ts@@ -19,8 +19,10 @@import { anthropicPromptCachingMiddleware, createAgent, tool } from "langchain";import { z } from "zod";+import type { CursorTelemetryPoint } from "../../../src/components/video-editor/types";import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks";import type { AxcutDocument } from "../../../src/lib/ai-edition/schema";+import { collectAutoZoomSuggestionsForDocument } from "../../../src/lib/ai-edition/timeline/apply-auto-zooms";// ponytail: the legend is DERIVED from the depth→scale table, not retyped. Both// tool descriptions used to assert "depth 1–6 maps to 1.0×–3.5×" — a formula// (`depth/2 + 0.5`) that was purged from two rendering sites and survived here,@@ -35,8 +37,10 @@ import type { SegmentationSupport } from "../../../src/native/contracts";import {AGENT_EDITS_SETTING,type AgentToolExecution,+ type AutoZoomPlan,addAnnotationArgs,addAudioArgs,+ addAutoZoomsArgs,addCameraFullscreenArgs,addSpeedArgs,addTrimArgs,@@ -44,10 +48,12 @@ import {addZoomArgs,addZoomsArgs,type CursorTelemetryLoad,+ duplicateClipArgs,executeAgentTool,getCursorTrackArgs,getTranscriptArgs,getTranscriptWordsArgs,+ insertClipArgs,isMutatingTool,moveClipArgs,removeClipArgs,@@ -132,7 +138,8 @@ const BASE_SYSTEM_PROMPT = ["How the tools map to intent — pick the most specific one, and prefer the smallest edit that satisfies the request:","- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.","- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.",- "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.",+ "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim. It also crops the clip (cropRegion).",+ "- insertClip places footage of any video asset at a position on the timeline; duplicateClip copies a clip right after itself. addAutoZooms plans zooms on the recorded clicks, as the timeline's Automatic zooms does.",`- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`,"- setEditorSettings sets the look of the whole video: the background (wallpaper, its motion and blur), padding, rounded corners, shadow, a device frame, motion blur, depth of field, the shape of the video (aspectRatio: auto follows the recording) and how a recording of another shape fills it, the camera's layout (picture-in-picture, side by side, stacked, its shape, size, position, framing and background cutout), the cursor, and the master volume. getCurrentDocument returns all of them under settings, in the shape this tool takes, and settingsContext.inactive names each one the editor hides in the current state and why: setting one of those changes nothing, so say so instead of reporting the look as changed. setAspectRatio is the same write for the shape alone.","- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.",@@ -199,13 +206,19 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {addTrims: `Add MANY trim ranges in one call: \`ranges\` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in \`refused\` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed. ${ZOOM_TRANSITIONS_NOTE} Each applied entry carries the same \`cutTransitions\` warning addTrim reports.`,setTrim: `Move or resize an existing trim range by id. Times are source-time seconds. The cut follows to whichever clip the new range lands in, when that clip is unambiguous. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`,setClipRange:- "Set a clip's in/out points (source-time seconds) to shorten its head or tail — distinct from a trim (which cuts a span inside the clip). All clips are re-laid back-to-back afterwards, so downstream clips shift automatically. Use this ONLY when the user explicitly asks to shorten or extend a user-placed clip. Do NOT use this for 'remove silences' or 'cut pauses' — for those, use addTrim.",+ "Set a clip's in/out points (source-time seconds) to shorten its head or tail — distinct from a trim (which cuts a span inside the clip) — and/or its crop. Only the fields you pass are changed. All clips are re-laid back-to-back afterwards, so downstream clips shift automatically. Change the range ONLY when the user explicitly asks to shorten or extend a clip. Do NOT use this for 'remove silences' or 'cut pauses' — for those, use addTrim. cropRegion is the part of the source frame the clip shows, {x, y, width, height} in 0–1 fractions of it, per clip as in the editor's Edit clip; null shows the whole frame again. getCurrentDocument reports it on the clip, absent when the clip shows the whole frame. Two neighbouring clips of the same asset whose source ranges meet and whose crops match are one clip: the result says joinedIntoNeighbour when an edit made them so.",moveClip:"Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.",+ insertClip:+ "Place a clip of a video asset on the timeline, just before `beforeClipId` (null, or omitted, places it last): another take, or footage imported in the editor. Any video asset of getCurrentDocument works, the primary one too. sourceStartSec/sourceEndSec pick the stretch of the asset in its source seconds, the whole asset by default. Clips stay back to back, so the clips after it shift. Its media continuing the clip before it makes the two one clip (joinedIntoNeighbour). An audio asset is not footage: use addAudio. Nothing here imports a file from disk.",+ duplicateClip:+ "Copy a clip, placed right after it: the same footage, crop and source range, with its own copy of the clip's trims and of the zooms, speed regions, annotations and camera-fullscreen regions on it. Edits to one no longer touch the other. Returns the copy's clipId; moveClip puts it somewhere else.",replaceTimeline:- "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.",+ "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). To place another asset's footage, use insertClip. It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.",addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`,addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry, and the same \`cutTransitions\` warning. ${ZOOM_TRANSITIONS_NOTE}`,+ addAutoZooms:+ "Plan zooms on the clicks recorded with each take on the timeline and add them, as the timeline's Automatic zooms does: clicks close in time and on screen share one zoom, the first seconds and the click that stopped the take get none, and no zoom lands on one already there. Takes no arguments. Use it after cutting or rearranging a project, or on one whose zooms were removed. The result lists the zooms added, and telemetry gives each recording's read: ok, no-sidecar (it has no recorded clicks), or unavailable (it could not be read from here, which says nothing about the recording). added 0 is an answer, not a failure.",setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`,addSpeed:"Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down, from 0.25× to 16× (default 1.5×). Use to speed through slow stretches without cutting them.",@@ -342,13 +355,36 @@ export async function runDocumentTool(const load = TOOLS_READING_CURSOR.has(name)? await loadCursorTelemetry(document, args, runtime): undefined;+ const autoZooms =+ name === "addAutoZooms" && editsAllowed ? await planAutoZooms(document, runtime) : undefined;return executeAgentTool(document, name, JSON.stringify(args ?? {}), {editsAllowed,cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load },cameraSegmentation: runtime.cameraSegmentation,+ autoZooms,});}+/** The timeline wand's plan (`collectAutoZoomSuggestionsForDocument`), each recording's+ * clicks read through the runtime's cursor reader, and what each read gave. */+async function planAutoZooms(+ document: AxcutDocument,+ runtime: ToolRuntime,+): Promise<AutoZoomPlan | undefined> {+ const cursor = runtime.cursor;+ if (!cursor) return undefined;+ const telemetry: AutoZoomPlan["telemetry"] = {};+ const suggestions = await collectAutoZoomSuggestionsForDocument(document, async (videoPath) => {+ const asset = document.assets.find((a) => a.originalPath === videoPath);+ if (!asset) return [];+ const load = await readCursor(cursor, asset.id, videoPath);+ telemetry[asset.id] = load.status;+ // The sidecar's samples, read as the renderer's `getTelemetry` reads them.+ return load.status === "ok" ? (load.samples as CursorTelemetryPoint[]) : [];+ });+ return { suggestions, telemetry };+}+/** Reads the sidecar for whichever asset the call names, defaulting to the* primary one — the same resolution the executor will use to report it. */async function loadCursorTelemetry(@@ -363,8 +399,16 @@ async function loadCursorTelemetry(return { status: "unavailable", assetId };}const asset = document.assets.find((a) => a.id === assetId);+ return readCursor(runtime.cursor, assetId, asset?.originalPath ?? null);+}++async function readCursor(+ cursor: CursorTelemetryReader,+ assetId: string,+ originalPath: string | null,+): Promise<CursorTelemetryLoad> {try {- return await runtime.cursor.read({ assetId, originalPath: asset?.originalPath ?? null });+ return await cursor.read({ assetId, originalPath });} catch (error) {// A reader that throws is still "we could not look" — never "there is// none". Collapsing the two is the exact failure this tool exists to stop.@@ -414,9 +458,12 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray<readonly [string, z.ZodObject]> = [["setTrim", setTrimArgs],["setClipRange", setClipRangeArgs],["moveClip", moveClipArgs],+ ["insertClip", insertClipArgs],+ ["duplicateClip", duplicateClipArgs],["replaceTimeline", replaceTimelineArgs],["addZoom", addZoomArgs],["addZooms", addZoomsArgs],+ ["addAutoZooms", addAutoZoomsArgs],["setZoom", setZoomArgs],["addSpeed", addSpeedArgs],["setSpeed", setSpeedArgs],