Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(agent): region tools set and read every zoom, annotation and audio setting
EtienneLescot94096a10e260
diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.tsindex 7ecf29988..4a9b07a6e 100644--- a/electron/ai-edition/agent-tools.test.ts+++ b/electron/ai-edition/agent-tools.test.ts@@ -7,6 +7,7 @@ import {documentSchema,} from "../../src/lib/ai-edition/schema";import { ZOOM_DEPTH_SCALES } from "../../src/lib/ai-edition/timeline/zoom-scale";+import { zoomScaleLimit } from "../../src/native/sceneDescription";import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES } from "./agent-tools";import { measureTextInShippedFonts } from "./text-measure";@@ -2507,6 +2508,34 @@ describe("addAudio / setAudio", () => {});});+ it("fades a track in and out, reads the fades back, and refuses one past the inspector's cap", () => {+ const placed = place(withAudioAsset(), {+ assetId: "audio_1",+ startSec: 2,+ endSec: 6,+ fadeInMs: 800,+ });+ expect(placed.document?.audioTracks[0]).toMatchObject({ fadeInMs: 800, fadeOutMs: 0 });+ const id = JSON.parse(placed.resultJson).audioId;+ const faded = executeAgentTool(+ placed.document as AxcutDocument,+ "setAudio",+ JSON.stringify({ audioId: id, fadeOutMs: 1500 }),+ );+ expect(JSON.parse(faded.resultJson)).toMatchObject({ fadeInMs: 800, fadeOutMs: 1500 });+ const snapshot = JSON.parse(+ executeAgentTool(faded.document as AxcutDocument, "getCurrentDocument", "").resultJson,+ );+ expect(snapshot.audioTracks[0]).toMatchObject({ fadeInMs: 800, fadeOutMs: 1500 });+ expect(+ executeAgentTool(+ placed.document as AxcutDocument,+ "setAudio",+ JSON.stringify({ audioId: id, fadeInMs: 5001 }),+ ).ok,+ ).toBe(false);+ });+it("setAudio applies the same offset guard as addAudio", () => {const placed = place(withAudioAsset(20), { assetId: "audio_1", startSec: 2, endSec: 6 });const id = JSON.parse(placed.resultJson).audioId;@@ -3198,3 +3227,250 @@ describe("zoom transitions are reported, not hidden (#1028)", () => {]);});});++// #1180 batch C: what the inspector sets on a zoom or an annotation, the agent sets and reads back.+describe("region tools reach the inspector's settings (#1180)", () => {+ const run = (doc: AxcutDocument, name: string, args: Record<string, unknown>) => {+ const result = executeAgentTool(doc, name, JSON.stringify(args));+ return { ...result, payload: JSON.parse(result.resultJson) };+ };+ const snapshot = (doc: AxcutDocument | undefined) =>+ JSON.parse(executeAgentTool(doc as AxcutDocument, "getCurrentDocument", "").resultJson);++ it("sets a zoom's focus mode, hidden cursor and free level, and reads them back", () => {+ const added = run(fixtureDocument(), "addZoom", {+ startSec: 2,+ endSec: 4,+ focusMode: "auto",+ hideCursor: true,+ customScale: 1.6,+ });+ expect(added.ok).toBe(true);+ expect(added.document?.zoomRanges[0]).toMatchObject({+ focusMode: "auto",+ hideCursor: true,+ customScale: 1.6,+ });+ expect(added.payload).toMatchObject({+ focusMode: "auto",+ hideCursor: true,+ renderedScale: 1.6,+ });+ expect(() => documentSchema.parse(added.document)).not.toThrow();+ expect(snapshot(added.document).zoomRanges[0]).toMatchObject({+ focusMode: "auto",+ hideCursor: true,+ customScale: 1.6,+ renderedScale: 1.6,+ });++ const id = added.payload.zoomId;+ const shown = run(added.document as AxcutDocument, "setZoom", {+ zoomId: id,+ hideCursor: false,+ focusMode: "manual",+ });+ expect(shown.document?.zoomRanges[0]).not.toHaveProperty("hideCursor");+ expect(shown.document?.zoomRanges[0]).toMatchObject({ focusMode: "manual", customScale: 1.6 });+ expect(snapshot(shown.document).zoomRanges[0]).not.toHaveProperty("hideCursor");+ });++ it("writes a level the depth table has as that depth, as the inspector does", () => {+ const added = run(fixtureDocument(), "addZoom", { startSec: 2, endSec: 4, customScale: 2.2 });+ expect(added.document?.zoomRanges[0]).toMatchObject({ depth: 4 });+ expect(added.document?.zoomRanges[0]).not.toHaveProperty("customScale");++ const custom = run(added.document as AxcutDocument, "setZoom", {+ zoomId: added.payload.zoomId,+ customScale: 1.333,+ });+ // To the hundredth, like a typed level.+ expect(custom.document?.zoomRanges[0]).toMatchObject({ depth: 4, customScale: 1.33 });+ const preset = run(custom.document as AxcutDocument, "setZoom", {+ zoomId: added.payload.zoomId,+ customScale: 1.5,+ });+ expect(preset.document?.zoomRanges[0]).toMatchObject({ depth: 2 });+ expect(preset.document?.zoomRanges[0]).not.toHaveProperty("customScale");+ });++ it("refuses a level past the clip's own limit, naming it, and depth with customScale", () => {+ const added = run(fixtureDocument(), "addZoom", { startSec: 2, endSec: 4 });+ const limit = zoomScaleLimit(added.document as AxcutDocument, added.payload.zoomId);+ // The fixture's 1080p take at the default padding blurs before the renderer's own 5×.+ expect(limit).toBeLessThan(5);+ const past = run(added.document as AxcutDocument, "setZoom", {+ zoomId: added.payload.zoomId,+ customScale: limit + 0.1,+ });+ expect(past.ok).toBe(false);+ expect(past.payload.error).toContain(`${limit}×`);+ expect(past.document).toBeUndefined();+ expect(+ run(fixtureDocument(), "addZoom", { startSec: 2, endSec: 4, customScale: limit + 0.1 }).ok,+ ).toBe(false);+ expect(+ run(added.document as AxcutDocument, "setZoom", {+ zoomId: added.payload.zoomId,+ customScale: limit,+ }).ok,+ ).toBe(true);+ const both = run(fixtureDocument(), "addZoom", {+ startSec: 2,+ endSec: 4,+ depth: 3,+ customScale: 2,+ });+ expect(both.ok).toBe(false);+ });++ it("reads an annotation back in the terms its writes take", () => {+ const added = run(fixtureDocument(), "addAnnotation", {+ startSec: 1,+ endSec: 3,+ text: "Hi",+ x: 30,+ y: 70,+ fontSize: 48,+ background: "none",+ animation: "pop",+ });+ const [ann] = snapshot(added.document).annotations;+ expect(ann).toMatchObject({+ type: "text",+ text: "Hi",+ box: "frame",+ x: expect.closeTo(30, 1),+ y: expect.closeTo(70, 1),+ fontSize: 48,+ color: "#ffffff",+ background: "none",+ bold: true,+ animation: "pop",+ });+ expect(ann.width).toBeGreaterThan(0);+ expect(added.payload).toMatchObject({ type: "text", fontSize: 48 });+ });++ it("adds a privacy blur on the recording's picture, sized and placed as asked", () => {+ const added = run(fixtureDocument(), "addAnnotation", {+ startSec: 1,+ endSec: 3,+ type: "blur",+ x: 20,+ y: 10,+ width: 30,+ height: 6,+ blurData: { type: "blur", intensity: 20 },+ });+ expect(added.ok).toBe(true);+ const stored = added.document?.annotations[0];+ expect(stored).toMatchObject({+ type: "blur",+ content: "",+ position: { x: 5, y: 7 },+ size: { width: 30, height: 6 },+ blurData: { type: "blur", shape: "rectangle", intensity: 20 },+ });+ // A blur stays on what it hides: measured on the footage, never on the frame.+ expect(stored?.space).toBeUndefined();+ expect(() => documentSchema.parse(added.document)).not.toThrow();+ expect(snapshot(added.document).annotations[0]).toMatchObject({+ type: "blur",+ box: "footage",+ x: 20,+ y: 10,+ width: 30,+ height: 6,+ blurData: { type: "blur", shape: "rectangle", color: "white", intensity: 20, blockSize: 12 },+ });+ expect(snapshot(added.document).annotations[0]).not.toHaveProperty("text");++ // A patch of blurData changes what it names and keeps the rest.+ const oval = run(added.document as AxcutDocument, "setAnnotation", {+ annotationId: added.payload.annotationId,+ blurData: { shape: "oval" },+ });+ expect(oval.document?.annotations[0].blurData).toMatchObject({+ type: "blur",+ shape: "oval",+ intensity: 20,+ });+ });++ it("adds an arrow on the frame with its figureData", () => {+ const added = run(fixtureDocument(), "addAnnotation", {+ startSec: 1,+ endSec: 3,+ type: "figure",+ figureData: { arrowDirection: "down-left", color: "#ff0000" },+ });+ expect(added.ok).toBe(true);+ expect(added.document?.annotations[0]).toMatchObject({+ type: "figure",+ space: "frame",+ size: { width: 30, height: 20 },+ figureData: { arrowDirection: "down-left", color: "#ff0000", strokeWidth: 4 },+ });+ expect(() => documentSchema.parse(added.document)).not.toThrow();+ expect(snapshot(added.document).annotations[0]).toMatchObject({+ type: "figure",+ box: "frame",+ x: 50,+ y: 50,+ figureData: { arrowDirection: "down-left" },+ });+ });++ it("switches an annotation's type as the inspector does, keeping the text to come back to", () => {+ const added = run(fixtureDocument(), "addAnnotation", {+ startSec: 1,+ endSec: 3,+ text: "[email protected]",+ });+ const id = added.payload.annotationId;+ const blurred = run(added.document as AxcutDocument, "setAnnotation", {+ annotationId: id,+ type: "blur",+ });+ expect(blurred.ok).toBe(true);+ const blur = blurred.document?.annotations[0];+ expect(blur).toMatchObject({ type: "blur", content: "", textContent: "[email protected]" });+ expect(blur?.space).toBeUndefined();+ expect(() => documentSchema.parse(blurred.document)).not.toThrow();++ const back = run(blurred.document as AxcutDocument, "setAnnotation", {+ annotationId: id,+ type: "text",+ });+ expect(back.document?.annotations[0]).toMatchObject({+ type: "text",+ content: "[email protected]",+ space: "frame",+ });+ });++ it("refuses arguments that mean nothing for the annotation's kind", () => {+ const text = run(fixtureDocument(), "addAnnotation", { startSec: 1, endSec: 3, text: "Hi" });+ const id = text.payload.annotationId;+ const doc = text.document as AxcutDocument;+ expect(run(doc, "setAnnotation", { annotationId: id, width: 40 }).ok).toBe(false);+ expect(run(doc, "setAnnotation", { annotationId: id, blurData: { type: "blur" } }).ok).toBe(+ false,+ );+ expect(+ run(doc, "setAnnotation", { annotationId: id, type: "blur", blurData: { type: "blur" } }).ok,+ ).toBe(true);+ expect(+ run(fixtureDocument(), "addAnnotation", {+ startSec: 1,+ endSec: 3,+ type: "blur",+ blurData: { shape: "freehand" },+ }).ok,+ ).toBe(false);+ expect(+ run(fixtureDocument(), "addAnnotation", { startSec: 1, endSec: 3, type: "image" }).ok,+ ).toBe(false);+ });+});diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.tsindex c63a56d4c..9e01e253b 100644--- a/electron/ai-edition/agent-tools.ts+++ b/electron/ai-edition/agent-tools.ts@@ -17,8 +17,13 @@import { z } from "zod";import { DEFAULT_TEXT_PLATE } from "../../src/lib/ai-edition/annotations/background";-import { fitTextInFrame } from "../../src/lib/ai-edition/annotations/placement";import {+ convertAnnotationKind,+ DEFAULT_ANNOTATION_BOX,+ fitTextInFrame,+} from "../../src/lib/ai-edition/annotations/placement";+import {+ AUDIO_FADE_MAX_MS,collapseTracksToPills,patchAudioTrack,placeAudioTrackInDocument,@@ -36,7 +41,12 @@ import {setClipSourceRange,} from "../../src/lib/ai-edition/document/timeline";import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript";-import type { AxcutAnnotationRegion, AxcutDocument } from "../../src/lib/ai-edition/schema";+import {+ type AxcutAnnotationRegion,+ type AxcutDocument,+ blurDataSchema,+ figureDataSchema,+} from "../../src/lib/ai-edition/schema";import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings";import { applySettingsPatch } from "../../src/lib/ai-edition/store/settingsPatch";import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera";@@ -58,7 +68,10 @@ import {} from "../../src/lib/ai-edition/timeline/trim-mapping";import {effectiveZoomScale,+ MAX_ZOOM_SCALE,+ MIN_ZOOM_SCALE,ZOOM_DEPTH_LEGEND,+ zoomLevelFor,} from "../../src/lib/ai-edition/timeline/zoom-scale";import {chainedPans,@@ -66,6 +79,7 @@ import {zoomTransitions,} from "../../src/lib/ai-edition/timeline/zoom-transitions";import { SETTING_BOUNDS } from "../../src/lib/projectDefaults";+import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription";import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils";import { measureTextInShippedFonts } from "./text-measure";@@ -198,6 +212,129 @@ function applyAnnotationLook(return next;}+const round2 = (n: number) => Math.round(n * 100) / 100;++/**+ * One annotation as the agent reads it back, in the terms its writes take: the centre (x, y) and+ * size of its box, which box that is, and the look of its kind. A text reports the look arguments,+ * an arrow its figureData and a blur its blurData, the defaults filled in as the renderer fills+ * them. An image reports its box only: its content is a data URL, megabytes of nothing to read.+ */+function annotationForAgent(a: AxcutAnnotationRegion): Record<string, unknown> {+ const geometry = {+ box: a.space === "frame" ? "frame" : "footage",+ x: round2(a.position.x + a.size.width / 2),+ y: round2(a.position.y + a.size.height / 2),+ width: round2(a.size.width),+ height: round2(a.size.height),+ };+ switch (a.type) {+ case "text":+ return {+ type: a.type,+ text: a.content || a.textContent || "",+ ...geometry,+ fontSize: a.style.fontSize,+ color: a.style.color,+ background: a.style.backgroundColor === "transparent" ? "none" : a.style.backgroundColor,+ bold: a.style.fontWeight === "bold",+ animation: a.style.textAnimation ?? "none",+ };+ case "figure":+ return {+ type: a.type,+ ...geometry,+ figureData: figureDataSchema.unwrap().parse(a.figureData ?? {}),+ };+ case "blur":+ return {+ type: a.type,+ ...geometry,+ blurData: blurDataSchema.unwrap().parse(a.blurData ?? {}),+ };+ default:+ return { type: a.type, ...geometry };+ }+}++/** The arguments that only mean something for one kind of annotation, refused on another. */+function refuseShapeMismatch(+ type: AxcutAnnotationRegion["type"],+ data: {+ text?: string;+ width?: number;+ height?: number;+ figureData?: unknown;+ blurData?: { shape?: string };+ },+): AgentToolExecution | null {+ if (type === "text" && (data.width !== undefined || data.height !== undefined)) {+ return failure("A text box is sized to its words: change fontSize, not width or height.");+ }+ if (type !== "text" && data.text !== undefined) {+ return failure(`text is for a text annotation; this one is ${type}. Pass type "text" with it.`);+ }+ if (type !== "figure" && data.figureData !== undefined) {+ return failure(`figureData is an arrow's; this annotation is ${type}. Pass type "figure".`);+ }+ if (type !== "blur" && data.blurData !== undefined) {+ return failure(`blurData is a blur's; this annotation is ${type}. Pass type "blur".`);+ }+ // The editor no longer offers it either: its input was broken and it renders as its bounding+ // box, and a privacy mask that is half reliable is worse than none.+ if (data.blurData?.shape === "freehand") {+ return failure('A freehand blur is not offered: use shape "rectangle" or "oval".');+ }+ return null;+}++/** The fields of `parsed` the caller actually sent: the schema fills in defaults, and a patch+ * must not reset what it never named. */+function sentKeys<T extends object>(parsed: T | undefined, raw: unknown): Partial<T> {+ if (!parsed || !raw || typeof raw !== "object") return {};+ return Object.fromEntries(Object.entries(parsed).filter(([key]) => key in raw)) as Partial<T>;+}++/**+ * An arrow's, a blur's or an image's box and data: `size` centred on `center` and kept inside its+ * reference box, as the editor keeps one it moves; then the figureData or blurData the caller sent,+ * over what the annotation had or the schema's defaults.+ */+function shapeAnnotation(+ region: AxcutAnnotationRegion,+ center: { x: number; y: number },+ size: { width: number; height: number },+ data: { figureData?: object; blurData?: object },+ rawArgs: unknown,+): AxcutAnnotationRegion {+ const raw = (rawArgs ?? {}) as { figureData?: unknown; blurData?: unknown };+ const { width, height } = size;+ return {+ ...region,+ position: {+ x: Math.min(Math.max(center.x - width / 2, 0), Math.max(0, 100 - width)),+ y: Math.min(Math.max(center.y - height / 2, 0), Math.max(0, 100 - height)),+ },+ size: { width, height },+ ...(region.type === "figure"+ ? {+ figureData: {+ ...figureDataSchema.unwrap().parse(region.figureData ?? {}),+ ...sentKeys(data.figureData, raw.figureData),+ },+ }+ : {}),+ ...(region.type === "blur"+ ? {+ blurData: {+ ...blurDataSchema.unwrap().parse(region.blurData ?? {}),+ ...sentKeys(data.blurData, raw.blurData),+ },+ }+ : {}),+ } as AxcutAnnotationRegion;+}+/*** A text annotation on the output frame, its box sized to its text around `center` (frame* percentages): what the editor's "add text" does, so the agent's text sits where the editor's@@ -584,12 +721,28 @@ const focusSchema = z.object({ cx: z.number().min(0).max(1), cy: z.number().min(* (getCurrentDocument) as `rotationPreset`, absent when flat. */const rotationPresetSchema = z.enum(["none", "left", "right", "orbit"]);+/** The inspector's focus mode: `manual` holds `focus`, `auto` follows the recorded pointer. */+const focusModeSchema = z.enum(["manual", "auto"]);++/** The inspector's free zoom level: a factor, refused outside the range it accepts. The clip's own+ * limit (`zoomScaleLimit`) is checked once the zoom is placed, as the inspector checks it. */+const customScaleSchema = z+ .number()+ .min(MIN_ZOOM_SCALE)+ .max(MAX_ZOOM_SCALE)+ .describe(+ `A free zoom level, as a factor (${MIN_ZOOM_SCALE}–${MAX_ZOOM_SCALE}, e.g. 2 for 2×), in place of depth. Refused past the clip's own limit, the deepest its recording takes before it blurs; the refusal names it.`,+ );+export const addZoomArgs = z.object({startSec: secondsSchema,endSec: secondsSchema,depth: depthSchema.default(3),focus: focusSchema.default({ cx: 0.5, cy: 0.5 }),rotationPreset: rotationPresetSchema.default("none"),+ focusMode: focusModeSchema.default("manual"),+ hideCursor: z.boolean().default(false),+ customScale: customScaleSchema.optional(),});/** A zoom with its 3D camera set: `none` drops the key, as a flat zoom never carries it. */@@ -601,6 +754,59 @@ function withRotationPreset<Z extends { rotationPreset?: unknown }>(return (preset === "none" ? flat : { ...flat, rotationPreset: preset }) as Z;}+/** `customScale` as the inspector's typed level writes it: to the hundredth, and a level the depth+ * table has as that depth (`zoomLevelFor`). Empty when the caller passed none. */+function zoomLevelArg(customScale: number | undefined): {+ depth?: 1 | 2 | 3 | 4 | 5 | 6;+ customScale?: number;+} {+ return customScale === undefined ? {} : zoomLevelFor(Math.round(customScale * 100) / 100);+}++/** depth and customScale are two ways of saying one thing: both at once has no single meaning. */+function refuseTwoLevels(args: unknown): AgentToolExecution | null {+ const raw = (args ?? {}) as { depth?: unknown; customScale?: unknown };+ return raw.depth !== undefined && raw.customScale !== undefined+ ? failure("Pass depth or customScale, not both: they set the same zoom level.")+ : null;+}++/**+ * The inspector's bound on a typed level: the deepest the clip under the zoom takes before its+ * recording blurs (`zoomScaleLimit`, the bound the scene applies), the tightest over every+ * fragment. Refused rather than clamped, as the inspector refuses it, so the caller hears the limit.+ */+function refuseScalePastLimit(+ next: AxcutDocument,+ ids: string[],+ customScale: number | undefined,+): AgentToolExecution | null {+ if (customScale === undefined) return null;+ const scale = Math.round(customScale * 100) / 100;+ const limit = Math.min(...ids.map((id) => zoomScaleLimit(next, id)));+ return scale > limit+ ? failure(+ `The zoom was NOT written: customScale ${scale} is deeper than this clip takes (${limit}× at most), past which its recording blurs. Pick ${limit} or less.`,+ )+ : null;+}++/** focusMode, hideCursor and customScale as the snapshot reports them, for a write's result. */+function zoomBehaviourReport(+ zoom: AxcutDocument["zoomRanges"][number],+ document: AxcutDocument,+): Record<string, unknown> {+ const autoFocusAll = (document.legacyEditor as Record<string, unknown> | null)?.autoFocusAll;+ return {+ focusMode: autoFocusAll === true ? "auto" : (zoom.focusMode ?? "manual"),+ ...(autoFocusAll === true && zoom.focusMode !== "auto"+ ? { focusModeOverriddenByAutoFocusAll: true }+ : {}),+ ...(zoom.hideCursor ? { hideCursor: true } : {}),+ ...(zoom.customScale != null ? { customScale: zoom.customScale } : {}),+ };+}+/** Same contract as `addTrimsArgs`: the element schema IS the unitary one, and* it is advertised rather than enforced so a bad region is refused by itself. */export const addZoomsArgs = z.object({@@ -614,6 +820,9 @@ export const setZoomArgs = z.object({depth: depthSchema.optional(),focus: focusSchema.optional(),rotationPreset: rotationPresetSchema.optional(),+ focusMode: focusModeSchema.optional(),+ hideCursor: z.boolean().optional(),+ customScale: customScaleSchema.optional(),});// The bound every speed reader applies (`readSpeedRegions`): refused here, so the agent hears@@ -656,25 +865,61 @@ const annotationLookArgs = {.describe("How the text enters: typewriter types it out, pop springs it in, and so on."),};+/** The kinds the inspector's type switch offers, image aside: an image needs a file read. */+const annotationTypeSchema = z+ .enum(["text", "figure", "blur"])+ .describe(+ "text: words on the frame. figure: an arrow. blur: a privacy mask that blurs or pixelates what is under it, such as an email address or a token on screen.",+ );++/** What an arrow or a blur has instead of a text's look: its box, and its own data. */+const annotationShapeArgs = {+ width: z+ .number()+ .positive()+ .max(100)+ .optional()+ .describe(+ "Box width, in percent of the annotation's box: the output frame for an arrow, the recording's picture for a blur. Not for text, whose box is its words.",+ ),+ height: z+ .number()+ .positive()+ .max(100)+ .optional()+ .describe("Box height, in percent of the same box as width."),+ figureData: figureDataSchema.describe("An arrow's direction, colour and stroke width."),+ blurData: blurDataSchema.describe(+ "A blur's look: type blur or mosaic, shape rectangle or oval, color of the mosaic grid, intensity, blockSize.",+ ),+};+export const addAnnotationArgs = z.object({startSec: secondsSchema,endSec: secondsSchema,+ type: annotationTypeSchema.default("text"),text: z.string().default(""),x: z.number().min(0).max(100).default(50),y: z.number().min(0).max(100).default(50),...annotationLookArgs,+ ...annotationShapeArgs,});export const setAnnotationArgs = z.object({annotationId: z.string().min(1),startSec: secondsSchema.optional(),endSec: secondsSchema.optional(),+ type: annotationTypeSchema.optional(),text: z.string().optional(),x: z.number().min(0).max(100).optional(),y: z.number().min(0).max(100).optional(),...annotationLookArgs,+ ...annotationShapeArgs,});+/** A track's fade, capped where the inspector's slider stops. */+const fadeMsSchema = z.number().int().min(0).max(AUDIO_FADE_MAX_MS);+export const addAudioArgs = z.object({assetId: z.string().min(1),startSec: secondsSchema,@@ -682,6 +927,8 @@ export const addAudioArgs = z.object({kind: z.enum(["voiceover", "music"]).default("music"),offsetSec: secondsSchema.default(0),gainDb: z.number().min(-60).max(12).default(0),+ fadeInMs: fadeMsSchema.default(0),+ fadeOutMs: fadeMsSchema.default(0),});export const setAudioArgs = z.object({@@ -693,6 +940,8 @@ export const setAudioArgs = z.object({gainDb: z.number().min(-60).max(12).optional(),muted: z.boolean().optional(),loop: z.boolean().optional(),+ fadeInMs: fadeMsSchema.optional(),+ fadeOutMs: fadeMsSchema.optional(),});export const addCameraFullscreenArgs = z.object({@@ -996,7 +1245,10 @@ export function documentSnapshotForModel("timeline, longer above 1x and shorter below, so read the bounds rather than adding transitionSec). cutByTrim names the move a trim cuts into (in, out or both), cutByJunction the move a clip junction cuts (the clip ends before the move does): the export jumps at that cut. " +"Two zooms at most 1.5 s apart on screen, with no trim or clip junction between them, are CHAINED: " +"no zoom-out and back in, the camera pans from the first (panToZoomId, panUntilSec: 1 s of screen " +- "time from its endSec, which can run into the next span) to the second (panFromZoomId).",+ "time from its endSec, which can run into the next span) to the second (panFromZoomId). " ++ "focusMode auto follows the recorded pointer instead of holding focus; hideCursor: true hides the pointer while the zoom is on screen (absent, it shows).",+ annotationNote:+ "Each annotation reports its box the way addAnnotation and setAnnotation take it: x/y its centre, width/height its size, in percent of `box`. box 'frame' is the output frame (text and arrows, which padding and zooms never move); box 'footage' is the recording's picture (a blur, which stays on what it hides when the picture moves). A text also reports its look (fontSize, color, background, bold, animation), an arrow (type 'figure') its figureData, a blur its blurData.",project: { id: document.project.id, title: document.project.title },primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null,autoFocusAll,@@ -1074,6 +1326,7 @@ export function documentSnapshotForModel(...(z.rotationPreset ? { rotationPreset: z.rotationPreset } : {}),focus: z.focus,focusMode: autoFocusAll ? "auto" : (z.focusMode ?? "manual"),+ ...(z.hideCursor ? { hideCursor: true } : {}),source: z.source ?? "manual",}),),@@ -1087,8 +1340,7 @@ export function documentSnapshotForModel(id: a.id,startSec: roundSec(a.startMs),endSec: roundSec(a.endMs),- type: a.type,- text: a.textContent ?? a.content ?? "",+ ...annotationForAgent(a),})),cameraFullscreenRegions: coalesceForAgent(cameraFullscreenRegions).map((c) => ({id: c.id,@@ -1110,6 +1362,8 @@ export function documentSnapshotForModel(gainDb: t.gainDb,muted: t.muted,loop: t.loop,+ fadeInMs: t.fadeInMs,+ fadeOutMs: t.fadeOutMs,})),hasTranscript: document.transcripts.length > 0 || document.transcript !== null,};@@ -2170,16 +2424,21 @@ export function executeAgentTool(case "addZoom": {const parsed = addZoomArgs.safeParse(args);if (!parsed.success) return failure(parsed.error.message);+ const twoLevels = refuseTwoLevels(args);+ if (twoLevels) return twoLevels;const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));+ const level = zoomLevelArg(parsed.data.customScale);const zoom = withRotationPreset({id: createId("zoom"),startMs,endMs,depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6,+ ...level,focus: parsed.data.focus,- focusMode: "manual" as const,+ focusMode: parsed.data.focusMode,+ ...(parsed.data.hideCursor ? { hideCursor: true } : {}),source: "manual" as const,} as AxcutDocument["zoomRanges"][number],parsed.data.rotationPreset,@@ -2191,6 +2450,8 @@ export function executeAgentTool(...document,zoomRanges: [...document.zoomRanges, ...placed] as AxcutDocument["zoomRanges"],};+ const pastLimit = refuseScalePastLimit(next, landing.ids, parsed.data.customScale);+ if (pastLimit) return pastLimit;// Measured over what was STORED, never over what was asked for: `placed`// is the clamped, ventilated truth, so the report cannot end up// describing a window the zoom does not occupy.@@ -2205,6 +2466,7 @@ export function executeAgentTool(// The depth alone is an ordinal; reported on its own it is what the// model turns into "3×" for a frame that renders 1.80×.renderedScale: effectiveZoomScale(zoom),+ ...zoomBehaviourReport(zoom, document),...landingReport(landing, startMs / 1000, endMs / 1000),...(anchor ? { cursorAnchor: anchor } : {}),...cutTransitionsReport(document, next),@@ -2225,6 +2487,8 @@ export function executeAgentTool(case "setZoom": {const parsed = setZoomArgs.safeParse(args);if (!parsed.success) return failure(parsed.error.message);+ const twoLevels = refuseTwoLevels(args);+ if (twoLevels) return twoLevels;const { zoomId } = parsed.data;const existing = document.zoomRanges.find((z) => z.id === zoomId);if (!existing) return failure(`Unknown zoom: ${zoomId}`);@@ -2244,17 +2508,26 @@ export function executeAgentTool(const clearsCustomScale =parsed.data.depth !== undefined &&document.zoomRanges.some((z) => zoomPill.has(z.id) && z.customScale != null);+ const level = zoomLevelArg(parsed.data.customScale);+ const { hideCursor, focusMode } = parsed.data;const rebuiltZooms = replacePillSpan(// payload edits first, applied to every region under the pill…document.zoomRanges.map((z) => {if (!zoomPill.has(z.id)) return z;- const { customScale, ...rest } = z;+ const { customScale, hideCursor: wasHidden, ...rest } = z;+ // A level the table has is written as its depth, which drops the custom scale.+ const keepsScale = !clearsCustomScale && !("depth" in level);const edited = {- ...(clearsCustomScale ? rest : z),+ ...rest,+ ...(keepsScale && customScale != null ? { customScale } : {}),+ // As the inspector writes it: shown is the absence of the flag.+ ...((hideCursor ?? wasHidden) ? { hideCursor: true } : {}),...(parsed.data.depth !== undefined? { depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6 }: {}),+ ...level,...(parsed.data.focus ? { focus: parsed.data.focus } : {}),+ ...(focusMode ? { focusMode } : {}),};return parsed.data.rotationPreset === undefined? edited@@ -2270,6 +2543,8 @@ export function executeAgentTool(const landing = landingAfterPillEdit(document.zoomRanges, rebuiltZooms, zoomPill, document);if (!landing.anchored) return coversNoClip("zoom", startMs / 1000, endMs / 1000, document);const next: AxcutDocument = { ...document, zoomRanges: rebuiltZooms };+ const pastLimit = refuseScalePastLimit(next, landing.ids, parsed.data.customScale);+ if (pastLimit) return pastLimit;// Read back off the document, not off the request: the pill may have been// re-ventilated, and `renderedScale` is the only number the viewer sees.const landed = new Set(landing.ids);@@ -2296,6 +2571,7 @@ export function executeAgentTool(depth: strength.depth,renderedScale: effectiveZoomScale(strength),...(strength.rotationPreset ? { rotationPreset: strength.rotationPreset } : {}),+ ...zoomBehaviourReport(strength, document),}: {}),...(clearsCustomScale ? { clearedCustomScale: true } : {}),@@ -2393,11 +2669,17 @@ export function executeAgentTool(case "addAnnotation": {const parsed = addAnnotationArgs.safeParse(args);if (!parsed.success) return failure(parsed.error.message);+ const { type } = parsed.data;+ const mismatch = refuseShapeMismatch(type, {+ ...parsed.data,+ text: parsed.data.text || undefined,+ });+ if (mismatch) return mismatch;const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));// The editor's new text: its defaults, then whatever look the caller asked for, on the// output frame with the box fitted to the text around x/y (the text's centre).- const ann = placeTextOnFrame(+ const text = placeTextOnFrame({id: createId("ann"),startMs,@@ -2426,6 +2708,29 @@ export function executeAgentTool({ x: parsed.data.x, y: parsed.data.y },document,);+ // An arrow or a blur is that text switched to its kind, as the inspector's type switch+ // does it, then given its box around x/y and its data. Nothing to convert through: the+ // box is set right after, in the kind's own reference box.+ const ann =+ type === "text"+ ? text+ : shapeAnnotation(+ {+ ...text,+ ...convertAnnotationKind(text, type, {+ footage: null,+ frameAspect: 0,+ measure: measureTextInShippedFonts,+ }),+ },+ { x: parsed.data.x, y: parsed.data.y },+ {+ width: parsed.data.width ?? DEFAULT_ANNOTATION_BOX.width,+ height: parsed.data.height ?? DEFAULT_ANNOTATION_BOX.height,+ },+ parsed.data,+ args,+ );const placed = anchorForAgent(ann, document, "ann");const landing = landingOf(placed, document);if (!landing.anchored) {@@ -2441,9 +2746,13 @@ export function executeAgentTool(resultJson: JSON.stringify({annotationId: landing.ids[0],...landingReport(landing, startMs / 1000, endMs / 1000),+ ...annotationForAgent(placed[0] as AxcutAnnotationRegion),}),summary:- `added annotation "${parsed.data.text.slice(0, 24)}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` ++ (type === "text"+ ? `added annotation "${parsed.data.text.slice(0, 24)}"`+ : `added ${type === "blur" ? "blur" : "arrow"}`) ++ ` ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +landingSuffix(landing, startMs / 1000, endMs / 1000),};}@@ -2461,8 +2770,28 @@ export function executeAgentTool(endMs,});if (blocked) return blocked;- const { x, y, text } = parsed.data;- const edit = (a: AxcutAnnotationRegion): AxcutAnnotationRegion => {+ const { x, y, text, type, width, height } = parsed.data;+ const mismatch = refuseShapeMismatch(type ?? existing.type, parsed.data);+ if (mismatch) return mismatch;+ const frameAspect = resolveAspectRatioValue(+ document,+ getEditorSettings(document).aspectRatio,+ );+ const edit = (original: AxcutAnnotationRegion): AxcutAnnotationRegion => {+ // The inspector's type switch, through the footage under this very fragment: the box+ // changes reference (a blur lives on the footage, the rest on the frame) without+ // moving on screen, and the previous kind's content is parked, not lost.+ const converted = type !== undefined && type !== original.type;+ const a = converted+ ? {+ ...original,+ ...convertAnnotationKind(original, type, {+ footage: annotationFootageRect(document, original.id),+ frameAspect,+ measure: measureTextInShippedFonts,+ }),+ }+ : original;const next: AxcutAnnotationRegion = {...a,...(text !== undefined ? { content: text, textContent: text } : {}),@@ -2485,16 +2814,23 @@ export function executeAgentTool(? placeTextOnFrame(next, centre, document): next;}- if (!moved) return next;- // An image or an arrow keeps its size and moves its centre, inside its own box.- const { width, height } = a.size;- return {- ...next,- position: {- x: Math.min(Math.max(centre.x - width / 2, 0), Math.max(0, 100 - width)),- y: Math.min(Math.max(centre.y - height / 2, 0), Math.max(0, 100 - height)),- },- };+ const reshaped =+ moved ||+ converted ||+ width !== undefined ||+ height !== undefined ||+ parsed.data.figureData !== undefined ||+ parsed.data.blurData !== undefined;+ if (!reshaped) return next;+ // An image, an arrow or a blur keeps its size unless given one, and moves its centre,+ // inside its own box.+ return shapeAnnotation(+ next,+ centre,+ { width: width ?? a.size.width, height: height ?? a.size.height },+ parsed.data,+ args,+ );};const rebuiltAnnotations = replacePillSpan(document.annotations.map((a) => (annPill.has(a.id) ? edit(a) : a)),@@ -2514,12 +2850,14 @@ export function executeAgentTool(return coversNoClip("annotation", startMs / 1000, endMs / 1000, document);}const next: AxcutDocument = { ...document, annotations: rebuiltAnnotations };+ const landed = rebuiltAnnotations.find((a) => a.id === landing.ids[0]);return {ok: true,document: next,resultJson: JSON.stringify({annotationId: landing.ids[0] ?? annotationId,...landingReport(landing, startMs / 1000, endMs / 1000),+ ...(landed ? annotationForAgent(landed) : {}),}),summary:`updated annotation ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +@@ -2611,7 +2949,7 @@ export function executeAgentTool(case "addAudio": {const parsed = addAudioArgs.safeParse(args);if (!parsed.success) return failure(parsed.error.message);- const { assetId, kind, offsetSec, gainDb } = parsed.data;+ const { assetId, kind, offsetSec, gainDb, fadeInMs, fadeOutMs } = parsed.data;const asset = document.assets.find((a) => a.id === assetId);// Two distinct refusals, because they need two different corrections: an// unknown id is a hallucinated asset, a video id is the model reaching for@@ -2662,8 +3000,8 @@ export function executeAgentTool(offsetMs: toMs(offsetSec),gainDb,loop: false,- fadeInMs: 0,- fadeOutMs: 0,+ fadeInMs,+ fadeOutMs,muted: false,label: asset.label,origin: "agent",@@ -2716,6 +3054,8 @@ export function executeAgentTool(...(parsed.data.gainDb !== undefined ? { gainDb: parsed.data.gainDb } : {}),...(parsed.data.muted !== undefined ? { muted: parsed.data.muted } : {}),...(parsed.data.loop !== undefined ? { loop: parsed.data.loop } : {}),+ ...(parsed.data.fadeInMs !== undefined ? { fadeInMs: parsed.data.fadeInMs } : {}),+ ...(parsed.data.fadeOutMs !== undefined ? { fadeOutMs: parsed.data.fadeOutMs } : {}),...(parsed.data.offsetSec !== undefined ? { offsetMs: toMs(parsed.data.offsetSec) } : {}),});@@ -2762,6 +3102,8 @@ export function executeAgentTool(audioId,startSec: roundSec(after?.startMs ?? pill.startMs),endSec: roundSec(after?.endMs ?? pill.endMs),+ fadeInMs: after?.fadeInMs ?? pill.fadeInMs,+ fadeOutMs: after?.fadeOutMs ?? pill.fadeOutMs,}),summary: `updated audio ${audioId} ${formatSec(roundSec(after?.startMs ?? pill.startMs))} – ${formatSec(roundSec(after?.endMs ?? pill.endMs))}`,};diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.tsindex 28390d4bf..d14b3e9ef 100644--- a/electron/ai-edition/deep-agent/service.test.ts+++ b/electron/ai-edition/deep-agent/service.test.ts@@ -18,6 +18,7 @@ import { ChatAnthropic } from "@langchain/anthropic";import type { StructuredToolInterface } from "@langchain/core/tools";import { ChatOpenAI } from "@langchain/openai";import { describe, expect, it } from "vitest";+import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks";import {type AxcutDocument,createEmptyDocument,@@ -388,6 +389,17 @@ describe("what the descriptions say about zoom strength", () => {expect(ZOOM_DEPTH_LEGEND).toContain(`3=${ZOOM_DEPTH_SCALES[3].toFixed(2)}×`);});+ it("offers the blur as the way to hide a private detail, and says images are not here (#1180)", () => {+ expect(TOOL_DESCRIPTIONS.addAnnotation).toMatch(/'blur'.*email/);+ expect(TOOL_DESCRIPTIONS.addAnnotation).toMatch(/Image annotations are not available/);+ expect(SYSTEM_PROMPT).toMatch(/privacy blur/);+ for (const name of ["addZoom", "setZoom"]) {+ expect(TOOL_DESCRIPTIONS[name]).toMatch(/hideCursor/);+ expect(TOOL_DESCRIPTIONS[name]).toMatch(/customScale/);+ }+ expect(TOOL_DESCRIPTIONS.setAudio).toContain(`${AUDIO_FADE_MAX_MS} ms`);+ });+it("tells the model a full-camera region needs a linked webcam", () => {expect(TOOL_DESCRIPTIONS.addCameraFullscreen).toMatch(/hasCameraTrack/);expect(SYSTEM_PROMPT).toMatch(/hasCameraTrack/);diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.tsindex 345721a60..7b05c0444 100644--- a/electron/ai-edition/deep-agent/service.ts+++ b/electron/ai-edition/deep-agent/service.ts@@ -19,12 +19,17 @@import { anthropicPromptCachingMiddleware, createAgent, tool } from "langchain";import { z } from "zod";+import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks";import type { AxcutDocument } from "../../../src/lib/ai-edition/schema";// ponytail: the legend is DERIVED from the depth→scale table, not retyped. Both// tool descriptions used to assert "depth 1–6 maps to 1.0×–3.5×" — a formula// (`depth/2 + 0.5`) that was purged from two rendering sites and survived here,// wrong at both ends of a table that actually runs 1.25× to 5.0×.-import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale";+import {+ MAX_ZOOM_SCALE,+ MIN_ZOOM_SCALE,+ ZOOM_DEPTH_LEGEND,+} from "../../../src/lib/ai-edition/timeline/zoom-scale";import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions";import {AGENT_EDITS_SETTING,@@ -126,7 +131,7 @@ const BASE_SYSTEM_PROMPT = ["- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.","- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.","- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.",- `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`,+ `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`,"- setAspectRatio picks the shape of the whole video, for where it will be shown; auto follows the recording.","- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.","- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.",@@ -169,12 +174,14 @@ export const SYSTEM_PROMPT = buildSystemPrompt({ editsAllowed: true });const ZOOM_CAMERA_NOTE ="rotationPreset is the zoom's 3D camera: `left` or `right` turns the screen in 3D to that side, `orbit` moves the camera around the screen while the zoom holds, `none` (the default) keeps it flat. It is a strong effect: keep it for a reveal or a key moment, not every zoom. getCurrentDocument reports it as rotationPreset, absent when flat.";+const ZOOM_BEHAVIOUR_NOTE = `The zoom pane's other settings: focusMode "manual" holds focus, "auto" follows the recorded pointer instead (getCurrentDocument reports the effective mode, auto everywhere while autoFocusAll is on); hideCursor hides the pointer while the zoom is on screen; customScale is a free level in place of depth, a factor from ${MIN_ZOOM_SCALE} to ${MAX_ZOOM_SCALE} rounded to the hundredth. A level the depth table has is stored as that depth, and one deeper than the clip takes before its recording blurs is refused with the limit, as the editor refuses it. Pass depth or customScale, not both.`;+const ZOOM_TRANSITIONS_NOTE = `A zoom HOLDS over its span: it animates in during the window just before startSec and out during the window just after endSec. Each move lasts 0.6 s + 0.55 s × ln(renderedScale) of SCREEN time (${ZOOM_TRANSITION_LEGEND}), so a speed region scales the timeline it covers by its speed: longer above 1×, shorter below. getCurrentDocument gives each zoom's zoomInFromSec and zoomOutUntilSec, in virtual seconds like the zoom. A trim inside either window cuts that move and the export jumps at the cut: leave the windows untrimmed. Two zooms at most 1.5 s apart on screen, on the same clip with no trim between them, are chained: the camera pans from one to the next over 1 s of screen time from the first one's end (panUntilSec), with no zoom-out or zoom-in between them.`;const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it carries \`cutTransitions\`: which zoom, which side, cutSec, the timeline seconds of the move lost, and by: trim, or junction when the move runs past its clip's edge into the next clip. It is a warning, nothing is refused: move the cut or the zoom unless a hard cut is what you want.`;export const TOOL_DESCRIPTIONS: Record<string, string> = {getCurrentDocument:- "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation effects (virtual, edited-timeline time). Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.",+ "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.",getTranscript:"Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.",getCursorTrack:@@ -195,27 +202,25 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {"Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.",replaceTimeline:"Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.",- addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`,+ addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`,addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry, and the same \`cutTransitions\` warning. ${ZOOM_TRANSITIONS_NOTE}`,- setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`,+ setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`,addSpeed:"Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down, from 0.25× to 16× (default 1.5×). Use to speed through slow stretches without cutting them.",setSpeed:"Move, resize, or change the multiplier of an existing speed region by id (virtual-timeline seconds). Only the fields you pass are changed.",addAnnotation:- "Add a text annotation over a span of the edited timeline (virtual seconds). It sits on the output frame, the whole picture padding included, as text added in the editor does: zooms and padding never move it. x/y place the text's centre in frame percentages (0–100, default the centre); the box is sized to the text and kept inside the frame. By default it is drawn on a dark plate so it reads on any footage; fontSize, color, background, bold and animation set its look. Use for callouts, labels and titles.",+ "Add an annotation over a span of the edited timeline (virtual seconds). type picks what it is: 'text' (the default) for callouts, labels and titles; 'figure' for an arrow pointing at something; 'blur' for a privacy mask that blurs or pixelates what is under it, the way to hide an email address, a token, a password field or any private detail the recording shows. Image annotations are not available here: they need a file, added in the editor. x/y place the centre, 0–100 percent of the annotation's box (default the centre). Text and arrows sit on the output frame, the whole picture padding included, as the editor places them: zooms and padding never move them. A blur sits on the recording's picture instead, so x/y/width/height are percent of the recording and it stays on what it hides when padding or a zoom moves the picture. A text box is sized to its words and kept inside the frame; it is drawn on a dark plate by default, and fontSize, color, background, bold and animation set its look. An arrow or a blur takes width/height (default 30 × 20) and its own data: figureData {arrowDirection, color, strokeWidth} for an arrow, blurData {type 'blur' or 'mosaic', shape 'rectangle' or 'oval', color, intensity, blockSize} for a blur. The result reads the annotation back as getCurrentDocument reports it.",setAnnotation:- "Change an existing annotation by id: its span (virtual-timeline seconds), its text, its centre (x/y, frame percentages) or its look (fontSize, color, background, bold, animation). Only the fields you pass are changed; a text box is refitted to its text around its centre. A new span that would run into another annotation is refused, naming it.",+ "Change an existing annotation by id: its span (virtual-timeline seconds), its type, its centre (x/y, percent of its box), its size (width/height, arrow and blur only), its text and look (text only: fontSize, color, background, bold, animation), or its figureData / blurData, of which only the keys you pass change. Only the fields you pass are changed. A type change converts it as the editor's type switch does: it stays where it is on screen, a text turned into an arrow or a blur takes the default box, and the text is kept to come back to. A text box is refitted to its text around its centre. Arguments that belong to another kind (width on a text, blurData on an arrow) are refused. A new span that would run into another annotation is refused, naming it.",setAspectRatio:"Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording. Annotations on the frame keep their place and shape.",addCameraFullscreen:"Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.",setCameraFullscreen:"Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.",- addAudio:- "Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.",- setAudio:- "Move, resize, re-level, re-lane, mute, loop or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.",+ addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`,+ setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`,removeTrim:"Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.",removeModifier:diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsxindex 17f335048..c0551227d 100644--- a/src/components/ai-edition/RightPanes.tsx+++ b/src/components/ai-edition/RightPanes.tsx@@ -57,7 +57,11 @@ import {} from "@/components/video-editor/types";import { useI18n, useScopedT } from "@/contexts/I18nContext";import { resolveCaptionLane } from "@/lib/ai-edition/captions/settings";-import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks";+import {+ AUDIO_FADE_MAX_MS,+ collapseTracksToPills,+ trackGroupId,+} from "@/lib/ai-edition/document/audioTracks";import {collectNativeFormats,formatFillAvailability,@@ -3482,11 +3486,6 @@ type TimelineApi = ReturnType<typeof useTimeline>;// with the file name, then the volume (a local live value during the drag,// committed as one undo step on release), then a delete button styled like the// region panes' (position and mute are edited on the lane itself).-// Longest fade the inspector offers. Past a few seconds a fade stops reading as-// a fade and starts reading as a level change, and the track's own span caps it-// anyway (`resolveFadeSecs` reduces one that does not fit).-const FADE_MAX_MS = 5000;-/*** Per-track controls for the selected imported audio track (issue #350). Shown by* the inspector in place of the facet when an audio track is selected (see@@ -3582,7 +3581,7 @@ export function AudioTrackPane({ tl, onClose }: { tl: TimelineApi; onClose?: ()label={ts("audioTrack.fadeIn")}value={liveFadeIn ?? track.fadeInMs}min={0}- max={FADE_MAX_MS}+ max={AUDIO_FADE_MAX_MS}step={50}decimals={0}suffix=" ms"@@ -3602,7 +3601,7 @@ export function AudioTrackPane({ tl, onClose }: { tl: TimelineApi; onClose?: ()label={ts("audioTrack.fadeOut")}value={liveFadeOut ?? track.fadeOutMs}min={0}- max={FADE_MAX_MS}+ max={AUDIO_FADE_MAX_MS}step={50}decimals={0}suffix=" ms"diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsxindex d8eea283a..96070498b 100644--- a/src/components/ai-edition/v4/FloatingInspector.tsx+++ b/src/components/ai-edition/v4/FloatingInspector.tsx@@ -61,6 +61,7 @@ import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings";import type { useTimeline } from "@/lib/ai-edition/store/useTimeline";import { formatSeconds } from "@/lib/ai-edition/timeline/format";import { coalescedTrimGroups } from "@/lib/ai-edition/timeline/trim-mapping";+import { zoomLevelFor } from "@/lib/ai-edition/timeline/zoom-scale";import { clampToBound } from "@/lib/projectDefaults";import { annotationFootageRect, zoomScaleLimit } from "@/native/sceneDescription";import { ColorField } from "../ColorField";@@ -414,8 +415,6 @@ const BLUR_DEFAULTS = {blockSize: 12,} as const;-const ZOOM_DEPTHS: readonly ZoomDepth[] = [1, 2, 3, 4, 5, 6];-// The row: the default and one step either side, plus a strong close-up. The two ends of the// table (1.25×, 5×) and every level between are one entry in the free field below, which is// why the row stays short. Labels read the table, not a formula: a formula once announced@@ -488,11 +487,11 @@ export function ZoomLevelControl({setRequested(scale);const gen = ++genRef.current;pendingRef.current.add(gen);- const depth = ZOOM_DEPTHS.find((d) => ZOOM_DEPTH_SCALES[d] === scale);+ const level = zoomLevelFor(scale);const write =- depth === undefined- ? tl.updateZoomCustomScale(region.id, scale)- : tl.updateZoomDepth(region.id, depth);+ "depth" in level+ ? tl.updateZoomDepth(region.id, level.depth)+ : tl.updateZoomCustomScale(region.id, level.customScale);// A refused write hands the level back to the document, so the same one can be retried.const settle = (ok: boolean) => {pendingRef.current.delete(gen);diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.tsindex 027f330da..83dd74a1e 100644--- a/src/lib/ai-edition/document/audioTracks.ts+++ b/src/lib/ai-edition/document/audioTracks.ts@@ -329,6 +329,13 @@ export function patchAudioTrack(};}+/**+ * Longest fade a track is given, by the inspector and the agent alike. Past a few seconds a fade+ * stops reading as a fade and starts reading as a level change, and the track's own span caps it+ * anyway (`resolveFadeSecs` reduces one that does not fit).+ */+export const AUDIO_FADE_MAX_MS = 5000;+/*** Fade lengths in seconds, reduced to fit inside `spanSec`.*diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.tsindex 4d8caf600..05edecab0 100644--- a/src/lib/ai-edition/schema/index.ts+++ b/src/lib/ai-edition/schema/index.ts@@ -390,7 +390,7 @@ export const timelineOperationSchema = z.discriminatedUnion("type", [// (AnnotationRegion / ZoomRegion) — duplicated here so the schema package has no// dependency on the React editor module.-const blurDataSchema = z+export const blurDataSchema = z.object({type: z.enum(["blur", "mosaic"]).default("mosaic"),shape: z.enum(["rectangle", "oval", "freehand"]).default("rectangle"),@@ -403,7 +403,7 @@ const blurDataSchema = z}).optional();-const figureDataSchema = z+export const figureDataSchema = z.object({arrowDirection: z.enum(["up", "down", "left", "right", "up-right", "up-left", "down-right", "down-left"])diff --git a/src/lib/ai-edition/timeline/zoom-scale.ts b/src/lib/ai-edition/timeline/zoom-scale.tsindex 2cc8b23bf..b1baea484 100644--- a/src/lib/ai-edition/timeline/zoom-scale.ts+++ b/src/lib/ai-edition/timeline/zoom-scale.ts@@ -59,6 +59,18 @@ export function effectiveZoomScale(region: ZoomScaleInput): number {return ZOOM_DEPTH_SCALES[region.depth];}+/**+ * A zoom level as the document stores it: a level the table has is written as its depth, so a+ * document keeps naming its presets, and anything else as a `customScale`. The inspector's level+ * control and the agent's zoom tools both write through this.+ */+export function zoomLevelFor(scale: number): { depth: ZoomDepth } | { customScale: number } {+ const depth = (Object.keys(ZOOM_DEPTH_SCALES).map(Number) as ZoomDepth[]).find(+ (d) => ZOOM_DEPTH_SCALES[d] === scale,+ );+ return depth === undefined ? { customScale: scale } : { depth };+}+/*** How far a recording may be blown up on the finished frame before it visibly blurs: output* pixels per source pixel. Crop and zoom multiply, so the zoom a clip can take depends on