Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
// Agent tool layer (P1.1/P1.2): the zod argument schemas — the single source of
// truth, shared with the LangChain `tool()`s built in `deep-agent/service.ts` —
// plus the executor that validates arguments and applies each tool against an
// AxcutDocument snapshot. Pure — no IPC, no fs. The chat-service tool loop owns
// checkpoints and persistence; this module only knows how to turn
// (document, toolName, argsJson) into a new document.
//
// Read tools return JSON the model can reason over; write tools return the
// mutated document plus a human-readable summary line for the chat panel
// ("applied: added trim 0:02.1 – 0:02.4").
//
// The PROSE the model reads lives in `TOOL_DESCRIPTIONS` (deep-agent/service.ts)
// — not here. This header used to claim the file held "the JSON-schema tool
// definitions fed to the LLM as tools[]", which stopped being true when the
// deep-agent landed and stayed on the page for a whole release; the specs it
// described are gone (see `MUTATING_TOOL_NAMES`).
import { z } from "zod";
import { DEFAULT_TEXT_PLATE } from "../../src/lib/ai-edition/annotations/background";
import {
convertAnnotationKind,
DEFAULT_ANNOTATION_BOX,
fitTextInFrame,
} from "../../src/lib/ai-edition/annotations/placement";
import {
AUDIO_FADE_MAX_MS,
collapseTracksToPills,
patchAudioTrack,
placeAudioTrackInDocument,
trackGroupId,
} from "../../src/lib/ai-edition/document/audioTracks";
import { createId } from "../../src/lib/ai-edition/document/ids";
import {
addLayoutChange,
applyLayoutSpan,
defaultLayoutPreset,
layoutRuns,
moveLayoutBoundary,
setLayoutPreset,
} from "../../src/lib/ai-edition/document/layout";
import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat";
import {
duplicateClip,
insertAssetClip,
moveClip,
planTimelineReplacement,
type RegionKind,
removeClip,
removeRegion,
replaceTimeline,
setClipCrop,
setClipSourceRange,
} from "../../src/lib/ai-edition/document/timeline";
import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript";
import {
type AxcutAnnotationRegion,
type AxcutDocument,
type AxcutShaderEffect,
type AxcutShaderEffectRange,
blurDataSchema,
clipCropRegionSchema,
figureDataSchema,
shaderEffectParamSchema,
shaderEffectSchema,
shaderEffectValueSchema,
} from "../../src/lib/ai-edition/schema";
import {
findShaderEffect,
SHADER_EFFECT_CONTRACT,
type ShaderEffectCheck,
type ShaderEffectCheckLookup,
shaderEffectCostAdvice,
storedShaderEffectValues,
} from "../../src/lib/ai-edition/shaderEffects";
import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings";
import { appendAutoZoomSuggestions } from "../../src/lib/ai-edition/timeline/apply-auto-zooms";
import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera";
import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts";
import {
buildCursorTrack,
type CursorTrackSample,
} from "../../src/lib/ai-edition/timeline/cursor-track";
import { breatheCut, mergeCloseCuts } from "../../src/lib/ai-edition/timeline/cut-breath";
import {
anchorRegionsWithDerivedMs,
coalesceRegionsForRuler,
replacePillSpan,
resolvePillIds,
} from "../../src/lib/ai-edition/timeline/timelineMap";
import {
trimAppliesToClip,
trimToTimelineSpan,
} from "../../src/lib/ai-edition/timeline/trim-mapping";
import {
effectiveZoomScale,
MAX_ZOOM_SCALE,
MIN_ZOOM_SCALE,
ZOOM_DEPTH_LEGEND,
zoomLevelFor,
} from "../../src/lib/ai-edition/timeline/zoom-scale";
import type { AutoZoomSuggestion } from "../../src/lib/ai-edition/timeline/zoom-suggestions";
import {
chainedPans,
transitionCutsMs,
zoomTransitions,
} from "../../src/lib/ai-edition/timeline/zoom-transitions";
import { LAYOUT_PRESETS } from "../../src/lib/layoutTypes";
import { SETTING_BOUNDS } from "../../src/lib/projectDefaults";
import type { SegmentationSupport } from "../../src/native/contracts";
import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription";
import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils";
import { CAPTION_TOOL_NAMES, executeCaptionTool, MUTATING_CAPTION_TOOLS } from "./caption-tools";
import { editorSettingsSnapshot, executeSetEditorSettings } from "./editor-settings-tool";
import { measureTextInShippedFonts } from "./text-measure";
export interface AgentToolExecution {
ok: boolean;
/** JSON payload returned to the model as the tool result. */
resultJson: string;
/** Updated document — only set when the tool mutated it. */
document?: AxcutDocument;
/** One-line human summary for the chat panel (mutating tools only). */
summary?: string;
}
function formatSec(sec: number): string {
if (!Number.isFinite(sec) || sec < 0) return "0:00.0";
const m = Math.floor(sec / 60);
const s = (sec % 60).toFixed(1);
return `${m}:${s.padStart(4, "0")}`;
}
function toMs(sec: number): number {
return Math.max(0, Math.round(sec * 1000));
}
type ModifierKind = Exclude<RegionKind, "trim">;
function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] {
const legacy = (document.legacyEditor as Record<string, unknown>) ?? {};
switch (kind) {
case "zoom":
return document.zoomRanges.map((region) => region.id);
case "annotation":
return document.annotations.map((region) => region.id);
case "speed":
return ((legacy.speedRegions as Array<{ id: string }> | undefined) ?? []).map(
(region) => region.id,
);
case "cameraFullscreen":
return ((legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? []).map(
(region) => region.id,
);
case "layout":
return layoutRuns(document).flatMap((run) => run.ids);
case "audio":
// Fragments of one user-visible track share `trackId` and render as a single pill,
// so the id that disappears is the group key once, not one per fragment.
// `removeRegion` applies the same grouping by routing this kind through
// `removeAudioTrack`.
return [...new Set(document.audioTracks.map(trackGroupId))];
case "shaderEffect":
return (document.shaderEffectRanges ?? []).map((region) => region.id);
}
}
// For the effect set* tools: keep the stored span unless the caller passes new
// edges, and normalise so start ≤ end. Input seconds are virtual-timeline time.
function resolveSpanMs(
existing: { startMs: number; endMs: number },
startSec: number | undefined,
endSec: number | undefined,
): { startMs: number; endMs: number } {
if (startSec === undefined && endSec === undefined) {
return { startMs: existing.startMs, endMs: existing.endMs };
}
const s = startSec ?? existing.startMs / 1000;
const e = endSec ?? existing.endMs / 1000;
return { startMs: toMs(Math.min(s, e)), endMs: toMs(Math.max(s, e)) };
}
/**
* For the effect set* tools: the region another span of the same kind would have to cross.
*
* `replacePillSpan` follows the editor's drag rule: a region dragged into a neighbour stops
* against it, part of the way back to where it started. That is right under a
* mouse, which shows the stop as it happens. For a caller who named an exact span it put the
* region somewhere it never asked for (asked 7.6 – 9.5 s, landed 1.5 – 3.3 s), so a set that
* would be stopped is refused instead, naming what is in the way. The neighbours are the same
* walls the drag rule sees: every other pill, look-alikes included (#1133), but not one the region
* already overlaps.
*/
function refuseIfBlocked<T extends { id: string; startMs: number; endMs: number }>(
kind: string,
regions: T[],
id: string,
span: { startMs: number; endMs: number },
): AgentToolExecution | null {
const pills = coalesceRegionsForRuler(regions);
const pill = pills.find((p) => p.ids.includes(id));
if (!pill) return null;
const start = span.startMs / 1000;
const end = span.endMs / 1000;
const wall = pills.find(
(other) =>
other !== pill &&
(other.end <= pill.start || other.start >= pill.end) &&
other.start < end &&
other.end > start,
);
if (!wall) return null;
return failure(
`The ${kind} was NOT changed: ${formatSec(start)} – ${formatSec(end)} would run into ${kind} ` +
`${wall.ids[0]} at ${formatSec(wall.start)} – ${formatSec(wall.end)}, and two ${kind}s ` +
"cannot overlap. Move or shorten that one first, or pick a span clear of it.",
);
}
type AnnotationLook = {
fontSize?: number;
color?: string;
background?: string;
bold?: boolean;
animation?: NonNullable<AxcutAnnotationRegion["style"]["textAnimation"]>;
};
/** The look arguments, written onto a style the way the editor's inspector writes them. */
function applyAnnotationLook(
style: AxcutAnnotationRegion["style"],
look: AnnotationLook,
): AxcutAnnotationRegion["style"] {
const next = { ...style };
if (look.fontSize !== undefined) next.fontSize = look.fontSize;
if (look.color !== undefined) next.color = look.color;
if (look.bold !== undefined) next.fontWeight = look.bold ? "bold" : "normal";
if (look.animation !== undefined) next.textAnimation = look.animation;
if (look.background !== undefined) {
if (look.background === "none") {
// As the inspector's switch does: keep the colour, so turning it back on restores it.
if (style.backgroundColor !== "transparent") next.lastBackgroundColor = style.backgroundColor;
next.backgroundColor = "transparent";
} else {
next.backgroundColor = look.background;
}
}
return next;
}
const round2 = (n: number) => Math.round(n * 100) / 100;
/**
* One annotation as the agent reads it back, in the terms its writes take: the centre (x, y) and
* size of its box, which box that is, and the look of its kind. A text reports the look arguments,
* an arrow its figureData and a blur its blurData, the defaults filled in as the renderer fills
* them. An image reports its box only: its content is a data URL, megabytes of nothing to read.
*/
function annotationForAgent(a: AxcutAnnotationRegion): Record<string, unknown> {
const geometry = {
box: a.space === "frame" ? "frame" : "footage",
x: round2(a.position.x + a.size.width / 2),
y: round2(a.position.y + a.size.height / 2),
width: round2(a.size.width),
height: round2(a.size.height),
};
switch (a.type) {
case "text":
return {
type: a.type,
text: a.content || a.textContent || "",
...geometry,
fontSize: a.style.fontSize,
color: a.style.color,
background: a.style.backgroundColor === "transparent" ? "none" : a.style.backgroundColor,
bold: a.style.fontWeight === "bold",
animation: a.style.textAnimation ?? "none",
};
case "figure":
return {
type: a.type,
...geometry,
figureData: figureDataSchema.unwrap().parse(a.figureData ?? {}),
};
case "blur":
return {
type: a.type,
...geometry,
blurData: blurDataSchema.unwrap().parse(a.blurData ?? {}),
};
default:
return { type: a.type, ...geometry };
}
}
/** The arguments that only mean something for one kind of annotation, refused on another. */
function refuseShapeMismatch(
type: AxcutAnnotationRegion["type"],
data: {
text?: string;
width?: number;
height?: number;
figureData?: unknown;
blurData?: { shape?: string };
},
): AgentToolExecution | null {
if (type === "text" && (data.width !== undefined || data.height !== undefined)) {
return failure("A text box is sized to its words: change fontSize, not width or height.");
}
if (type !== "text" && data.text !== undefined) {
return failure(`text is for a text annotation; this one is ${type}. Pass type "text" with it.`);
}
if (type !== "figure" && data.figureData !== undefined) {
return failure(`figureData is an arrow's; this annotation is ${type}. Pass type "figure".`);
}
if (type !== "blur" && data.blurData !== undefined) {
return failure(`blurData is a blur's; this annotation is ${type}. Pass type "blur".`);
}
// The editor no longer offers it either: its input was broken and it renders as its bounding
// box, and a privacy mask that is half reliable is worse than none.
if (data.blurData?.shape === "freehand") {
return failure('A freehand blur is not offered: use shape "rectangle" or "oval".');
}
return null;
}
/** The fields of `parsed` the caller actually sent: the schema fills in defaults, and a patch
* must not reset what it never named. */
function sentKeys<T extends object>(parsed: T | undefined, raw: unknown): Partial<T> {
if (!parsed || !raw || typeof raw !== "object") return {};
return Object.fromEntries(Object.entries(parsed).filter(([key]) => key in raw)) as Partial<T>;
}
/**
* An arrow's, a blur's or an image's box and data: `size` centred on `center` and kept inside its
* reference box, as the editor keeps one it moves; then the figureData or blurData the caller sent,
* over what the annotation had or the schema's defaults.
*/
function shapeAnnotation(
region: AxcutAnnotationRegion,
center: { x: number; y: number },
size: { width: number; height: number },
data: { figureData?: object; blurData?: object },
rawArgs: unknown,
): AxcutAnnotationRegion {
const raw = (rawArgs ?? {}) as { figureData?: unknown; blurData?: unknown };
const { width, height } = size;
return {
...region,
position: {
x: Math.min(Math.max(center.x - width / 2, 0), Math.max(0, 100 - width)),
y: Math.min(Math.max(center.y - height / 2, 0), Math.max(0, 100 - height)),
},
size: { width, height },
...(region.type === "figure"
? {
figureData: {
...figureDataSchema.unwrap().parse(region.figureData ?? {}),
...sentKeys(data.figureData, raw.figureData),
},
}
: {}),
...(region.type === "blur"
? {
blurData: {
...blurDataSchema.unwrap().parse(region.blurData ?? {}),
...sentKeys(data.blurData, raw.blurData),
},
}
: {}),
} as AxcutAnnotationRegion;
}
/**
* A text annotation on the output frame, its box sized to its text around `center` (frame
* percentages): what the editor's "add text" does, so the agent's text sits where the editor's
* would, is measured in the same fonts, and stays put when the picture zooms. A size too large for
* the frame's width is brought down to fit on its line, as a format change does.
*/
function placeTextOnFrame<T extends AxcutAnnotationRegion>(
region: T,
center: { x: number; y: number },
document: AxcutDocument,
): T {
const frameAspect = resolveAspectRatioValue(document, getEditorSettings(document).aspectRatio);
const centred = {
...region,
space: "frame" as const,
position: center,
size: { width: 0, height: 0 },
};
return fitTextInFrame(centred, frameAspect, measureTextInShippedFonts);
}
// v5: a modifier is stored as clip-anchored fragment(s), and adjacent regions with the
// same properties read as ONE pill (timelineMap merge rule). The agent reasons in VIRTUAL
// seconds over whole regions, so we present exactly the pills the user sees, keyed by the
// first region under each — which every set*/remove* tool accepts as the id.
function coalesceForAgent<T extends { id: string; startMs: number; endMs: number }>(
regions: T[],
annotations = false,
): T[] {
return coalesceRegionsForRuler(regions, 0.001, { annotations }).map((pill) => ({
...pill.member,
id: pill.ids[0],
startMs: Math.round(pill.start * 1000),
endMs: Math.round(pill.end * 1000),
}));
}
/** Every agent write anchors the region to the clip(s) it covers, exactly like the UI. */
function anchorForAgent<T extends { id: string; startMs: number; endMs: number }>(
region: T,
document: AxcutDocument,
prefix: string,
) {
return anchorRegionsWithDerivedMs([region], document.timeline.clips, () => createId(prefix));
}
// ─── What the write actually LANDED ────────────────────────────────────────
//
// ponytail: every add*/set* used to echo back the span it was ASKED for. Three
// ways that is a lie, all reproduced against the real fixtures:
// • CLAMP — `addZoom {20,40}` on a 24.70 s clip stores 20–24.704 and reported
// 20–40. Ventilation trims the span to the clip, silently.
// • FRAGMENTATION — `addZoom {25,35}` across two clips stores TWO fragments,
// the second with a brand-new id (`anchorRawRegionsToClips:74`), and the
// result named one id and one span.
// • NOTHING PLACED — `addZoom {90,95}` on a 24.70 s timeline covers no clip.
// `anchorRegionsWithDerivedMs` passes it through UNANCHORED by design
// (timelineMap.ts:360-375, so a v2 project with a zero-extent clip does not
// lose data), and it was stored, reported ok, and can never play.
// The document is the referee; the report is read back off it.
interface Landing {
/** Every id the write produced — more than one when it straddled a clip. */
ids: string[];
startSec: number;
endSec: number;
/** False when nothing covered a clip: the region cannot ever play. */
anchored: boolean;
fragments: number;
}
function overlapsAClip(
region: { startMs: number; endMs: number },
document: AxcutDocument,
): boolean {
const startSec = region.startMs / 1000;
const endSec = region.endMs / 1000;
return document.timeline.clips.some(
(c) => Math.min(endSec, c.timelineEndSec) - Math.max(startSec, c.timelineStartSec) > 0,
);
}
/**
* `anchored` is BOTH structural and positional on purpose. `replacePillSpan`
* re-anchors from `pill.member`, whose payload still carries the OLD `clipId`
* even when the new span covers nothing — so a `clipId` alone proves nothing
* about where the region ended up. The ruler overlap is what decides whether a
* viewer will ever see it.
*/
function landingOf(
regions: Array<{ id: string; startMs: number; endMs: number; clipId?: string }>,
document: AxcutDocument,
): Landing {
const starts = regions.map((r) => r.startMs);
const ends = regions.map((r) => r.endMs);
return {
ids: regions.map((r) => r.id),
startSec: regions.length ? Math.min(...starts) / 1000 : 0,
endSec: regions.length ? Math.max(...ends) / 1000 : 0,
anchored:
regions.length > 0 &&
regions.every((r) => typeof r.clipId === "string" && overlapsAClip(r, document)),
fragments: regions.length,
};
}
/** The regions a pill edit produced: everything in `after` that is not an
* untouched original. `replacePillSpan` re-ventilates the pill, so the count and
* the ids can both change under a caller that only passed one id. */
function landingAfterPillEdit<T extends { id: string; startMs: number; endMs: number }>(
before: T[],
after: T[],
pillIds: Set<string>,
document: AxcutDocument,
): Landing {
const untouched = new Set(before.filter((r) => !pillIds.has(r.id)).map((r) => r.id));
return landingOf(
after.filter((r) => !untouched.has(r.id)) as Array<T & { clipId?: string }>,
document,
);
}
/**
* Refuse a full-camera region on footage that carries no webcam.
*
* ponytail: `addCameraFullscreen` took a span and nothing else, and answered
* `ok: true` whether or not any camera existed anywhere in the project — it
* writes into `legacyEditor.cameraFullscreenRegions`, which the schema does not
* validate. A pair of scenarios identical but for a linked webcam produced
* identical turns, because the evidence really was identical. The snapshot now
* carries `hasCameraTrack`, and this is the other half: a region that can only
* render nothing is not written, and the refusal names the reason.
*/
function cameraUnderSpan(
document: AxcutDocument,
startSec: number,
endSec: number,
): { clips: number; withCamera: number } {
const covered = document.timeline.clips.filter(
(c) => Math.min(endSec, c.timelineEndSec) - Math.max(startSec, c.timelineStartSec) > 0,
);
return {
clips: covered.length,
withCamera: covered.filter(
(c) => document.assets.find((a) => a.id === c.assetId)?.cameraTrack != null,
).length,
};
}
function noCameraUnderSpan(
document: AxcutDocument,
startSec: number,
endSec: number,
): AgentToolExecution | null {
const coverage = cameraUnderSpan(document, startSec, endSec);
if (coverage.clips === 0 || coverage.withCamera > 0) return null;
const anywhere = hasAnyClipWithCamera(document.assets, document.timeline.clips);
return failure(
`No webcam is linked to the footage under ${startSec.toFixed(1)}–${endSec.toFixed(1)} s, ` +
"so a full-camera region there would render nothing and none was written. " +
(anywhere
? "Other clips in this project do carry a camera — check assets[].hasCameraTrack in getCurrentDocument and pick a span over one of those."
: "No asset in this project carries a cameraTrack at all (assets[].hasCameraTrack is false everywhere): this recording has no webcam. Tell the user instead of placing a region."),
);
}
/** The span the edited timeline actually occupies, for an actionable refusal. */
function editedExtentSec(document: AxcutDocument): { startSec: number; endSec: number } {
const clips = document.timeline.clips;
if (clips.length === 0) return { startSec: 0, endSec: 0 };
return {
startSec: Math.min(...clips.map((c) => c.timelineStartSec)),
endSec: Math.max(...clips.map((c) => c.timelineEndSec)),
};
}
/**
* Refuse rather than store a region that covers no clip. The message carries the
* real bounds so the model can retry once with a usable span instead of looping
* — there is no AbortSignal and no timeout on the product path.
*/
function coversNoClip(
kind: string,
requestedStartSec: number,
requestedEndSec: number,
document: AxcutDocument,
): AgentToolExecution {
const extent = editedExtentSec(document);
return failure(
`The span ${requestedStartSec.toFixed(1)}–${requestedEndSec.toFixed(1)} s covers no clip, ` +
`so no ${kind} was placed (it could never play). The edited timeline runs ` +
`${extent.startSec.toFixed(1)}–${extent.endSec.toFixed(1)} s. ` +
`Pick a span inside it, or place a clip there first.`,
);
}
// The landing fields shared by every add / set result, so the model reads the
// same three things everywhere: what it asked for, what it got, and whether the
// two differ. `clamped` and `fragments` are omitted when there is nothing to
// say — an unconditional `clamped: false` on every result trains a reader to
// stop looking at it.
function landingReport(
landing: Landing,
requestedStartSec: number,
requestedEndSec: number,
): Record<string, unknown> {
const clamped =
Math.abs(landing.startSec - requestedStartSec) > 0.001 ||
Math.abs(landing.endSec - requestedEndSec) > 0.001;
return {
startSec: landing.startSec,
endSec: landing.endSec,
ids: landing.ids,
...(clamped ? { clamped: true, requestedStartSec, requestedEndSec } : {}),
...(landing.fragments > 1 ? { fragments: landing.fragments } : {}),
};
}
function landingSuffix(
landing: Landing,
requestedStartSec: number,
requestedEndSec: number,
): string {
const parts: string[] = [];
if (
Math.abs(landing.startSec - requestedStartSec) > 0.001 ||
Math.abs(landing.endSec - requestedEndSec) > 0.001
) {
parts.push(
`clamped from ${formatSec(requestedStartSec)} – ${formatSec(requestedEndSec)} to fit the clips`,
);
}
if (landing.fragments > 1) parts.push(`split across ${landing.fragments} clips`);
return parts.length ? ` (${parts.join(", ")})` : "";
}
/** Ids of every modifier in the document, all four families at once — the basis
* for naming what a destructive edit took with it. */
function modifierIdsOf(document: AxcutDocument): string[] {
const legacy = (document.legacyEditor as Record<string, unknown>) ?? {};
const speedRegions = (legacy.speedRegions as Array<{ id: string }> | undefined) ?? [];
const cameraFullscreenRegions =
(legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? [];
return [
...document.zoomRanges.map((r) => r.id),
...(document.layoutRegions ?? []).map((r) => r.id),
...document.annotations.map((r) => r.id),
...speedRegions.map((r) => r.id),
...cameraFullscreenRegions.map((r) => r.id),
...(document.shaderEffectRanges ?? []).map((r) => r.id),
];
}
/** ponytail: `setClipRange` and `removeClip` both delete anchored modifiers as a
* side effect (`setClipSourceRange` drops the pills the new window no longer
* covers; `removeClip` takes everything anchored to the clip). Both used to
* report only what was asked — "trimmed clip to 0:00.0 – 0:03.0" while a zoom
* ceased to exist. Nothing in the result NAMED the casualty, which is what made
* "the zoom is preserved" an easy thing for a model to say. */
function droppedByEdit(before: AxcutDocument, after: AxcutDocument) {
const survivingModifiers = new Set(modifierIdsOf(after));
const survivingTrims = new Set(after.timeline.trimRanges.map((t) => t.id));
return {
droppedModifierIds: modifierIdsOf(before).filter((id) => !survivingModifiers.has(id)),
droppedTrimIds: before.timeline.trimRanges
.map((t) => t.id)
.filter((id) => !survivingTrims.has(id)),
};
}
// Zod arg schemas — the SINGLE source of truth for every tool's arguments. The executor
// below validates against them, and the deep-agent (LangChain) layer imports the same
// objects to build its `tool()`s, so the two can never advertise a different shape than the
// one we actually validate. (The primitives `secondsSchema`/`depthSchema`/`focusSchema` stay
// private — callers only ever need the composed `*Args`.)
const secondsSchema = z.number().finite().nonnegative();
/** Span given to an agent-placed audio track when the asset has no probed duration
* yet. Short on purpose: a wrong guess the user has to lengthen beats one that
* silently covers the whole programme. */
const DEFAULT_AGENT_AUDIO_SEC = 10;
export const addTrimArgs = z.object({
startSec: secondsSchema,
endSec: secondsSchema,
assetId: z.string().min(1).optional(),
// A cut belongs to ONE clip. Without this, a project where two clips draw from the same
// asset (a duplicated clip) cannot say which of them the model meant, and the cut lands
// on both. Resolved from the source range when the model omits it and only one clip
// matches; ambiguity is reported back rather than guessed.
clipId: z.string().min(1).optional(),
reason: z.string().default(""),
});
/**
* ponytail: the element schema is `addTrimArgs` itself, not a copy of it.
*
* A batch is N unitary calls and nothing else — same validation, same clip
* resolution, same refusal wording — so the two can never drift into meaning
* different things. A separate element schema would be one more place to forget
* `clipId` the next time the unitary one grows a field.
*
* The `union(…, unknown)` is what makes "each item stands or falls alone" true for
* MALFORMED items too, not just unplaceable ones. A bare `z.array(addTrimArgs)`
* rejects the whole call the moment one entry is bad — and it rejects it in
* LangChain, before `applyBatch` runs — so nine good cuts would be thrown away
* with the tenth and `refused[index]` could never name it. Advertising the
* union keeps the element shape in the JSON schema the model reads (it shows up
* as `anyOf: [addTrim, {}]`) while letting a bad entry through to the unitary
* executor, which refuses it by itself with the wording it always uses.
*
* No cap on the array. A half-hour recording has hundreds of silences, and the
* point of this tool is precisely that it should not have to guess how many are
* too many. Picking a number here would repeat the mistake `getTranscript` made
* with its 800.
*/
export const addTrimsArgs = z.object({
ranges: z.array(z.union([addTrimArgs, z.unknown()])).min(1),
});
export const removeWordsArgs = z.object({
assetId: z.string().min(1).optional(),
wordIds: z.array(z.string().min(1)).min(1),
});
export const setTrimArgs = z.object({
trimRangeId: z.string().min(1),
startSec: secondsSchema,
endSec: secondsSchema,
});
/** A crop the frame can hold: a non-empty rectangle inside it, in 0–1 fractions. */
const cropRegionArgs = clipCropRegionSchema.refine(
(c) => c.width > 0 && c.height > 0 && c.x + c.width <= 1 + 1e-6 && c.y + c.height <= 1 + 1e-6,
"cropRegion must be a non-empty rectangle inside the frame: width and height above 0, x + width and y + height at most 1.",
);
export const setClipRangeArgs = z.object({
clipId: z.string().min(1),
sourceStartSec: secondsSchema.optional(),
sourceEndSec: secondsSchema.optional(),
cropRegion: cropRegionArgs
.nullable()
.optional()
.describe(
"The part of the source frame this clip shows, as 0–1 fractions of it ({x, y, width, height}); null shows the whole frame again.",
),
});
/** `beforeClipId`, never an index, for the reason `moveClipArgs` gives. */
export const insertClipArgs = z.object({
assetId: z.string().min(1),
beforeClipId: z.string().min(1).nullish(),
sourceStartSec: secondsSchema.default(0),
sourceEndSec: secondsSchema.optional(),
});
export const duplicateClipArgs = z.object({
clipId: z.string().min(1),
});
export const addAutoZoomsArgs = z.object({});
export const setAspectRatioArgs = z.object({
aspectRatio: z
.string()
.describe(
`"auto", one of ${ASPECT_RATIO_PRESETS.join(", ")}, or a token from getCurrentDocument's settingsContext.originalAspectRatios.`,
),
});
export const replaceTimelineArgs = z.object({
intervals: z.array(z.object({ startSec: secondsSchema, endSec: secondsSchema })).min(1),
reason: z.string().default(""),
});
/**
* ponytail: `beforeClipId`, never an index.
*
* `moveClip` (document/timeline.ts) takes an `insertIndex` interpreted against
* the array AFTER the moved clip is removed — V4Timeline does the `-1` by hand
* at its call site, with a comment about it. That is a fine internal contract
* and a terrible one to hand a language model: off by one is silent here, and
* "put the demo first" would land it second. A neighbour's id has no such
* ambiguity. Null (or omitted) means last, which is the only position no
* existing clip can name.
*
* There is deliberately NO `reason` field: `reason` is the clip's label ("intro",
* "demo"), the only thing in the snapshot that lets the model tell two clips
* apart. A reorder must not be able to overwrite it.
*/
export const moveClipArgs = z.object({
clipId: z.string().min(1),
beforeClipId: z.string().min(1).nullish(),
});
export const getTranscriptArgs = z.object({
assetId: z.string().min(1).optional(),
});
// ponytail: an assetId, never a path. The model names a row of the document and
// the runtime resolves it to `asset.originalPath` behind the allow-list; letting
// it name a file instead would turn a read-only reporting tool into an arbitrary
// JSON reader on the user's disk.
export const getCursorTrackArgs = z.object({
assetId: z.string().min(1).optional(),
});
// Effects (zoom / speed / annotation) are authored in *virtual* (edited-
// timeline) seconds — the position on the ruler the user sees — unlike clips
// and trims, which are source-time. The executor converts to the stored ms.
const depthSchema = z.number().int().min(1).max(6);
const focusSchema = z.object({ cx: z.number().min(0).max(1), cy: z.number().min(0).max(1) });
/** The zoom's 3D camera, as the editor's "3D camera" control offers it: the screen turned left or
* right, the camera orbiting the screen, or `none` for a flat screen. Stored and read back
* (getCurrentDocument) as `rotationPreset`, absent when flat. */
const rotationPresetSchema = z.enum(["none", "left", "right", "orbit"]);
/** The inspector's focus mode: `manual` holds `focus`, `auto` follows the recorded pointer. */
const focusModeSchema = z.enum(["manual", "auto"]);
/** The inspector's free zoom level: a factor, refused outside the range it accepts. The clip's own
* limit (`zoomScaleLimit`) is checked once the zoom is placed, as the inspector checks it. */
const customScaleSchema = z
.number()
.min(MIN_ZOOM_SCALE)
.max(MAX_ZOOM_SCALE)
.describe(
`A free zoom level, as a factor (${MIN_ZOOM_SCALE}–${MAX_ZOOM_SCALE}, e.g. 2 for 2×), in place of depth. Refused past the clip's own limit, the deepest its recording takes before it blurs; the refusal names it.`,
);
export const addZoomArgs = z.object({
startSec: secondsSchema,
endSec: secondsSchema,
// No schema default: the MCP server, the in-app agent and addZooms all parse the arguments
// before the tool runs, and a default filled in there reads as a depth passed next to
// customScale. The handler applies 3 when neither level is given.
depth: depthSchema
.optional()
.describe("Defaults to 3 when neither depth nor customScale is given."),
focus: focusSchema.default({ cx: 0.5, cy: 0.5 }),
rotationPreset: rotationPresetSchema.default("none"),
focusMode: focusModeSchema.default("manual"),
hideCursor: z.boolean().default(false),
customScale: customScaleSchema.optional(),
});
/** A zoom with its 3D camera set: `none` drops the key, as a flat zoom never carries it. */
function withRotationPreset<Z extends { rotationPreset?: unknown }>(
zoom: Z,
preset: z.infer<typeof rotationPresetSchema>,
): Z {
const { rotationPreset: _previous, ...flat } = zoom;
return (preset === "none" ? flat : { ...flat, rotationPreset: preset }) as Z;
}
/** `customScale` as the inspector's typed level writes it: to the hundredth, and a level the depth
* table has as that depth (`zoomLevelFor`). Empty when the caller passed none. */
function zoomLevelArg(customScale: number | undefined): {
depth?: 1 | 2 | 3 | 4 | 5 | 6;
customScale?: number;
} {
return customScale === undefined ? {} : zoomLevelFor(Math.round(customScale * 100) / 100);
}
/** depth and customScale are two ways of saying one thing: both at once has no single meaning. */
function refuseTwoLevels(args: unknown): AgentToolExecution | null {
const raw = (args ?? {}) as { depth?: unknown; customScale?: unknown };
return raw.depth !== undefined && raw.customScale !== undefined
? failure("Pass depth or customScale, not both: they set the same zoom level.")
: null;
}
/**
* The inspector's bound on a typed level: the deepest the clip under the zoom takes before its
* recording blurs (`zoomScaleLimit`, the bound the scene applies), the tightest over every
* fragment. Refused rather than clamped, as the inspector refuses it, so the caller hears the limit.
*/
function refuseScalePastLimit(
next: AxcutDocument,
ids: string[],
customScale: number | undefined,
): AgentToolExecution | null {
if (customScale === undefined) return null;
const scale = Math.round(customScale * 100) / 100;
const limit = Math.min(...ids.map((id) => zoomScaleLimit(next, id)));
return scale > limit
? failure(
`The zoom was NOT written: customScale ${scale} is deeper than this clip takes (${limit}× at most), past which its recording blurs. Pick ${limit} or less.`,
)
: null;
}
/** focusMode, hideCursor and customScale as the snapshot reports them, for a write's result. */
function zoomBehaviourReport(
zoom: AxcutDocument["zoomRanges"][number],
document: AxcutDocument,
): Record<string, unknown> {
const autoFocusAll = (document.legacyEditor as Record<string, unknown> | null)?.autoFocusAll;
return {
focusMode: autoFocusAll === true ? "auto" : (zoom.focusMode ?? "manual"),
...(autoFocusAll === true && zoom.focusMode !== "auto"
? { focusModeOverriddenByAutoFocusAll: true }
: {}),
...(zoom.hideCursor ? { hideCursor: true } : {}),
...(zoom.customScale != null ? { customScale: zoom.customScale } : {}),
};
}
/** Same contract as `addTrimsArgs`: the element schema IS the unitary one, and
* it is advertised rather than enforced so a bad region is refused by itself. */
export const addZoomsArgs = z.object({
regions: z.array(z.union([addZoomArgs, z.unknown()])).min(1),
});
export const setZoomArgs = z.object({
zoomId: z.string().min(1),
startSec: secondsSchema.optional(),
endSec: secondsSchema.optional(),
depth: depthSchema.optional(),
focus: focusSchema.optional(),
rotationPreset: rotationPresetSchema.optional(),
focusMode: focusModeSchema.optional(),
hideCursor: z.boolean().optional(),
customScale: customScaleSchema.optional(),
});
// The bound every speed reader applies (`readSpeedRegions`): refused here, so the agent hears
// why instead of seeing its 100x quietly play at 16x.
const speedSchema = z
.number()
.min(SETTING_BOUNDS.playbackSpeed[0])
.max(SETTING_BOUNDS.playbackSpeed[1]);
export const addSpeedArgs = z.object({
startSec: secondsSchema,
endSec: secondsSchema,
speed: speedSchema.default(1.5),
});
export const setSpeedArgs = z.object({
speedId: z.string().min(1),
startSec: secondsSchema.optional(),
endSec: secondsSchema.optional(),
speed: speedSchema.optional(),
});
/** How a text annotation looks: the settings the editor's inspector offers for one. */
const annotationLookArgs = {
fontSize: z
.number()
.min(8)
.max(200)
.optional()
.describe("Text size; 32 is the editor's default, a title reads well at 56 to 80."),
color: z.string().optional().describe("Text colour, a CSS colour such as #ffffff."),
background: z
.string()
.optional()
.describe('Colour of the plate behind the text, or "none" for no plate.'),
bold: z.boolean().optional(),
animation: z
.enum(["none", "fade", "rise", "pop", "slide-left", "typewriter", "pulse"])
.optional()
.describe("How the text enters: typewriter types it out, pop springs it in, and so on."),
};
/** The kinds the inspector's type switch offers, image aside: an image needs a file read. */
const annotationTypeSchema = z
.enum(["text", "figure", "blur"])
.describe(
"text: words on the frame. figure: an arrow. blur: a privacy mask that blurs or pixelates what is under it, such as an email address or a token on screen.",
);
/** What an arrow or a blur has instead of a text's look: its box, and its own data. */
const annotationShapeArgs = {
width: z
.number()
.positive()
.max(100)
.optional()
.describe(
"Box width, in percent of the annotation's box: the output frame for an arrow, the recording's picture for a blur. Not for text, whose box is its words.",
),
height: z
.number()
.positive()
.max(100)
.optional()
.describe("Box height, in percent of the same box as width."),
figureData: figureDataSchema.describe("An arrow's direction, colour and stroke width."),
blurData: blurDataSchema.describe(
"A blur's look: type blur or mosaic, shape rectangle or oval, color of the mosaic grid, intensity, blockSize.",
),
};
export const addAnnotationArgs = z.object({
startSec: secondsSchema,
endSec: secondsSchema,
type: annotationTypeSchema.default("text"),
text: z.string().default(""),
x: z.number().min(0).max(100).default(50),
y: z.number().min(0).max(100).default(50),
...annotationLookArgs,
...annotationShapeArgs,
});
export const setAnnotationArgs = z.object({
annotationId: z.string().min(1),
startSec: secondsSchema.optional(),
endSec: secondsSchema.optional(),
type: annotationTypeSchema.optional(),
text: z.string().optional(),
x: z.number().min(0).max(100).optional(),
y: z.number().min(0).max(100).optional(),
...annotationLookArgs,
...annotationShapeArgs,
});
/** A track's fade, capped where the inspector's slider stops. */
const fadeMsSchema = z.number().int().min(0).max(AUDIO_FADE_MAX_MS);
export const addAudioArgs = z.object({
assetId: z.string().min(1),
startSec: secondsSchema,
endSec: secondsSchema.optional(),
kind: z.enum(["voiceover", "music"]).default("music"),
offsetSec: secondsSchema.default(0),
gainDb: z.number().min(-60).max(12).default(0),
fadeInMs: fadeMsSchema.default(0),
fadeOutMs: fadeMsSchema.default(0),
});
export const setAudioArgs = z.object({
audioId: z.string().min(1),
startSec: secondsSchema.optional(),
endSec: secondsSchema.optional(),
kind: z.enum(["voiceover", "music"]).optional(),
offsetSec: secondsSchema.optional(),
gainDb: z.number().min(-60).max(12).optional(),
muted: z.boolean().optional(),
loop: z.boolean().optional(),
fadeInMs: fadeMsSchema.optional(),
fadeOutMs: fadeMsSchema.optional(),
});
export const addCameraFullscreenArgs = z.object({
startSec: secondsSchema,
endSec: secondsSchema,
});
const effectTargetSchema = z.enum(["image", "text", "both"]);
const effectValuesSchema = z.record(z.string(), shaderEffectValueSchema);
export const addEffectArgs = z.object({
startSec: secondsSchema,
endSec: secondsSchema,
effectId: z.string().min(1).optional(),
effect: z
.object({
name: z.string().min(1).max(60),
description: z.string().max(500),
code: z.string().min(1).max(65_536),
params: z.array(shaderEffectParamSchema).max(32).optional(),
})
.optional(),
values: effectValuesSchema.optional(),
target: effectTargetSchema.optional(),
});
export const setEffectArgs = z.object({
effectRegionId: z.string().min(1),
startSec: secondsSchema.optional(),
endSec: secondsSchema.optional(),
values: effectValuesSchema.optional(),
target: effectTargetSchema.optional(),
});
export const setEffectDefinitionArgs = z.object({
effectId: z.string().min(1),
name: z.string().min(1).max(60).optional(),
description: z.string().max(500).optional(),
code: z.string().min(1).max(65_536).optional(),
params: z.array(shaderEffectParamSchema).max(32).optional(),
});
export const getEffectArgs = z.object({
effectId: z.string().min(1),
});
export const setCameraFullscreenArgs = z.object({
cameraFullscreenId: z.string().min(1),
startSec: secondsSchema.optional(),
endSec: secondsSchema.optional(),
});
export const getTranscriptWordsArgs = z.object({
assetId: z.string().min(1).optional(),
startSec: secondsSchema.optional(),
endSec: secondsSchema.optional(),
});
export const setWordTextArgs = z.object({
wordId: z.string().min(1),
text: z.string(),
assetId: z.string().min(1).optional(),
});
export const removeTrimArgs = z.object({
trimRangeId: z.string().min(1),
});
export const addLayoutChangeArgs = z.object({
timeSec: secondsSchema,
preset: z.enum(LAYOUT_PRESETS),
});
export const setLayoutPresetArgs = z.object({
layoutId: z.string().min(1),
preset: z.enum(LAYOUT_PRESETS),
});
export const moveLayoutBoundaryArgs = z.object({
layoutId: z.string().min(1),
timeSec: secondsSchema,
});
export const removeModifierArgs = z.object({
id: z.string().min(1),
});
export const removeClipArgs = z.object({
clipId: z.string().min(1),
});
/**
* Every tool the model is handed, in the order `buildTools` builds them.
*
* The roster lives here, beside `MUTATING_TOOL_NAMES`, rather than in
* `deep-agent/service.ts` where `buildTools` is: the workbench needs to name the
* surface from its L0 layer, and importing the service would drag LangChain into
* a layer that deliberately runs on zod and pure document helpers alone.
*
* It is hand-written — the schemas differ per tool, so nothing can generate it —
* but it is not free-floating: `deep-agent/service.test.ts` asserts it equals
* `buildTools(...).map(t => t.name)`, and that test runs in CI. Adding a tool
* without adding it here fails the suite.
*
* ponytail: there used to be two more copies of this list, one in that test and
* one in `workbench/lib/prompts.ts`, neither derived from anything. The
* workbench's copy sat at 19 entries from the day it was written while the agent
* grew to 21 (`addTrims`/`addZooms`, commit 560d368e). Nothing caught it,
* because `npm run wb` is not part of CI — so the bench asserted a surface the
* product had not had for some time.
*/
export const OPENSCREEN_TOOL_NAMES = [
"getCurrentDocument",
"getTranscript",
"getTranscriptWords",
"getCursorTrack",
"setWordText",
"removeWords",
"addTrim",
"addTrims",
"setTrim",
"setClipRange",
"moveClip",
"insertClip",
"duplicateClip",
"replaceTimeline",
"addZoom",
"addZooms",
"addAutoZooms",
"setZoom",
"addSpeed",
"setSpeed",
"addAnnotation",
"setAnnotation",
"addLayoutChange",
"setLayoutPreset",
"moveLayoutBoundary",
"addEffect",
"setEffect",
"setEffectDefinition",
"getEffect",
"addAudio",
"setAudio",
"removeTrim",
"removeModifier",
"removeClip",
"setAspectRatio",
"setEditorSettings",
...CAPTION_TOOL_NAMES,
] as const;
/**
* The tools `createDeepAgent` used to inject on top of ours, over an in-memory
* backend that was EMPTY and that the model was not told was empty — the
* mechanical cause of D1, where the agent ran `ls`/`glob` against that sandbox
* and reported in good faith that the project held no cursor telemetry.
*
* The surface is gone, so this is no longer "tools we also get": it is the list
* of names that must never appear again. A call to one of them now means the
* model is hallucinating a filesystem it was never offered, which is a rarer but
* still exact D1 tell — which is why the workbench scores it as well as pinning
* it here.
*
* `execute` is included even though it vanished at runtime: it is in the
* middleware's list too and only disappeared because the default backend is not
* a sandbox. A sandbox backend would have made it a 26th tool. The workbench's
* own copy of this list omitted it, so `isPhantomTool` could not flag the one
* name a sandbox backend would have brought back.
*/
export const PHANTOM_TOOL_NAMES = [
"ls",
"read_file",
"write_file",
"edit_file",
"glob",
"grep",
"execute",
"write_todos",
"task",
] as const;
/**
* The tools that change the document. A LIST, not an inference: it gates the
* checkpoint the chat-service takes before a write and the mutating/non-mutating
* split the workbench scores its DSL axis on, and neither should quietly change
* because someone edited a switch case.
*
* ponytail: this replaces `AGENT_TOOL_SPECS`, ~300 lines of JSON schema whose
* own comment said "sent verbatim to the provider". It has not been sent
* anywhere since the deep-agent landed: the model receives the zod schemas
* built in `deep-agent/service.ts` and the prose in `TOOL_DESCRIPTIONS`. Two
* descriptions of the same tools, only one of them reaching the model — and it
* was the other one that humans read and kept up to date. The surviving
* documentation duty is `TOOL_DESCRIPTIONS`; `service.test.ts` pins the three
* remaining surfaces (descriptions, built tools, executor cases) to each other.
*/
export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set([
// Writes the transcript, not the timeline — but it writes the document, so it is a
// consented edit like any other.
"setWordText",
"removeWords",
"addTrim",
"addTrims",
"addZooms",
"setTrim",
"setClipRange",
"moveClip",
"insertClip",
"duplicateClip",
"replaceTimeline",
"addZoom",
"addAutoZooms",
"setZoom",
"addSpeed",
"setSpeed",
"addAnnotation",
"setAnnotation",
"addLayoutChange",
"setLayoutPreset",
"moveLayoutBoundary",
"addCameraFullscreen",
"setCameraFullscreen",
"addEffect",
"setEffect",
"setEffectDefinition",
"addAudio",
"setAudio",
"removeTrim",
"removeModifier",
"removeClip",
"setAspectRatio",
"setEditorSettings",
...MUTATING_CAPTION_TOOLS,
]);
export function isMutatingTool(name: string): boolean {
return MUTATING_TOOL_NAMES.has(name);
}
function roundSec(ms: number): number {
return Math.round(ms) / 1000;
}
// Each zoom pill's camera moves in virtual ms, and how much of each a cut takes off (#1028).
// The pill shows the hold only: the move in runs before it, the move out after it, and a
// trim or a clip junction over either makes the export jump at the cut.
function zoomTransitionsForAgent(document: AxcutDocument) {
const legacy = document.legacyEditor as Record<string, unknown> | null;
const speedRegions =
(legacy?.speedRegions as
| Array<{ id: string; startMs: number; endMs: number; speed: number }>
| undefined) ?? [];
// A trim with no clipId cuts every clip of its asset in playback, while the ruler maps it
// through one: measured per clip, so a zoom on any of them learns of the cut.
const trims = document.timeline.trimRanges.flatMap((trim) => {
const clips =
trim.clipId !== undefined
? [trim]
: document.timeline.clips
.filter((clip) => clip.assetId === trim.assetId)
.map((clip) => ({ ...trim, clipId: clip.id }));
return clips.flatMap((anchored) => {
const span = trimToTimelineSpan(anchored, document.timeline.clips);
return span ? [{ startMs: span.start * 1000, endMs: span.end * 1000 }] : [];
});
});
// Chained zooms pan from one to the next instead: no zoom-out, no zoom-in, nothing to cut
// (a trim or a junction between them would have broken the chain). A junction is where a
// clip starts after another: the timeline's own end has nothing after it to jump to.
const zooms = coalesceForAgent(document.zoomRanges);
const junctions = document.timeline.clips
.filter((clip) => clip.timelineStartSec > 0)
.map((clip) => ({
startMs: clip.timelineStartSec * 1000,
endMs: clip.timelineStartSec * 1000,
}));
const pans = chainedPans(zooms, speedRegions, [...trims, ...junctions]);
return zooms.map((zoom, i) => {
const transitions = zoomTransitions({ ...zoom, scale: effectiveZoomScale(zoom) }, speedRegions);
const panOut = pans.find((p) => p.from === i);
const panIn = pans.find((p) => p.to === i);
const cut = transitionCutsMs(zoom, transitions, trims, junctions);
return {
zoom,
transitions,
cut: { ...cut, inMs: panIn ? 0 : cut.inMs, outMs: panOut ? 0 : cut.outMs },
panTo: panOut && { zoomId: zooms[panOut.to].id, untilMs: panOut.untilMs },
panFromZoomId: panIn && zooms[panIn.from].id,
};
});
}
/** The transitions this edit cut that were not cut, or not this way, before it: the warning a
* zoom or trim write carries, like `cursorAnchor`: reported, never refused. */
function cutTransitionsReport(before: AxcutDocument, after: AxcutDocument) {
const cutsOf = (document: AxcutDocument) =>
zoomTransitionsForAgent(document).flatMap(({ zoom, cut }) =>
(["in", "out"] as const)
.map((side) => ({
zoomId: zoom.id,
side,
cutSec: roundSec(side === "in" ? cut.inMs : cut.outMs),
by: (side === "in" ? cut.inByJunction : cut.outByJunction) ? "junction" : "trim",
key: `${zoom.id}:${side}:${zoom.startMs}:${zoom.endMs}`,
}))
.filter((c) => c.cutSec > 0),
);
const known = new Set(cutsOf(before).map((c) => `${c.key}:${c.cutSec}`));
const cuts = cutsOf(after)
.filter((c) => !known.has(`${c.key}:${c.cutSec}`))
.map(({ key: _key, ...c }) => c);
return cuts.length ? { cutTransitions: cuts } : {};
}
/** The moves a trim (or, with `byJunction`, a clip junction) cuts: in, out, both or none. */
function cutSides(cut: ReturnType<typeof transitionCutsMs>, byJunction: boolean) {
const cutIn = cut.inMs > 0 && cut.inByJunction === byJunction;
const cutOut = cut.outMs > 0 && cut.outByJunction === byJunction;
return cutIn ? (cutOut ? "both" : "in") : cutOut ? "out" : undefined;
}
// Compact projection of the document for the model: everything it needs to
// reference ids and times, nothing it doesn't (no waveform paths, no history).
//
// Three clearly-separated groups, each with its OWN time-base spelled out so the
// model never has to guess:
// • clips — arranged segments; source-time in/out + their timeline position.
// • trims — source-time cuts inside a clip (do not split the clip).
// • effects — zoom / speed / annotation, in *virtual* (edited-timeline)
// seconds, i.e. positions on the ruler after clips + trims are applied.
//
// ponytail: what a projection LEAVES OUT is a claim too, and two omissions here
// were being read by the model as facts about the project.
// • `cameraTrack` — the assets went out as `{id, label, durationSec}`, so two
// projects identical except for a linked webcam were literally
// indistinguishable. `addCameraFullscreen` answered ok either way, and the
// model placed a full-camera region on a project with no camera and
// reported it done. `hasCameraTrack` is the exact parity of `hasTranscript`
// below, which had already solved the same problem for the transcript.
// • the zoom's real strength — `depth` went out bare. It is an ORDINAL, so a
// reader turns "3" into "3×" while the frame renders 1.80×; and when a
// migrated v1.7 project carries `customScale`, `depth` is inert and nothing
// said so. `renderedScale` is `effectiveZoomScale`, the renderer's own
// function, so the number the model reports is the number the viewer sees.
export function documentSnapshotForModel(
document: AxcutDocument,
cursorTelemetry?: CursorTelemetryContext,
): Record<string, unknown> {
const availability = cursorTelemetry?.availableByAssetId;
const legacy = document.legacyEditor as Record<string, unknown> | null;
const speedRegions =
(legacy?.speedRegions as
| Array<{ id: string; startMs: number; endMs: number; speed: number }>
| undefined) ?? [];
const cameraFullscreenRegions =
(legacy?.cameraFullscreenRegions as
| Array<{ id: string; startMs: number; endMs: number }>
| undefined) ?? [];
// The global Auto-Focus toggle OVERRIDES each region's own focusMode at
// render (sceneDescription.ts:703) and the inspector disables the per-region
// control while it is on. Reporting the stored mode would hand the model a
// second thing to be confidently wrong about, so the projection reports the
// EFFECTIVE mode and the flag that decides it.
const autoFocusAll = legacy?.autoFocusAll === true;
return {
timeBaseNote:
"clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions, effects and audioTracks are in virtual (edited-timeline) seconds: the editor's ruler, where a trim leaves a hole instead of shortening it and a speed region stretches nothing. The exported video's clock skips those holes and plays sped-up spans faster, so it differs from the ruler once there are any.",
audioNote:
"audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio': if the project has none, a file has to be imported first (importMedia, where it is offered), so never invent an id.",
zoomNote:
`renderedScale is what the viewer sees (depth is an ordinal, not a factor: ${ZOOM_DEPTH_LEGEND}). ` +
"When a zoom carries customScale it wins over depth and depthIsOverridden is true — " +
"a setZoom that only changes depth on such a zoom clears customScale so the depth takes effect. " +
"startSec–endSec is where the zoom HOLDS: it animates in over zoomInFromSec–startSec and out over " +
"endSec–zoomOutUntilSec (transitionSec of screen time each; a speed region scales that on the " +
"timeline, longer above 1x and shorter below, so read the bounds rather than adding transitionSec). cutByTrim names the move a trim cuts into (in, out or both), cutByJunction the move a clip junction cuts (the clip ends before the move does): the export jumps at that cut. " +
"Two zooms at most 1.5 s apart on screen, with no trim or clip junction between them, are CHAINED: " +
"no zoom-out and back in, the camera pans from the first (panToZoomId, panUntilSec: 1 s of screen " +
"time from its endSec, which can run into the next span) to the second (panFromZoomId). " +
"focusMode auto follows the recorded pointer instead of holding focus; hideCursor: true hides the pointer while the zoom is on screen (absent, it shows).",
annotationNote:
"Each annotation reports its box the way addAnnotation and setAnnotation take it: x/y its centre, width/height its size, in percent of `box`. box 'frame' is the output frame (text and arrows, which padding and zooms never move); box 'footage' is the recording's picture (a blur, which stays on what it hides when the picture moves). A text also reports its look (fontSize, color, background, bold, animation), an arrow (type 'figure') its figureData, a blur its blurData.",
project: { id: document.project.id, title: document.project.title },
layoutSegments: layoutRuns(document).map((r) => ({
id: r.ids[0],
startSec: r.start,
endSec: r.end,
preset: r.member.preset,
})),
layoutNote:
"layoutSegments use RAW document-timeline seconds, matching addLayoutChange.timeSec and moveLayoutBoundary.timeSec. Layouts form a continuous partition. A change lasts until the next one. Removing a change extends the preceding layout. Camera presets require linked webcam footage (assets[].hasCameraTrack).",
primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null,
autoFocusAll,
hasAnyCamera: hasAnyClipWithCamera(document.assets, document.timeline.clips),
cursorNote:
"assets[].hasCursorTelemetry says whether recorded pointer telemetry exists for that " +
"asset. true — call getCursorTrack to read the recorded pointer track. " +
"false — this asset was checked and has none (imported footage, or a recording made " +
"without the cursor recorder). null — it was NOT checked from here; say that, and do " +
"not report it as the project having no cursor data.",
assets: document.assets.map((a) => ({
id: a.id,
label: a.label,
// "audio" is an imported voiceover / music file: it is never a clip, it is
// played by an audio track. Without this the model sees an asset it cannot
// explain and tries to place it on the timeline as footage.
kind: a.kind,
durationSec: a.durationSec ?? null,
hasCameraTrack: a.cameraTrack != null,
cameraVisible: a.cameraTrack?.visible ?? false,
// Three-valued on purpose (see `CursorTelemetryContext`): `null` is
// "not checked", and it must never render as `false`. The whole defect
// was a runtime that could not look being read as a project that has
// nothing — same field, same three states, one honest projection.
//
// `?? null`, not `?? false`: an asset MISSING from the map is one whose
// probe threw. Defaulting that to `false` would put our failure back in
// the answer as their fact, one layer lower down.
hasCursorTelemetry: availability?.[a.id] ?? null,
// Whether getTranscriptWords can read it; transcribe makes one when it is false.
hasTranscript: document.transcripts.some((t) => t.assetId === a.id),
// A verdict a past run left on the media ("no-audio", "unsupported-audio").
...(a.transcriptionFailure
? {
transcriptionFailure: {
kind: a.transcriptionFailure.kind,
message: a.transcriptionFailure.message,
},
}
: {}),
})),
// ponytail: `index`, `reason` and `origin` are here because without them a
// clip cannot be DESIGNATED. "Put the demo first" is unanswerable when the
// only handles are `clip_1`/`clip_2` and two indistinguishable source
// windows — the label the user sees lives in `reason`, and it was not being
// sent. A reorder tool without this is a tool the model cannot aim.
clips: document.timeline.clips.map((c, index) => ({
id: c.id,
index,
assetId: c.assetId,
reason: c.reason,
origin: c.origin,
sourceStartSec: c.sourceStartSec,
sourceEndSec: c.sourceEndSec ?? null,
timelineStartSec: c.timelineStartSec,
timelineEndSec: c.timelineEndSec,
// Only when set: absent is the whole frame.
...(c.cropRegion ? { cropRegion: c.cropRegion } : {}),
})),
trimRanges: document.timeline.trimRanges.map((s) => ({
id: s.id,
assetId: s.assetId,
// The clip the cut is on — the only thing separating two cuts over the same
// media. `null` is a pre-v7 cut that still applies to every clip of its asset.
clipId: s.clipId ?? null,
startSec: s.startSec,
endSec: s.endSec,
reason: s.reason,
})),
zoomRanges: zoomTransitionsForAgent(document).map(
({ zoom: z, transitions, cut, panTo, panFromZoomId }) => ({
id: z.id,
startSec: roundSec(z.startMs),
endSec: roundSec(z.endMs),
...(panFromZoomId ? { panFromZoomId } : { zoomInFromSec: roundSec(transitions.inFromMs) }),
...(panTo
? { panToZoomId: panTo.zoomId, panUntilSec: roundSec(panTo.untilMs) }
: { zoomOutUntilSec: roundSec(transitions.outUntilMs) }),
transitionSec: roundSec(transitions.durationMs),
...(cutSides(cut, false) ? { cutByTrim: cutSides(cut, false) } : {}),
...(cutSides(cut, true) ? { cutByJunction: cutSides(cut, true) } : {}),
depth: z.depth,
renderedScale: effectiveZoomScale(z),
// Emitted only when set: an unconditional `customScale: null` on every
// zoom of every snapshot is noise the reader learns to skip, which is
// how the field would go unnoticed again.
...(z.customScale != null ? { customScale: z.customScale, depthIsOverridden: true } : {}),
...(z.rotationPreset ? { rotationPreset: z.rotationPreset } : {}),
focus: z.focus,
focusMode: autoFocusAll ? "auto" : (z.focusMode ?? "manual"),
...(z.hideCursor ? { hideCursor: true } : {}),
source: z.source ?? "manual",
}),
),
speedRegions: coalesceForAgent(speedRegions).map((s) => ({
id: s.id,
startSec: roundSec(s.startMs),
endSec: roundSec(s.endMs),
speed: s.speed,
})),
annotations: coalesceForAgent(document.annotations, true).map((a) => ({
id: a.id,
startSec: roundSec(a.startMs),
endSec: roundSec(a.endMs),
...annotationForAgent(a),
})),
cameraFullscreenRegions: coalesceForAgent(cameraFullscreenRegions).map((c) => ({
id: c.id,
startSec: roundSec(c.startMs),
endSec: roundSec(c.endMs),
})),
// Where each effect is posed, and how. The code is not here: getEffect reads it.
effects: coalesceForAgent(document.shaderEffectRanges ?? []).map((r) => ({
id: r.id,
effectId: r.effectId,
name: findShaderEffect(document, r.effectId)?.name ?? null,
startSec: roundSec(r.startMs),
endSec: roundSec(r.endMs),
target: r.target,
values: r.values,
})),
effectDefinitions: (document.shaderEffects ?? []).map((e) => ({
effectId: e.id,
name: e.name,
description: e.description,
contract: e.contract,
lines: e.code.split("\n").length,
regions: coalesceForAgent(
(document.shaderEffectRanges ?? []).filter((r) => r.effectId === e.id),
).length,
cost: e.cost ?? null,
params: e.params,
})),
// Imported audio, collapsed to the pills the ruler draws — a track ventilated
// across a clip boundary is several fragments the user sees as one thing, and
// the model has to name what the user sees.
audioTracks: collapseTracksToPills(document.audioTracks).map((t) => ({
id: trackGroupId(t),
startSec: roundSec(t.startMs),
endSec: roundSec(t.endMs),
assetId: t.assetId,
// Which lane it sits on. Also decides whether it is transcribed at all.
kind: t.kind,
// Where in the FILE the track starts playing, in that file's own seconds.
offsetSec: roundSec(t.offsetMs),
gainDb: t.gainDb,
muted: t.muted,
loop: t.loop,
fadeInMs: t.fadeInMs,
fadeOutMs: t.fadeOutMs,
})),
hasTranscript: document.transcripts.length > 0 || document.transcript !== null,
// The names and terms Whisper is primed with on every transcription (setVocabulary).
vocabulary: document.project.vocabulary ?? null,
};
}
/** A definition as it will be stored: schema-checked, then compiled by the compositor when one is
* wired, its cost cached. An error reads as the agent must act on it. */
function checkedEffectDefinition(
candidate: unknown,
checker: ShaderEffectCheckLookup | undefined,
): { definition: AxcutShaderEffect; check: ShaderEffectCheck | null } | { error: string } {
const parsed = shaderEffectSchema.safeParse(candidate);
if (!parsed.success) {
return {
error: parsed.error.issues
.map((i) => `${i.path.join(".") || "effect"}: ${i.message}`)
.join("; "),
};
}
const definition = parsed.data;
const check = checker?.(definition.code, definition.params) ?? null;
if (check?.error) return { error: `The effect does not compile:\n${check.error}` };
const cost =
check?.tier && typeof check.frameReads === "number"
? { tier: check.tier, frameReads: check.frameReads }
: undefined;
return { definition: { ...definition, cost }, check };
}
type ShaderEffectSource = {
code: string;
params: Array<{ name: string; type: AxcutShaderEffect["params"][number]["type"] }>;
};
/** The code and parameter layout `addEffect` or `setEffectDefinition` will have the compositor
* check, read from the same arguments the executor reads; `null` for a call that compiles nothing.
* The check is asynchronous (a Metal compile takes 120 ms and more) and the executor is not: the
* async wrapper asks first, then hands the answer over (`shaderEffectCheckLookup`). */
export function shaderEffectToCheck(
document: AxcutDocument,
name: string,
args: unknown,
): ShaderEffectSource | null {
const layout = (
params: ReadonlyArray<Pick<AxcutShaderEffect["params"][number], "name" | "type">>,
) => params.map(({ name, type }) => ({ name, type }));
if (name === "addEffect") {
const parsed = addEffectArgs.safeParse(args);
if (!parsed.success || !parsed.data.effect) return null;
return { code: parsed.data.effect.code, params: layout(parsed.data.effect.params ?? []) };
}
if (name === "setEffectDefinition") {
const parsed = setEffectDefinitionArgs.safeParse(args);
if (!parsed.success) return null;
const { effectId, code, params } = parsed.data;
const existing = findShaderEffect(document, effectId);
if (!existing || (code === undefined && params === undefined)) return null;
return { code: code ?? existing.code, params: layout(params ?? existing.params) };
}
return null;
}
/** The executor's view of an answer asked for `source`: that answer for the same code and
* layout, `null` (not checked) for anything else. */
export function shaderEffectCheckLookup(
source: ShaderEffectSource,
check: ShaderEffectCheck | null,
): ShaderEffectCheckLookup {
return (code, params) =>
code === source.code &&
params.length === source.params.length &&
params.every((p, i) => p.name === source.params[i].name && p.type === source.params[i].type)
? check
: null;
}
/** One effect at a time: the refusal names the one in the way, and the way out. */
function effectBlocked(
ranges: AxcutShaderEffectRange[],
startMs: number,
endMs: number,
except: ReadonlySet<string> = new Set(),
): string | null {
const wall = coalesceRegionsForRuler(ranges.filter((r) => !except.has(r.id))).find(
(pill) => pill.start * 1000 < endMs && pill.end * 1000 > startMs,
);
if (!wall) return null;
return (
`${formatSec(startMs / 1000)} – ${formatSec(endMs / 1000)} would run into effect ${wall.ids[0]} ` +
`at ${formatSec(wall.start)} – ${formatSec(wall.end)}, and one effect runs at a time. To combine ` +
"two looks, write them in one shader (setEffectDefinition); otherwise pick a span clear of it."
);
}
/** What an effect costs: its tier, and a recommendation when the tier runs longer than it is
* meant to. */
function effectCostReport(definition: AxcutShaderEffect, startSec: number, endSec: number) {
if (!definition.cost) return { cost: null };
const advice = shaderEffectCostAdvice(definition.cost.tier, endSec - startSec);
return {
cost: {
tier: definition.cost.tier,
frameReads: definition.cost.frameReads,
...(advice ? { advice } : {}),
},
};
}
function failure(message: string): AgentToolExecution {
return { ok: false, resultJson: JSON.stringify({ error: message }) };
}
/**
* Runs `unitName` once per item, folding the document forward.
*
* ponytail: the batch tools exist to save ROUND TRIPS, not to mean something new.
* Replaying the unitary executor is what guarantees that — anchoring, clip
* resolution, clamping, the wording of every refusal, all identical by
* construction rather than by a second implementation staying in step. A batch
* of N is exactly N unitary calls minus N-1 round trips, and `agent-tools.test`
* asserts that against a document built the long way.
*
* ponytail: PARTIAL application, deliberately. `replaceTimeline` is the repo's
* other array-taking tool and it refuses in one block — "Refused … Nothing was
* modified" — which is right for a tool that rebuilds the whole timeline and
* ruinous for one that adds ten independent cuts: a single bad bound would throw
* away nine good ones and the model would have to guess which. So each item
* stands or falls alone, and the result says which did what. `ok:false` is kept
* for the case where NOTHING landed, because that is the only one where the
* document did not move.
*/
function applyBatch(
document: AxcutDocument,
unitName: "addTrim" | "addZoom",
items: unknown[],
options: AgentToolOptions | undefined,
noun: string,
): AgentToolExecution {
let current = document;
const applied: Array<Record<string, unknown>> = [];
const refused: Array<{ index: number; error: string }> = [];
items.forEach((item, index) => {
const execution = executeAgentTool(current, unitName, JSON.stringify(item), options);
let payload: Record<string, unknown> = {};
try {
payload = JSON.parse(execution.resultJson) as Record<string, unknown>;
} catch {
payload = { error: execution.resultJson };
}
if (execution.ok && execution.document) {
current = execution.document;
applied.push({ index, ...payload });
} else {
refused.push({ index, error: String(payload.error ?? "refused") });
}
});
// Nothing landed: the document is untouched, so say so the way every other
// refusal does rather than reporting a success with an empty list.
if (applied.length === 0) {
return failure(
`No ${noun} was added. ` +
refused.map((r) => `[${r.index}] ${r.error}`).join(" | ") +
" Nothing was modified.",
);
}
const refusedSuffix = refused.length ? `, ${refused.length} refused` : "";
return {
ok: true,
document: current,
// The counts come first on purpose: the model must be able to see that one
// of ten was refused WITHOUT re-reading the document, and know which one.
resultJson: JSON.stringify({
requested: items.length,
appliedCount: applied.length,
refusedCount: refused.length,
applied,
...(refused.length ? { refused } : {}),
}),
summary: `added ${applied.length} ${noun}${applied.length === 1 ? "" : "s"}${refusedSuffix}`,
};
}
/** The clips as the model would have to name them, for an error about an id it
* got wrong — a bare "Unknown clip: demo" leaves it guessing twice. */
function clipRoster(document: AxcutDocument): string {
const clips = document.timeline.clips;
if (clips.length === 0) return "The timeline has no clips.";
return `The timeline is: ${clips.map((c) => `${c.id}${c.reason ? ` (${c.reason})` : ""}`).join(", ")}.`;
}
/** The in-app agent's edit switch, as AI settings label it (`providerSettings.projectEditsLabel`). */
export const AGENT_EDITS_SETTING = "'Agent edits' in AI settings";
/**
* The refusal the model gets for every write while `allowAgentEdits` is off.
*
* ponytail: it has to be ACTIONABLE, not merely negative. There is no
* AbortSignal and no timeout anywhere on the product path, and the agent runs
* at recursionLimit 1000: a bare "refused" invites a model to try the next
* write tool, and the next, for as long as the provider will answer. So the
* payload says what to do instead (state the edit, ask), that retrying is
* pointless (every write is refused, not just this one), that claiming the edit
* happened is forbidden, and where the user turns the setting back on. The
* arguments are echoed back so the model can quote the exact edit it wanted
* without a second round trip.
*
* The `{error}` envelope is the shape the workbench's wire oracle already
* recognises as a failed tool result — a new one would read as a success.
*
* `setting` names the switch as the user sees it: the in-app agent's by default,
* the MCP server's when an MCP client is refused (`openscreen-mcp-server.ts`).
*/
export function consentRequired(
name: string,
args: unknown,
setting: string = AGENT_EDITS_SETTING,
): AgentToolExecution {
return {
ok: false,
resultJson: JSON.stringify({
error:
"Edits are turned off: the user asked to be consulted before the timeline changes. " +
"Nothing was modified.",
code: "consent_required",
tool: name,
requestedArgs: args,
howToProceed:
"Describe the exact edit you would make — the tool, the times, the ids — and ask the " +
"user to confirm it. Do NOT retry this call and do NOT reach for another write tool: " +
"every one of them is refused while the setting is off. Never say an edit was applied. " +
`If they want you to go ahead, they can turn on ${setting}.`,
}),
};
}
export interface AgentToolOptions {
/**
* `allowAgentEdits` from the LLM config, threaded down as an explicit
* argument rather than read from a module global so this stays pure and
* testable. Only `false` refuses; `undefined` is "allowed", which keeps the
* three-argument call sites (and ~40 test assertions) working unchanged and
* mirrors `config.allowAgentEdits !== false`.
*/
editsAllowed?: boolean;
/** Cursor telemetry resolved for THIS turn — see `CursorTelemetryContext`.
* Absent means the runtime has no telemetry reader wired at all, which the
* executor reports as "could not look", never as "there is none". */
cursorTelemetry?: CursorTelemetryContext;
/** Whether this machine can cut out the camera background, asked only by the two tools that
* report it. Absent where nothing can ask the compositor: reported as not checked. */
cameraSegmentation?: () => SegmentationSupport;
/** The compositor's check of the WGSL this call writes (`checkShaderEffect`), asked beforehand
* by the async wrapper (`shaderEffectToCheck`). Absent where nothing can: the effect is stored
* unchecked, and the result says `validated: false`. */
checkShaderEffect?: ShaderEffectCheckLookup;
/** `addAutoZooms`' plan, made by the async wrapper from the recorded clicks. Absent means
* no cursor reader is wired here, which the tool reports as "could not look". */
autoZooms?: AutoZoomPlan;
}
/** What the timeline wand would add, and what each recording's cursor read gave. */
export interface AutoZoomPlan {
suggestions: AutoZoomSuggestion[];
/** assetId → the read's verdict, for every video asset with a clip on the timeline. */
telemetry: Record<string, CursorTelemetryLoad["status"]>;
}
/**
* What the runtime managed to find out about cursor telemetry this turn.
*
* ponytail: the three states are the whole point, and collapsing any two of them
* is the defect. "There is no telemetry for this asset" is a fact about the
* project. "I have no way to look" is a fact about me. The measured failure was
* the model reporting the second as the first — it probed an empty sandbox and
* concluded the recording had no pointer data — so the payloads keep them apart
* and the prompt tells the model to keep them apart too.
*/
export interface CursorTelemetryContext {
/** assetId → whether a sidecar exists. A MISSING key means "not checked",
* which the snapshot reports as null, never false. */
availableByAssetId?: Record<string, boolean>;
/** The samples the async tool wrapper loaded before entering the executor.
* `executeAgentTool` is synchronous — deliberately, it is the pure gate every
* mutation passes through — so the IO happens outside and the verdict comes
* in as data. */
load?: CursorTelemetryLoad;
}
export type CursorTelemetryLoad =
| { status: "ok"; assetId: string; samples: CursorTrackSample[]; durationSec?: number }
/** We looked; this asset has no sidecar. */
| { status: "no-sidecar"; assetId: string }
/** We could NOT look: no reader wired, path refused, file unreadable. */
| { status: "unavailable"; assetId: string | null; note?: string };
/** Resolves the asset a cursor/transcript question is about: the argument if it
* names one, otherwise the project's primary asset. Shared so the async wrapper
* in `deep-agent/service.ts` loads telemetry for exactly the asset the executor
* will then report on. */
export function resolveCursorAssetId(
document: AxcutDocument,
assetId?: string | null,
): string | null {
return assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id ?? null;
}
// ─── What the pointer was doing where the zoom landed ──────────────────────
//
// ponytail: `focus` is the one thing a zoom write says that nothing ever
// checked. A span covering no clip is refused, a depth outside the table is
// refused, and the result reports the span that really landed — but a focus on
// the pointer and a focus half a frame off it produced byte-identical results,
// so a caller had no way to find out which of the two it had just written. The
// result now carries where the pointer ACTUALLY was over the window the zoom
// landed on, beside the focus the call used.
//
// It informs; it decides nothing. Framing a slide, a face, or a corner the
// pointer never visits is a legitimate zoom: nothing is moved, nothing is
// refused, and this is a measurement the caller is free to disagree with. The
// only thing that changes is that the difference is on the page instead of
// nowhere.
/** Per-axis median, not the mean. A pointer that crosses the frame and comes
* back averages to the middle of a path it spent no time on, while the median
* lands where it actually sat. `spread` is what says whether either number
* describes anything. */
function medianOf(values: number[]): number {
const sorted = [...values].sort((a, b) => a - b);
const mid = sorted.length >> 1;
return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
}
function round3(value: number): number {
return Math.round(value * 1000) / 1000;
}
/**
* Where the recorded pointer was over the span a zoom write LANDED on.
*
* A zoom is authored in VIRTUAL seconds and the pointer is recorded on the
* asset's SOURCE clock, so something has to map between them — and that map
* already exists, computed once by `anchorRawRegionsToClips`, which writes
* `sourceStartSec`/`sourceEndSec` onto every fragment as it ventilates a span
* across clips. So the source windows are read back OFF THE FRAGMENTS the write
* just stored, never re-derived from `timelineStartSec`. A second derivation
* would be free to drift, and it would drift on exactly the cases that make this
* report worth having: a CLAMPED span and a span SPLIT across two clips both
* land on source windows that are not the ones asked for, and both are already
* right in the anchors. A fragment whose clip draws on another asset contributes
* nothing — this telemetry does not describe that media.
*
* Absence is never a claim. The field is left OFF when the runtime holds no
* telemetry for the footage under the span, which covers "no reader wired",
* "this asset has no sidecar" and "the zoom landed on another asset's clip"
* alike; none of those is evidence about the recording, and the tool description
* says so. `available: false` is emitted only for the two things that ARE
* findings about this span: nothing was recorded over it, or everything recorded
* over it is cut out of playback.
*/
function cursorAnchorReport(
// `id` is not read; it is what makes this a fragment of a stored region rather
// than an all-optional bag TypeScript would let any object satisfy.
regions: Array<{ id: string; clipId?: string; sourceStartSec?: number; sourceEndSec?: number }>,
document: AxcutDocument,
focus: { cx: number; cy: number },
telemetry: CursorTelemetryContext | undefined,
): Record<string, unknown> | undefined {
const load = telemetry?.load;
if (load?.status !== "ok") return undefined;
const byId = new Map(document.timeline.clips.map((c) => [c.id, c]));
const windows = regions.flatMap((region) => {
const clip = region.clipId ? byId.get(region.clipId) : undefined;
if (!clip || clip.assetId !== load.assetId) return [];
if (region.sourceStartSec === undefined || region.sourceEndSec === undefined) return [];
return [{ clip, startSec: region.sourceStartSec, endSec: region.sourceEndSec }];
});
if (windows.length === 0) return undefined;
const xs: number[] = [];
const ys: number[] = [];
let cutOut = 0;
for (const sample of load.samples) {
if (
!Number.isFinite(sample.timeMs) ||
!Number.isFinite(sample.cx) ||
!Number.isFinite(sample.cy)
) {
continue;
}
const atSec = sample.timeMs / 1000;
const covering = windows.find((w) => atSec >= w.startSec && atSec <= w.endSec);
if (!covering) continue;
// `trimAppliesToClip` is THE rule for "is this cut on this clip", and the
// fragment names its clip, so the question is answered exactly once here.
// A trimmed instant is one the viewer never reaches: a position argued from
// frames that do not play would be the same kind of untruth as a span that
// reports the edges it was asked for rather than the ones it got.
if (
document.timeline.trimRanges.some(
(t) => trimAppliesToClip(t, covering.clip) && atSec >= t.startSec && atSec <= t.endSec,
)
) {
cutOut += 1;
continue;
}
xs.push(sample.cx);
ys.push(sample.cy);
}
if (xs.length === 0) {
return cutOut > 0
? {
available: false,
reason: "trimmed-out",
note:
"The pointer WAS recorded over this span, but a trim cuts every one of those " +
"instants out of playback, so none of them describes what a viewer sees here.",
}
: {
available: false,
reason: "no-samples",
note:
"This recording's pointer telemetry covers no instant of this span. That is a " +
"fact about this span, not about the recording.",
};
}
const cx = medianOf(xs);
const cy = medianOf(ys);
let spread = 0;
for (let i = 0; i < xs.length; i += 1) {
spread = Math.max(spread, Math.hypot(xs[i] - cx, ys[i] - cy));
}
return {
available: true,
// Echoed, including the default a call that omitted `focus` silently got:
// "you asked for the centre" is the half of the comparison the caller
// cannot reconstruct from its own arguments.
focus: { cx: focus.cx, cy: focus.cy },
cursor: { cx: round3(cx), cy: round3(cy) },
offset: round3(Math.hypot(cx - focus.cx, cy - focus.cy)),
spread: round3(spread),
samples: xs.length,
};
}
export function executeAgentTool(
document: AxcutDocument,
name: string,
rawArgs: string,
options?: AgentToolOptions,
): AgentToolExecution {
let args: unknown = {};
if (rawArgs.trim()) {
try {
args = JSON.parse(rawArgs);
} catch {
return failure(`Tool arguments are not valid JSON: ${rawArgs.slice(0, 120)}`);
}
}
// ponytail: THE guard for `allowAgentEdits`. It sits here, in front of the
// switch, because this function is the single gate every document mutation
// passes through — the two production callers (`documentTool` for the main
// agent and, through it, any sub-agent) and the workbench alike. A guard in
// the event sink could not work: the sink is an observer, called from inside
// the tool body, and by the time it fires the edit has happened.
//
// Reads stay open on purpose. A model that cannot call getCurrentDocument or
// getTranscript cannot describe the edit it is asking permission for, and
// would answer "I have no tool for that" — false, and worse than silence.
if (options?.editsAllowed === false && isMutatingTool(name)) {
return consentRequired(name, args);
}
switch (name) {
case "getCaptions":
case "setCaptionSettings":
case "getCaptionTranslationUnits":
case "putCaptionTranslation":
case "removeCaptionTranslation":
case "setVocabulary":
return executeCaptionTool(document, name, args);
case "setAspectRatio": {
// The format alone, kept for clients that know it: setEditorSettings does the work.
const parsed = setAspectRatioArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const previous = getEditorSettings(document).aspectRatio;
const execution = executeSetEditorSettings(
document,
{ aspectRatio: parsed.data.aspectRatio },
options?.cameraSegmentation?.() ?? null,
);
if (!execution.ok) return execution;
return {
...execution,
resultJson: JSON.stringify({ aspectRatio: parsed.data.aspectRatio, previous }),
summary: `set the format to ${parsed.data.aspectRatio}`,
};
}
case "setEditorSettings":
return executeSetEditorSettings(document, args, options?.cameraSegmentation?.() ?? null);
case "getCurrentDocument": {
return {
ok: true,
resultJson: JSON.stringify({
...documentSnapshotForModel(document, options?.cursorTelemetry),
...editorSettingsSnapshot(document, options?.cameraSegmentation?.() ?? null),
}),
};
}
case "getCursorTrack": {
const parsed = getCursorTrackArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const assetId = resolveCursorAssetId(document, parsed.data.assetId);
if (!assetId)
return failure("Project has no assets — there is nothing to read telemetry for.");
if (!document.assets.some((a) => a.id === assetId)) {
return failure(`Unknown asset: ${assetId}`);
}
const load = options?.cursorTelemetry?.load;
// ponytail: `ok: true` on all three branches, including the two that
// return nothing. None of them is a failure of the CALL — the question
// was answered, and the answer is "none" or "I could not look". Marking
// them ok:false would push the model to retry a tool whose verdict will
// not change, and there is no timeout anywhere on this path.
if (!load || load.status === "unavailable") {
return {
ok: true,
resultJson: JSON.stringify({
available: false,
reason: "unavailable",
assetId,
note:
load?.note ??
"Cursor telemetry cannot be read in this run — no reader is wired to this " +
"runtime. This says nothing about whether the recording has any: report " +
"the limit as yours, and do not tell the user the project has no cursor data.",
}),
};
}
if (load.status === "no-sidecar") {
return {
ok: true,
resultJson: JSON.stringify({
available: false,
reason: "no-sidecar",
assetId: load.assetId,
note:
"Checked: this asset has no cursor-telemetry sidecar. That normally means it " +
"was imported rather than recorded with OpenScreen's cursor recorder. This is " +
"a fact about the asset, not a limit of yours.",
}),
};
}
const asset = document.assets.find((a) => a.id === load.assetId);
const track = buildCursorTrack({
assetId: load.assetId,
samples: load.samples,
durationSec: load.durationSec ?? asset?.durationSec ?? 0,
clips: document.timeline.clips,
trimRanges: document.timeline.trimRanges,
});
return { ok: true, resultJson: JSON.stringify({ available: true, ...track }) };
}
case "getTranscript": {
const parsed = getTranscriptArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const assetId =
parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id;
const transcript =
document.transcripts.find((t) => t.assetId === assetId) ??
(document.transcript?.assetId === assetId ? document.transcript : null);
if (!transcript) {
return failure(`No transcript for asset ${assetId ?? "(none)"}.`);
}
// ponytail: no cap. There used to be a `.slice(0, 800)` here, guarded by
// "words would blow the context" — written believing a segment was a
// phrase. On the production path a segment IS one word
// (src/lib/captioning/transcribe.ts: whisper's word timings are mapped
// one-to-one), so the cap cut the transcript at the 800th WORD — around
// five minutes of speech — and said nothing about it. The model read a
// fifth of a half-hour recording, cut the silences it could see, and
// reported the job done, because nothing in the payload told it otherwise.
//
// A whole 30-minute transcript is ~285k characters, ~70k tokens: large,
// and well inside every model this app talks to. If a recording ever does
// get near a window, the honest fix is to know the window — the app has no
// per-model context budget today — not to guess a number here and drop the
// rest in silence.
const segments = transcript.segments.map((s) => ({
id: s.id,
kind: s.kind,
startSec: s.startSec,
endSec: s.endSec,
text: s.text,
}));
return {
ok: true,
resultJson: JSON.stringify({ assetId, language: transcript.language, segments }),
};
}
// The word-level read. `getTranscript` answers in SEGMENTS, whose ids belong to a
// different namespace than the words — so on its own it cannot address anything
// `setWordText` takes. This is the one that can. It is separate rather than folded
// in because a whole transcript is already ~70k tokens and most turns never touch a
// word; the span filter is there so fixing one name costs one phrase, not the film.
case "getTranscriptWords": {
const parsed = getTranscriptWordsArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const assetId =
parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id;
const transcript =
document.transcripts.find((t) => t.assetId === assetId) ??
(document.transcript?.assetId === assetId ? document.transcript : null);
if (!transcript) {
return failure(`No transcript for asset ${assetId ?? "(none)"}.`);
}
const from = parsed.data.startSec ?? Number.NEGATIVE_INFINITY;
const to = parsed.data.endSec ?? Number.POSITIVE_INFINITY;
const words = transcript.words
.filter((word) => word.endSec >= from && word.startSec <= to)
.map((word) => ({
id: word.id,
text: word.text,
startSec: word.startSec,
endSec: word.endSec,
// Only the words that are NOT plain transcription say so, so the common
// case costs nothing to read.
...(word.source ? { source: word.source } : {}),
...(word.originalText !== undefined ? { originalText: word.originalText } : {}),
}));
return {
ok: true,
resultJson: JSON.stringify({
assetId,
language: transcript.language,
total: transcript.words.length,
returned: words.length,
words,
}),
};
}
// Correcting what the transcriber HEARD. This writes text and nothing else: the
// captions follow it, the film does not move. The tool for making a spoken word go
// away is addTrim, which removes its audio with it.
case "setWordText": {
const parsed = setWordTextArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const assetId =
parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id;
if (!assetId) return failure("Project has no assets — nothing to correct.");
const { wordId, text } = parsed.data;
const transcript = document.transcripts.find((t) => t.assetId === assetId);
const before = transcript?.words.find((word) => word.id === wordId);
if (!before) {
return failure(
`No word ${wordId} in the transcript for asset ${assetId}. ` +
`Call getTranscriptWords to read the ids.`,
);
}
if (before.text === text) {
return failure(`Word ${wordId} already reads "${text}" — nothing to change.`);
}
// This tool exists to fix a name the transcriber misheard. An INSERTED word was
// never heard: retyping it resizes the clip it plays on and asks for generated
// media of a new length, which is the gesture the editor gates on `insertionsEnabled`
// — and that gate lives in the renderer, where the chat does not run. Refused here
// unconditionally rather than mirrored, because the agent has no business authoring
// generated media at all.
if (isGeneratedAssetId(assetId)) {
return failure(
`Word ${wordId} was added to the transcript, not heard — the chat cannot rewrite it.`,
);
}
let next: AxcutDocument;
try {
next = setDocumentWordText(document, assetId, wordId, text);
} catch (error) {
return failure(error instanceof Error ? error.message : String(error));
}
const after = next.transcripts
.find((t) => t.assetId === assetId)
?.words.find((word) => word.id === wordId);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
wordId,
assetId,
text: after?.text ?? text,
was: before.text,
// Absent once the word is back to what the transcriber said — the pair is
// cleared on that round trip, and the model should be able to see it.
originalText: after?.originalText,
blanked: text.trim().length === 0,
}),
summary:
text.trim().length === 0 ? `blanked "${before.text}"` : `"${before.text}" → "${text}"`,
};
}
case "removeWords": {
const parsed = removeWordsArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { assetId: selectedAssetId, wordIds } = parsed.data;
if (new Set(wordIds).size !== wordIds.length) {
return failure("Duplicate word IDs in one request. Nothing was modified.");
}
// Resolve every ID before writing anything. Word IDs, rather than guessed
// timestamps, designate which occurrence of a repeated word is speech to cut.
const transcripts = [...document.transcripts];
if (
document.transcript &&
!transcripts.some((transcript) => transcript.assetId === document.transcript?.assetId)
) {
transcripts.push(document.transcript);
}
const selected: Array<{
wordId: string;
text: string;
assetId: string;
clipId: string;
index: number;
startSec: number;
endSec: number;
}> = [];
for (const wordId of wordIds) {
const matches = transcripts
.filter(
(transcript) => selectedAssetId === undefined || transcript.assetId === selectedAssetId,
)
.flatMap((transcript) =>
transcript.words.flatMap((word, index) =>
word.id === wordId ? [{ assetId: transcript.assetId, word, index }] : [],
),
);
if (matches.length !== 1) {
return failure(
matches.length === 0
? `Unknown transcript word ID: ${wordId}. Call getTranscriptWords to read the IDs. Nothing was modified.`
: `Word ID ${wordId} appears more than once in the transcripts. Pass the assetId used for getTranscriptWords. Nothing was modified.`,
);
}
const { assetId, word, index } = matches[0];
const asset = document.assets.find((candidate) => candidate.id === assetId);
if (!asset || isGeneratedAssetId(assetId) || word.source === "synth") {
return failure(`Word ${wordId} has no recorded source asset. Nothing was modified.`);
}
const { startSec, endSec } = word;
if (
!Number.isFinite(startSec) ||
!Number.isFinite(endSec) ||
startSec < 0 ||
endSec <= startSec ||
(asset.durationSec !== undefined &&
(!Number.isFinite(asset.durationSec) || endSec > asset.durationSec))
) {
return failure(`Word ${wordId} has invalid source timestamps. Nothing was modified.`);
}
const clips = document.timeline.clips.filter((clip) => {
const clipEnd = clip.sourceEndSec ?? asset.durationSec;
return (
clip.assetId === assetId &&
clipEnd !== undefined &&
Number.isFinite(clipEnd) &&
startSec >= clip.sourceStartSec &&
endSec <= clipEnd
);
});
if (clips.length !== 1) {
return failure(
`Word ${wordId} must be fully contained in exactly one clip; found ${clips.length}. Nothing was modified.`,
);
}
const clip = clips[0];
if (
document.timeline.trimRanges.some(
(trim) =>
trimAppliesToClip(trim, clip) && startSec < trim.endSec && endSec > trim.startSec,
)
) {
return failure(`Word ${wordId} is already cut by a trim. Nothing was modified.`);
}
selected.push({
wordId,
text: word.text,
assetId,
clipId: clip.id,
index,
startSec,
endSec,
});
}
// Words that follow one another in the transcript go out as ONE cut: a phrase or a
// flubbed take cut word by word would leave the gaps between its words playing.
selected.sort((a, b) => a.assetId.localeCompare(b.assetId) || a.index - b.index);
const runs: Array<typeof selected> = [];
for (const word of selected) {
const last = runs.at(-1)?.at(-1);
if (
last?.assetId === word.assetId &&
last.clipId === word.clipId &&
last.index + 1 === word.index
) {
runs.at(-1)?.push(word);
} else {
runs.push([word]);
}
}
let current = document;
const removed: Array<Record<string, unknown>> = [];
for (const run of runs) {
const first = run[0];
const ids = run.map((word) => word.wordId).join(", ");
const execution = executeAgentTool(
current,
"addTrim",
JSON.stringify({
assetId: first.assetId,
clipId: first.clipId,
startSec: first.startSec,
endSec: run[run.length - 1].endSec,
reason: "removed words",
}),
options,
);
if (!execution.ok || !execution.document) {
return failure(`Could not cut ${ids}: ${execution.resultJson}. Nothing was modified.`);
}
const { trimRangeId } = JSON.parse(execution.resultJson) as { trimRangeId: string };
const trim = execution.document.timeline.trimRanges.find(
(range) => range.id === trimRangeId,
);
const clip = current.timeline.clips.find((candidate) => candidate.id === first.clipId);
const asset = current.assets.find((candidate) => candidate.id === first.assetId);
const clipEnd = clip?.sourceEndSec ?? asset?.durationSec;
if (
!trim ||
!clip ||
clipEnd === undefined ||
trim.clipId !== first.clipId ||
!Number.isFinite(trim.startSec) ||
!Number.isFinite(trim.endSec) ||
trim.startSec < clip.sourceStartSec ||
trim.endSec > clipEnd ||
trim.endSec <= trim.startSec
) {
return failure(`Cut for ${ids} left its clip. Nothing was modified.`);
}
current = execution.document;
for (const word of run) {
removed.push({
wordId: word.wordId,
text: word.text,
assetId: word.assetId,
clipId: word.clipId,
wordStartSec: word.startSec,
wordEndSec: word.endSec,
trimRangeId,
startSec: trim.startSec,
endSec: trim.endSec,
});
}
}
return {
ok: true,
document: current,
resultJson: JSON.stringify({
requested: wordIds.length,
removedCount: removed.length,
removed,
...cutTransitionsReport(document, current),
}),
summary: `removed ${removed.length} word${removed.length === 1 ? "" : "s"}`,
};
}
case "addTrim": {
const parsed = addTrimArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const assetId =
parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id;
if (!assetId) return failure("Project has no assets — nothing to trim.");
if (!document.assets.some((a) => a.id === assetId)) {
return failure(`Unknown asset: ${assetId}`);
}
let startSec = Math.min(parsed.data.startSec, parsed.data.endSec);
let endSec = Math.max(parsed.data.startSec, parsed.data.endSec);
// Which clip the cut sits on. Named explicitly when the model says so; otherwise
// inferred from the source range — but only when the answer is unique. Two clips
// over the same asset covering that range is a real question the model has to
// answer (their ids are in the snapshot), not one to settle by picking the first.
const covering = document.timeline.clips.filter(
(c) =>
c.assetId === assetId &&
endSec > c.sourceStartSec &&
startSec < (c.sourceEndSec ?? Number.POSITIVE_INFINITY),
);
let clipId = parsed.data.clipId;
if (clipId) {
const target = document.timeline.clips.find((c) => c.id === clipId);
if (!target) return failure(`Unknown clip: ${clipId}`);
if (target.assetId !== assetId) {
return failure(`Clip ${clipId} does not use asset ${assetId}.`);
}
} else if (covering.length === 1) {
clipId = covering[0].id;
} else if (covering.length > 1) {
return failure(
`${covering.length} clips use asset ${assetId} over ${formatSec(startSec)} – ${formatSec(
endSec,
)} (${covering.map((c) => c.id).join(", ")}). Pass clipId to say which one to trim.`,
);
}
// Breath next to the speech that stays, and no one-frame flash against a cut
// already on the same clip — the same shaping the transcript pane applies.
const clip = document.timeline.clips.find((c) => c.id === clipId);
const siblings = document.timeline.trimRanges.filter((t) =>
clip ? trimAppliesToClip(t, clip) : t.assetId === assetId,
);
const transcript =
document.transcripts.find((t) => t.assetId === assetId) ??
(document.transcript?.assetId === assetId ? document.transcript : null);
const words = (transcript?.words ?? [])
.filter((w) => w.source !== "synth")
.map((w) => {
const mid = (w.startSec + w.endSec) / 2;
const inClip =
!clip || (mid >= clip.sourceStartSec && mid < (clip.sourceEndSec ?? Infinity));
const cut = siblings.some((t) => mid >= t.startSec && mid < t.endSec);
return { startSec: w.startSec, endSec: w.endSec, kept: inClip && !cut };
});
({ startSec, endSec } = mergeCloseCuts(
breatheCut({ startSec, endSec }, words),
siblings,
document.assets.find((a) => a.id === assetId)?.video?.fps,
));
const trim = {
id: createId("trim"),
assetId,
...(clipId ? { clipId } : {}),
startSec,
endSec,
reason: parsed.data.reason,
origin: "agent" as const,
};
const next: AxcutDocument = {
...document,
timeline: {
...document.timeline,
trimRanges: [...document.timeline.trimRanges, trim],
},
};
return {
ok: true,
document: next,
resultJson: JSON.stringify({
trimRangeId: trim.id,
startSec,
endSec,
...cutTransitionsReport(document, next),
}),
summary: `added trim ${formatSec(startSec)} – ${formatSec(endSec)}`,
};
}
case "addTrims": {
const parsed = addTrimsArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
return applyBatch(document, "addTrim", parsed.data.ranges, options, "trim");
}
case "setTrim": {
const parsed = setTrimArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { trimRangeId } = parsed.data;
if (!document.timeline.trimRanges.some((r) => r.id === trimRangeId)) {
return failure(`Unknown trim range: ${trimRangeId}`);
}
const startSec = Math.min(parsed.data.startSec, parsed.data.endSec);
const endSec = Math.max(parsed.data.startSec, parsed.data.endSec);
// Moving a cut out of the clip it is anchored to would leave it storing a range
// nothing plays — silently inert. Re-point it at the clip the new range actually
// lands in, but only when that clip is unique: with several candidates the old
// anchor is the better guess than an arbitrary one.
const reanchor = (trim: AxcutDocument["timeline"]["trimRanges"][number]) => {
if (!trim.clipId) return undefined;
const covers = (c: { sourceStartSec: number; sourceEndSec?: number }) =>
endSec > c.sourceStartSec && startSec < (c.sourceEndSec ?? Number.POSITIVE_INFINITY);
const current = document.timeline.clips.find((c) => c.id === trim.clipId);
if (current && covers(current)) return trim.clipId;
const candidates = document.timeline.clips.filter(
(c) => c.assetId === trim.assetId && covers(c),
);
return candidates.length === 1 ? candidates[0].id : trim.clipId;
};
const next: AxcutDocument = {
...document,
timeline: {
...document.timeline,
trimRanges: document.timeline.trimRanges.map((r) =>
r.id === trimRangeId ? { ...r, clipId: reanchor(r), startSec, endSec } : r,
),
},
};
return {
ok: true,
document: next,
resultJson: JSON.stringify({
trimRangeId,
startSec,
endSec,
...cutTransitionsReport(document, next),
}),
summary: `moved trim to ${formatSec(startSec)} – ${formatSec(endSec)}`,
};
}
case "setClipRange": {
const parsed = setClipRangeArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { clipId, cropRegion } = parsed.data;
const clip = document.timeline.clips.find((c) => c.id === clipId);
if (!clip) return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`);
const ranged =
parsed.data.sourceStartSec !== undefined || parsed.data.sourceEndSec !== undefined;
if (!ranged && cropRegion === undefined) {
return failure(
"Pass sourceStartSec / sourceEndSec, cropRegion, or both: nothing to change.",
);
}
const start = parsed.data.sourceStartSec ?? clip.sourceStartSec;
const end =
parsed.data.sourceEndSec ??
clip.sourceEndSec ??
document.assets.find((a) => a.id === clip.assetId)?.durationSec;
if (end === undefined) {
return failure(`Clip ${clipId} has no known end yet: pass sourceEndSec too.`);
}
const sourceStartSec = Math.min(start, end);
const sourceEndSec = Math.max(start, end);
// The clip's Edit modal, as one document: the range through the one shared mutator
// (which recomputes the clip's width and clamps / drops the anchored pills the new
// window no longer covers), then the crop. `null` clears it, and so does the whole
// frame: an untouched clip does not store the identity region.
const wholeFrame =
cropRegion?.x === 0 &&
cropRegion.y === 0 &&
cropRegion.width === 1 &&
cropRegion.height === 1;
const crop = cropRegion === undefined ? undefined : wholeFrame ? null : cropRegion;
// One pass for both, as the dialog's Apply: the fold weighs range and crop together.
const next = ranged
? setClipSourceRange(document, clipId, sourceStartSec, sourceEndSec, crop)
: setClipCrop(document, clipId, crop ?? null);
const dropped = droppedByEdit(document, next);
const casualties = dropped.droppedModifierIds.length + dropped.droppedTrimIds.length;
const after = next.timeline.clips.find((c) => c.id === clipId);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
clipId,
sourceStartSec,
sourceEndSec,
cropRegion: after?.cropRegion ?? null,
// Its media now continues the clip before it, with the same framing: one clip.
...(after
? {}
: { joinedIntoNeighbour: true, clipOrder: next.timeline.clips.map((c) => c.id) }),
...dropped,
}),
summary:
(ranged
? `trimmed clip to ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)}`
: `cropped clip ${clipId}`) +
(ranged && cropRegion !== undefined ? ", cropped" : "") +
(casualties > 0
? ` — dropped ${[...dropped.droppedModifierIds, ...dropped.droppedTrimIds].join(", ")}`
: ""),
};
}
case "insertClip": {
const parsed = insertClipArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { assetId } = parsed.data;
const asset = document.assets.find((a) => a.id === assetId);
const footage = document.assets.filter(
(a) => a.kind === "video" && !isGeneratedAssetId(a.id),
);
const roster = footage.length
? ` Footage in this project: ${footage.map((a) => `${a.id} (${a.label})`).join(", ")}.`
: " This project has no footage to place.";
if (!asset) return failure(`Unknown asset: ${assetId}.${roster}`);
if (asset.kind === "audio") {
return failure(
`Asset ${assetId} is audio, not footage: lay it over the recording with addAudio.`,
);
}
if (isGeneratedAssetId(assetId)) {
return failure(`Asset ${assetId} is generated media, not footage you can place.${roster}`);
}
const beforeClipId = parsed.data.beforeClipId ?? null;
const clips = document.timeline.clips;
const index =
beforeClipId === null ? clips.length : clips.findIndex((c) => c.id === beforeClipId);
if (index < 0) return failure(`Unknown clip: ${beforeClipId}. ${clipRoster(document)}`);
const durationSec = asset.durationSec;
const requestedEnd = parsed.data.sourceEndSec ?? durationSec;
if (requestedEnd === undefined) {
return failure(
`The length of ${assetId} is not known yet: pass sourceEndSec, in its source seconds, to say where the clip ends.`,
);
}
const bound = (sec: number) => (durationSec === undefined ? sec : Math.min(sec, durationSec));
const sourceStartSec = bound(parsed.data.sourceStartSec);
const sourceEndSec = bound(requestedEnd);
if (sourceEndSec <= sourceStartSec) {
return failure(
`The clip would be empty: ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)} of ${assetId}` +
(durationSec === undefined ? "." : `, which lasts ${formatSec(durationSec)}.`),
);
}
const clipId = createId("clip");
// The media panel's drop, through the same mutator.
const next = insertAssetClip(document, index, {
id: clipId,
assetId,
sourceStartSec,
sourceEndSec,
origin: "agent",
reason: "",
});
const placed = next.timeline.clips.find((c) => c.id === clipId);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
clipId: placed ? clipId : null,
assetId,
sourceStartSec,
sourceEndSec,
...(placed
? { timelineStartSec: placed.timelineStartSec, timelineEndSec: placed.timelineEndSec }
: // Its media continues the clip before it, with the same framing: one clip.
{ joinedIntoNeighbour: true }),
clipOrder: next.timeline.clips.map((c) => c.id),
}),
summary:
`inserted ${asset.label} ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)} ` +
(beforeClipId ? `before ${beforeClipId}` : "at the end"),
};
}
case "duplicateClip": {
const parsed = duplicateClipArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { clipId } = parsed.data;
if (!document.timeline.clips.some((c) => c.id === clipId)) {
return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`);
}
// The editor's Duplicate: the copy right after the original, with its own copy of
// the original's cuts and anchored effects.
const next = duplicateClip(document, clipId, "agent", "");
// By id, not position: the pass every clip edit ends with can fold clips that
// already met, which moves the positions.
const before = new Set(document.timeline.clips.map((c) => c.id));
const copy = next.timeline.clips.find((c) => !before.has(c.id));
if (!copy) return failure(`Clip ${clipId} could not be copied. Nothing was modified.`);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
clipId: copy.id,
of: clipId,
timelineStartSec: copy.timelineStartSec,
timelineEndSec: copy.timelineEndSec,
clipOrder: next.timeline.clips.map((c) => c.id),
copiedTrimIds: next.timeline.trimRanges
.filter((t) => t.clipId === copy.id)
.map((t) => t.id),
}),
summary: `duplicated ${clipId}`,
};
}
case "addAutoZooms": {
const parsed = addAutoZoomsArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
if (document.timeline.clips.length === 0) {
return failure("The timeline has no clips, so there is nothing to zoom on.");
}
const plan = options?.autoZooms;
const reads = Object.values(plan?.telemetry ?? {});
if (!plan || (reads.length > 0 && reads.every((status) => status === "unavailable"))) {
return failure(
"The recorded clicks could not be read from here, so no zoom was planned. This says nothing about whether the recording has any: report the limit as yours.",
);
}
// The timeline wand's write: zooms on the clicks, clear of the zooms already there.
const next = appendAutoZoomSuggestions(document, plan.suggestions);
const before = new Set(document.zoomRanges.map((z) => z.id));
const added = coalesceForAgent(next.zoomRanges).filter((z) => !before.has(z.id));
return {
ok: true,
...(added.length ? { document: next } : {}),
resultJson: JSON.stringify({
added: added.length,
zooms: added.map((z) => ({
id: z.id,
startSec: roundSec(z.startMs),
endSec: roundSec(z.endMs),
renderedScale: effectiveZoomScale(z),
focus: z.focus,
})),
telemetry: plan.telemetry,
...(added.length
? cutTransitionsReport(document, next)
: {
note: "No click on the timeline calls for a new zoom: none was recorded, each one is already under a zoom, or they fall in the first seconds or on the stop.",
}),
}),
...(added.length
? { summary: `added ${added.length} automatic zoom${added.length === 1 ? "" : "s"}` }
: {}),
};
}
case "moveClip": {
const parsed = moveClipArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { clipId } = parsed.data;
const beforeClipId = parsed.data.beforeClipId ?? null;
const clips = document.timeline.clips;
const moving = clips.find((c) => c.id === clipId);
if (!moving) return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`);
if (beforeClipId === clipId) {
return failure(
`beforeClipId must name a different clip than clipId (both were ${clipId}); ` +
`pass null to move it last.`,
);
}
const remaining = clips.filter((c) => c.id !== clipId);
let insertIndex = remaining.length;
if (beforeClipId !== null) {
insertIndex = remaining.findIndex((c) => c.id === beforeClipId);
if (insertIndex < 0) {
return failure(`Unknown clip: ${beforeClipId}. ${clipRoster(document)}`);
}
}
let next: AxcutDocument;
try {
// The clip's OWN origin is passed back in: `moveClip` stamps whatever it
// is given onto the clip, and a reorder is not a change of provenance —
// a user's clip stays the user's. Empty reason keeps its label.
next = moveClip(document, clipId, insertIndex, moving.origin, "");
} catch (err) {
return failure(err instanceof Error ? err.message : String(err));
}
const order = next.timeline.clips.map((c) => c.id);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
clipId,
beforeClipId,
clipOrder: order,
// Nothing is destroyed by a reorder — say so, since the alternative
// tool the model used to reach for destroyed plenty in silence.
trimCount: next.timeline.trimRanges.length,
...droppedByEdit(document, next),
}),
summary:
`moved ${clipId} ${beforeClipId ? `before ${beforeClipId}` : "to the end"} ` +
`(order: ${order.join(" → ")})`,
};
}
case "replaceTimeline": {
const parsed = replaceTimelineArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
// ponytail: refuse on the DAMAGE, not on the provenance. The old guard
// tested `origin === "user"`, which missed every clip the agent itself had
// placed — and disarmed itself, since a rebuild stamps its own output
// `origin: "agent"`. `planTimelineReplacement` answers the question that
// actually matters: what would this call cost?
const plan = planTimelineReplacement(document, parsed.data.intervals);
const objections: string[] = [];
if (plan.reorderRequested) {
objections.push(
"- the intervals are not in ascending order, so this reads as a REORDER. " +
"replaceTimeline sorts and merges its intervals, so the swap could not happen " +
"at all: use moveClip (it preserves ids, trims and anchored effects).",
);
}
if (plan.lostClipIds.length > 0) {
objections.push(
`- these clips would be merged away or dropped: ${plan.lostClipIds.join(", ")}. ` +
"To shorten one, use setClipRange; to delete one, removeClip; to change the " +
"order, moveClip; to cut a span inside one, addTrim.",
);
}
if (plan.slidRegionIds.length > 0) {
objections.push(
`- these effects are anchored to those clips and would be re-anchored onto ` +
`whatever footage moved under them: ${plan.slidRegionIds.join(", ")}.`,
);
}
if (objections.length > 0) {
return {
ok: false,
resultJson: JSON.stringify({
error:
"Refused: replaceTimeline would destroy work you were not asked to touch. " +
"Nothing was modified.\n" +
`${objections.join("\n")}\n` +
"replaceTimeline rebuilds the whole timeline and is only for an explicit " +
"'start over with these intervals' on a timeline with nothing to lose.",
code: "would_destroy",
reorderRequested: plan.reorderRequested,
lostClipIds: plan.lostClipIds,
slidRegionIds: plan.slidRegionIds,
}),
};
}
let next: AxcutDocument;
try {
next = replaceTimeline(document, parsed.data.intervals, parsed.data.reason, "agent");
} catch (err) {
return failure(err instanceof Error ? err.message : String(err));
}
const kept = parsed.data.intervals.length;
return {
ok: true,
document: next,
resultJson: JSON.stringify({
clipCount: next.timeline.clips.length,
trimCount: next.timeline.trimRanges.length,
// What survived, by name. The old result reported two counts, and a
// model reading `trimCount: 0` after a rebuild had nothing telling it
// WHICH cut had ceased to exist — which is what made "the silence trim
// is preserved" so easy to write.
preservedClipIds: plan.slots.map((s) => s.keepClipId).filter((id): id is string => !!id),
...(plan.absorbedTrimIds.length ? { absorbedTrimIds: plan.absorbedTrimIds } : {}),
...(plan.clippedTrimIds.length ? { clippedTrimIds: plan.clippedTrimIds } : {}),
}),
summary:
`rebuilt timeline from ${kept} interval${kept === 1 ? "" : "s"} ` +
`(${next.timeline.clips.length} clips, ${next.timeline.trimRanges.length} trims)` +
(plan.absorbedTrimIds.length
? ` — ${plan.absorbedTrimIds.join(", ")} now fall outside the kept spans`
: ""),
};
}
case "addZoom": {
const parsed = addZoomArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const twoLevels = refuseTwoLevels(args);
if (twoLevels) return twoLevels;
const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));
const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));
const level = zoomLevelArg(parsed.data.customScale);
const zoom = withRotationPreset(
{
id: createId("zoom"),
startMs,
endMs,
depth: (parsed.data.depth ?? 3) as 1 | 2 | 3 | 4 | 5 | 6,
...level,
focus: parsed.data.focus,
focusMode: parsed.data.focusMode,
...(parsed.data.hideCursor ? { hideCursor: true } : {}),
source: "manual" as const,
} as AxcutDocument["zoomRanges"][number],
parsed.data.rotationPreset,
);
const placed = anchorForAgent(zoom, document, "zoom");
const landing = landingOf(placed, document);
if (!landing.anchored) return coversNoClip("zoom", startMs / 1000, endMs / 1000, document);
const next: AxcutDocument = {
...document,
zoomRanges: [...document.zoomRanges, ...placed] as AxcutDocument["zoomRanges"],
};
const pastLimit = refuseScalePastLimit(next, landing.ids, parsed.data.customScale);
if (pastLimit) return pastLimit;
// Measured over what was STORED, never over what was asked for: `placed`
// is the clamped, ventilated truth, so the report cannot end up
// describing a window the zoom does not occupy.
const anchor = cursorAnchorReport(placed, document, zoom.focus, options?.cursorTelemetry);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
zoomId: landing.ids[0],
depth: zoom.depth,
...(zoom.rotationPreset ? { rotationPreset: zoom.rotationPreset } : {}),
// The depth alone is an ordinal; reported on its own it is what the
// model turns into "3×" for a frame that renders 1.80×.
renderedScale: effectiveZoomScale(zoom),
...zoomBehaviourReport(zoom, document),
...landingReport(landing, startMs / 1000, endMs / 1000),
...(anchor ? { cursorAnchor: anchor } : {}),
...cutTransitionsReport(document, next),
}),
summary:
`added zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)} ` +
`at ${effectiveZoomScale(zoom).toFixed(2)}×` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "addZooms": {
const parsed = addZoomsArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
return applyBatch(document, "addZoom", parsed.data.regions, options, "zoom");
}
case "setZoom": {
const parsed = setZoomArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const twoLevels = refuseTwoLevels(args);
if (twoLevels) return twoLevels;
const { zoomId } = parsed.data;
const existing = document.zoomRanges.find((z) => z.id === zoomId);
if (!existing) return failure(`Unknown zoom: ${zoomId}`);
const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec);
const blocked = refuseIfBlocked("zoom", document.zoomRanges, zoomId, { startMs, endMs });
if (blocked) return blocked;
const zoomPill = new Set(resolvePillIds(document.zoomRanges, zoomId));
// ponytail: a depth write CLEARS customScale, and that is the whole
// point. `effectiveZoomScale` returns customScale when it is set, so on a
// migrated v1.7 zoom (`migrate.ts:185` keeps both fields) a `setZoom
// {depth: 6}` used to store the 6, answer ok, and leave the picture at
// 1.10× forever — a write that could not do what every layer involved
// believed it did. Dropping the override is destructive of a fine-tuned
// value, so it happens only on an explicit depth write (which IS a
// request to change the strength) and is named in the result and the
// summary rather than done quietly.
const clearsCustomScale =
parsed.data.depth !== undefined &&
document.zoomRanges.some((z) => zoomPill.has(z.id) && z.customScale != null);
const level = zoomLevelArg(parsed.data.customScale);
const { hideCursor, focusMode } = parsed.data;
const rebuiltZooms = replacePillSpan(
// payload edits first, applied to every region under the pill…
document.zoomRanges.map((z) => {
if (!zoomPill.has(z.id)) return z;
const { customScale, hideCursor: wasHidden, ...rest } = z;
// A level the table has is written as its depth, which drops the custom scale.
const keepsScale = !clearsCustomScale && !("depth" in level);
const edited = {
...rest,
...(keepsScale && customScale != null ? { customScale } : {}),
// As the inspector writes it: shown is the absence of the flag.
...((hideCursor ?? wasHidden) ? { hideCursor: true } : {}),
...(parsed.data.depth !== undefined
? { depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6 }
: {}),
...level,
...(parsed.data.focus ? { focus: parsed.data.focus } : {}),
...(focusMode ? { focusMode } : {}),
};
return parsed.data.rotationPreset === undefined
? edited
: withRotationPreset(edited, parsed.data.rotationPreset);
}),
// …then the span: clamped against different-property pills, then re-ventilated.
zoomId,
startMs,
endMs,
document.timeline.clips,
() => createId("zoom"),
) as AxcutDocument["zoomRanges"];
const landing = landingAfterPillEdit(document.zoomRanges, rebuiltZooms, zoomPill, document);
if (!landing.anchored) return coversNoClip("zoom", startMs / 1000, endMs / 1000, document);
const next: AxcutDocument = { ...document, zoomRanges: rebuiltZooms };
const pastLimit = refuseScalePastLimit(next, landing.ids, parsed.data.customScale);
if (pastLimit) return pastLimit;
// Read back off the document, not off the request: the pill may have been
// re-ventilated, and `renderedScale` is the only number the viewer sees.
const landed = new Set(landing.ids);
const strength = rebuiltZooms.find((z) => landed.has(z.id));
// The EFFECTIVE focus, read off the document exactly like `renderedScale`
// is: a setZoom that moved only the span still gets told what its
// untouched focus now looks at, which is most of the reason to reshape a
// zoom at all.
const anchor = strength
? cursorAnchorReport(
rebuiltZooms.filter((z) => landed.has(z.id)),
document,
strength.focus,
options?.cursorTelemetry,
)
: undefined;
return {
ok: true,
document: next,
resultJson: JSON.stringify({
zoomId: landing.ids[0] ?? zoomId,
...(strength
? {
depth: strength.depth,
renderedScale: effectiveZoomScale(strength),
...(strength.rotationPreset ? { rotationPreset: strength.rotationPreset } : {}),
...zoomBehaviourReport(strength, document),
}
: {}),
...(clearsCustomScale ? { clearedCustomScale: true } : {}),
...landingReport(landing, startMs / 1000, endMs / 1000),
...(anchor ? { cursorAnchor: anchor } : {}),
...cutTransitionsReport(document, next),
}),
summary:
`updated zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
(strength ? ` at ${effectiveZoomScale(strength).toFixed(2)}×` : "") +
(clearsCustomScale ? " (cleared its custom scale so the depth applies)" : "") +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "addSpeed": {
const parsed = addSpeedArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));
const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));
const legacy = (document.legacyEditor as Record<string, unknown>) ?? {};
const prev = (legacy.speedRegions as unknown[] | undefined) ?? [];
const region = { id: createId("speed"), startMs, endMs, speed: parsed.data.speed };
const placed = anchorForAgent(region, document, "speed");
const landing = landingOf(placed, document);
if (!landing.anchored) {
return coversNoClip("speed region", startMs / 1000, endMs / 1000, document);
}
const next: AxcutDocument = {
...document,
legacyEditor: { ...legacy, speedRegions: [...prev, ...placed] },
};
return {
ok: true,
document: next,
resultJson: JSON.stringify({
speedId: landing.ids[0],
speed: region.speed,
...landingReport(landing, startMs / 1000, endMs / 1000),
}),
summary:
`added ${parsed.data.speed}× speed ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "setSpeed": {
const parsed = setSpeedArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const legacy = (document.legacyEditor as Record<string, unknown>) ?? {};
const prev =
(legacy.speedRegions as
| Array<{ id: string; startMs: number; endMs: number; speed: number }>
| undefined) ?? [];
const existing = prev.find((s) => s.id === parsed.data.speedId);
const speedPill = new Set(resolvePillIds(prev, parsed.data.speedId));
if (!existing) return failure(`Unknown speed region: ${parsed.data.speedId}`);
const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec);
const blocked = refuseIfBlocked("speed region", prev, parsed.data.speedId, {
startMs,
endMs,
});
if (blocked) return blocked;
const speed = parsed.data.speed ?? existing.speed;
const rebuiltSpeeds = replacePillSpan(
prev.map((s) => (speedPill.has(s.id) ? { ...s, speed } : s)),
parsed.data.speedId,
startMs,
endMs,
document.timeline.clips,
() => createId("speed"),
);
const landing = landingAfterPillEdit(prev, rebuiltSpeeds, speedPill, document);
if (!landing.anchored) {
return coversNoClip("speed region", startMs / 1000, endMs / 1000, document);
}
const next: AxcutDocument = {
...document,
legacyEditor: { ...legacy, speedRegions: rebuiltSpeeds },
};
return {
ok: true,
document: next,
resultJson: JSON.stringify({
speedId: landing.ids[0] ?? parsed.data.speedId,
speed,
...landingReport(landing, startMs / 1000, endMs / 1000),
}),
summary:
`updated speed to ${speed}× over ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "addAnnotation": {
const parsed = addAnnotationArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { type } = parsed.data;
const mismatch = refuseShapeMismatch(type, {
...parsed.data,
text: parsed.data.text || undefined,
});
if (mismatch) return mismatch;
const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));
const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));
// The editor's new text: its defaults, then whatever look the caller asked for, on the
// output frame with the box fitted to the text around x/y (the text's centre).
const text = placeTextOnFrame(
{
id: createId("ann"),
startMs,
endMs,
type: "text" as const,
content: parsed.data.text,
textContent: parsed.data.text,
position: { x: 0, y: 0 },
size: { width: 0, height: 0 },
style: applyAnnotationLook(
{
color: "#ffffff",
backgroundColor: DEFAULT_TEXT_PLATE,
fontSize: 32,
fontFamily: "Inter",
fontWeight: "bold",
fontStyle: "normal",
textDecoration: "none",
textAlign: "center",
textAnimation: "none",
},
parsed.data,
),
zIndex: Math.max(0, ...document.annotations.map((a) => a.zIndex)) + 1,
} as AxcutAnnotationRegion,
{ x: parsed.data.x, y: parsed.data.y },
document,
);
// An arrow or a blur is that text switched to its kind, as the inspector's type switch
// does it, then given its box around x/y and its data. Nothing to convert through: the
// box is set right after, in the kind's own reference box.
const ann =
type === "text"
? text
: shapeAnnotation(
{
...text,
...convertAnnotationKind(text, type, {
footage: null,
frameAspect: 0,
measure: measureTextInShippedFonts,
}),
},
{ x: parsed.data.x, y: parsed.data.y },
{
width: parsed.data.width ?? DEFAULT_ANNOTATION_BOX.width,
height: parsed.data.height ?? DEFAULT_ANNOTATION_BOX.height,
},
parsed.data,
args,
);
const placed = anchorForAgent(ann, document, "ann");
const landing = landingOf(placed, document);
if (!landing.anchored) {
return coversNoClip("annotation", startMs / 1000, endMs / 1000, document);
}
const next: AxcutDocument = {
...document,
annotations: [...document.annotations, ...placed] as AxcutDocument["annotations"],
};
return {
ok: true,
document: next,
resultJson: JSON.stringify({
annotationId: landing.ids[0],
...landingReport(landing, startMs / 1000, endMs / 1000),
...annotationForAgent(placed[0] as AxcutAnnotationRegion),
}),
summary:
(type === "text"
? `added annotation "${parsed.data.text.slice(0, 24)}"`
: `added ${type === "blur" ? "blur" : "arrow"}`) +
` ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "setAnnotation": {
const parsed = setAnnotationArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { annotationId } = parsed.data;
const existing = document.annotations.find((a) => a.id === annotationId);
const annPill = new Set(
resolvePillIds(document.annotations, annotationId, 0.001, { annotations: true }),
);
if (!existing) return failure(`Unknown annotation: ${annotationId}`);
const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec);
const { x, y, text, type, width, height } = parsed.data;
const mismatch = refuseShapeMismatch(type ?? existing.type, parsed.data);
if (mismatch) return mismatch;
const frameAspect = resolveAspectRatioValue(
document,
getEditorSettings(document).aspectRatio,
);
const edit = (original: AxcutAnnotationRegion): AxcutAnnotationRegion => {
// The inspector's type switch, through the footage under this very fragment: the box
// changes reference (a blur lives on the footage, the rest on the frame) without
// moving on screen, and the previous kind's content is parked, not lost.
const converted = type !== undefined && type !== original.type;
const a = converted
? {
...original,
...convertAnnotationKind(original, type, {
footage: annotationFootageRect(document, original.id),
frameAspect,
measure: measureTextInShippedFonts,
}),
}
: original;
const next: AxcutAnnotationRegion = {
...a,
...(text !== undefined ? { content: text, textContent: text } : {}),
style: applyAnnotationLook(a.style, parsed.data),
};
const centre = {
x: x ?? a.position.x + a.size.width / 2,
y: y ?? a.position.y + a.size.height / 2,
};
const moved = x !== undefined || y !== undefined;
if (a.type === "text") {
// A text box is the size of its text: refit it whenever the text, its size or its
// weight changes, around its centre or the new one. One saved before frame placement
// moves onto the frame the first time it is moved, as the editor does.
const resized =
text !== undefined ||
parsed.data.fontSize !== undefined ||
parsed.data.bold !== undefined;
return moved || (resized && a.space === "frame")
? placeTextOnFrame(next, centre, document)
: next;
}
const reshaped =
moved ||
converted ||
width !== undefined ||
height !== undefined ||
parsed.data.figureData !== undefined ||
parsed.data.blurData !== undefined;
if (!reshaped) return next;
// An image, an arrow or a blur keeps its size unless given one, and moves its centre,
// inside its own box.
return shapeAnnotation(
next,
centre,
{ width: width ?? a.size.width, height: height ?? a.size.height },
parsed.data,
args,
);
};
const rebuiltAnnotations = replacePillSpan(
document.annotations.map((a) => (annPill.has(a.id) ? edit(a) : a)),
annotationId,
startMs,
endMs,
document.timeline.clips,
() => createId("ann"),
0.001,
{},
{ annotations: true },
);
const landing = landingAfterPillEdit(
document.annotations,
rebuiltAnnotations,
annPill,
document,
);
if (!landing.anchored) {
return coversNoClip("annotation", startMs / 1000, endMs / 1000, document);
}
const next: AxcutDocument = { ...document, annotations: rebuiltAnnotations };
const landed = rebuiltAnnotations.find((a) => a.id === landing.ids[0]);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
annotationId: landing.ids[0] ?? annotationId,
...landingReport(landing, startMs / 1000, endMs / 1000),
...(landed ? annotationForAgent(landed) : {}),
}),
summary:
`updated annotation ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "addLayoutChange": {
const parsed = addLayoutChangeArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { timeSec, preset } = parsed.data;
const run = layoutRuns(document).find((r) => timeSec >= r.start && timeSec < r.end);
if (!run) return failure("The layout change must be inside a clip.");
if (preset !== "no-webcam") {
const blind = noCameraUnderSpan(document, timeSec, Math.min(run.end, timeSec + 0.001));
if (blind) return blind;
}
const next = addLayoutChange(document, timeSec, preset);
if (next === document) return failure("This layout already applies here; nothing to change.");
return {
ok: true,
document: next,
resultJson: JSON.stringify({
layoutSegments: documentSnapshotForModel(next).layoutSegments,
}),
summary: `layout ${preset} from ${formatSec(timeSec)} until ${formatSec(run.end)}`,
};
}
case "setLayoutPreset": {
const parsed = setLayoutPresetArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const run = layoutRuns(document).find((r) => r.ids.includes(parsed.data.layoutId));
if (!run) return failure(`Unknown layout: ${parsed.data.layoutId}`);
if (parsed.data.preset !== "no-webcam") {
const blind = noCameraUnderSpan(document, run.start, run.end);
if (blind) return blind;
}
const next = setLayoutPreset(document, parsed.data.layoutId, parsed.data.preset);
if (next === document)
return failure("This passage already uses this layout; nothing to change.");
return {
ok: true,
document: next,
resultJson: JSON.stringify({
layoutSegments: documentSnapshotForModel(next).layoutSegments,
}),
summary: `changed layout to ${parsed.data.preset}`,
};
}
case "moveLayoutBoundary": {
const parsed = moveLayoutBoundaryArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const runs = layoutRuns(document);
const index = runs.findIndex((r) => r.ids.includes(parsed.data.layoutId));
if (index <= 0)
return failure("Only an existing change after the first passage can be moved.");
const next = moveLayoutBoundary(document, parsed.data.layoutId, parsed.data.timeSec);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
layoutSegments: documentSnapshotForModel(next).layoutSegments,
}),
summary: "moved layout change",
};
}
case "addCameraFullscreen": {
const parsed = addCameraFullscreenArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));
const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));
const region = { id: createId("camfull"), startMs, endMs };
const placed = anchorForAgent(region, document, "camfull");
const landing = landingOf(placed, document);
if (!landing.anchored) {
return coversNoClip("full-camera region", startMs / 1000, endMs / 1000, document);
}
const blind = noCameraUnderSpan(document, landing.startSec, landing.endSec);
if (blind) return blind;
const next = applyLayoutSpan(document, landing.startSec, landing.endSec, "camera-only");
return {
ok: true,
document: next,
resultJson: JSON.stringify({
cameraFullscreenId: layoutRuns(next).find(
(r) => r.member.preset === "camera-only" && r.end > landing.startSec,
)?.ids[0],
...landingReport(landing, startMs / 1000, endMs / 1000),
}),
summary:
`full-camera ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "addEffect": {
const parsed = addEffectArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { effectId, effect, values = {}, target = "image" } = parsed.data;
if ((effectId === undefined) === (effect === undefined)) {
return failure(
"Pass either `effect` to write a new effect, or `effectId` to reuse one of this project's (getCurrentDocument lists them under effectDefinitions), not both.",
);
}
let definition: AxcutShaderEffect;
let check: ShaderEffectCheck | null = null;
const created = effect !== undefined;
if (effect) {
const checked = checkedEffectDefinition(
{
id: createId("fx"),
...effect,
contract: SHADER_EFFECT_CONTRACT,
params: effect.params ?? [],
origin: "agent",
},
options?.checkShaderEffect,
);
if ("error" in checked) return failure(checked.error);
({ definition, check } = checked);
} else {
const existing = findShaderEffect(document, effectId as string);
if (!existing) return failure(`Unknown effect: ${effectId}.`);
definition = existing;
}
const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));
const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));
const blocked = effectBlocked(document.shaderEffectRanges ?? [], startMs, endMs);
if (blocked) return failure(`The effect was NOT placed: ${blocked}`);
const region = {
id: createId("fxr"),
startMs,
endMs,
effectId: definition.id,
values: storedShaderEffectValues(definition, values),
target,
};
const placed = anchorForAgent(region, document, "fxr");
const landing = landingOf(placed, document);
if (!landing.anchored) return coversNoClip("effect", startMs / 1000, endMs / 1000, document);
const next: AxcutDocument = {
...document,
shaderEffects: created
? [...(document.shaderEffects ?? []), definition]
: (document.shaderEffects ?? []),
shaderEffectRanges: [...(document.shaderEffectRanges ?? []), ...placed],
};
return {
ok: true,
document: next,
resultJson: JSON.stringify({
effectRegionId: landing.ids[0],
effectId: definition.id,
...landingReport(landing, startMs / 1000, endMs / 1000),
...(created ? { validated: check !== null } : {}),
...effectCostReport(definition, landing.startSec, landing.endSec),
}),
summary:
`effect "${definition.name}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "setEffect": {
const parsed = setEffectArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const prev = document.shaderEffectRanges ?? [];
const existing = prev.find((r) => r.id === parsed.data.effectRegionId);
if (!existing) return failure(`Unknown effect region: ${parsed.data.effectRegionId}`);
const definition = findShaderEffect(document, existing.effectId);
if (!definition) return failure(`Effect ${existing.effectId} is missing from the project.`);
const pillIds = new Set(resolvePillIds(prev, existing.id));
const pill = coalesceRegionsForRuler(prev).find((p) => p.ids.includes(existing.id));
const { values, target } = parsed.data;
const { startMs, endMs } = resolveSpanMs(
pill
? { startMs: Math.round(pill.start * 1000), endMs: Math.round(pill.end * 1000) }
: existing,
parsed.data.startSec,
parsed.data.endSec,
);
const blocked = effectBlocked(prev, startMs, endMs, pillIds);
if (blocked) return failure(`The effect was NOT changed: ${blocked}`);
const moved = parsed.data.startSec !== undefined || parsed.data.endSec !== undefined;
// The span is replaced before the patch: patched first, the pill could take the same
// values as a touching neighbour, coalesce with it, and move it along.
const spanned = moved
? replacePillSpan(prev, existing.id, startMs, endMs, document.timeline.clips, () =>
createId("fxr"),
)
: prev;
const others = new Set(prev.filter((r) => !pillIds.has(r.id)).map((r) => r.id));
const rebuilt = spanned.map((r) =>
others.has(r.id)
? r
: {
...r,
values:
values === undefined
? r.values
: storedShaderEffectValues(definition, { ...r.values, ...values }),
target: target ?? r.target,
},
);
const landing = landingAfterPillEdit(prev, rebuilt, pillIds, document);
if (!landing.anchored) return coversNoClip("effect", startMs / 1000, endMs / 1000, document);
const next: AxcutDocument = { ...document, shaderEffectRanges: rebuilt };
return {
ok: true,
document: next,
resultJson: JSON.stringify({
effectRegionId: landing.ids[0] ?? existing.id,
effectId: definition.id,
...landingReport(landing, startMs / 1000, endMs / 1000),
...effectCostReport(definition, landing.startSec, landing.endSec),
}),
summary:
`updated effect "${definition.name}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "setEffectDefinition": {
const parsed = setEffectDefinitionArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { effectId, ...patch } = parsed.data;
const existing = findShaderEffect(document, effectId);
if (!existing) return failure(`Unknown effect: ${effectId}.`);
const recompile = patch.code !== undefined || patch.params !== undefined;
const candidate = {
...existing,
...Object.fromEntries(Object.entries(patch).filter(([, v]) => v !== undefined)),
cost: recompile ? undefined : existing.cost,
};
let definition: AxcutShaderEffect;
let check: ShaderEffectCheck | null = null;
if (recompile) {
const checked = checkedEffectDefinition(candidate, options?.checkShaderEffect);
if ("error" in checked) return failure(checked.error);
({ definition, check } = checked);
} else {
const checked = shaderEffectSchema.safeParse(candidate);
if (!checked.success) return failure(checked.error.message);
definition = checked.data;
}
// The regions follow the new parameters: a removed one's value drops, bounds re-clamp.
const next: AxcutDocument = {
...document,
shaderEffects: (document.shaderEffects ?? []).map((e) =>
e.id === effectId ? definition : e,
),
shaderEffectRanges: (document.shaderEffectRanges ?? []).map((r) =>
r.effectId === effectId
? { ...r, values: storedShaderEffectValues(definition, r.values) }
: r,
),
};
const regions = coalesceForAgent(
(next.shaderEffectRanges ?? []).filter((r) => r.effectId === effectId),
);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
effectId,
regions: regions.length,
...(recompile ? { validated: check !== null } : {}),
...(definition.cost ? { cost: definition.cost } : {}),
}),
summary: `updated effect "${definition.name}" (${regions.length} region${regions.length === 1 ? "" : "s"})`,
};
}
case "getEffect": {
const parsed = getEffectArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const effect = findShaderEffect(document, parsed.data.effectId);
if (!effect) return failure(`Unknown effect: ${parsed.data.effectId}.`);
const regions = coalesceForAgent(
(document.shaderEffectRanges ?? []).filter((r) => r.effectId === effect.id),
).map((r) => ({
id: r.id,
startSec: roundSec(r.startMs),
endSec: roundSec(r.endMs),
target: r.target,
values: r.values,
}));
return {
ok: true,
resultJson: JSON.stringify({ ...effect, regions }),
summary: `read effect "${effect.name}"`,
};
}
case "setCameraFullscreen": {
const parsed = setCameraFullscreenArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const runs = layoutRuns(document);
const index = runs.findIndex(
(r) => r.member.preset === "camera-only" && r.ids.includes(parsed.data.cameraFullscreenId),
);
const existing = runs[index];
if (!existing)
return failure(`Unknown full-camera region: ${parsed.data.cameraFullscreenId}`);
const { startMs, endMs } = resolveSpanMs(
{ startMs: existing.start * 1000, endMs: existing.end * 1000 },
parsed.data.startSec,
parsed.data.endSec,
);
const placed = anchorForAgent({ id: createId("layout"), startMs, endMs }, document, "layout");
const landing = landingOf(placed, document);
if (!landing.anchored) {
return coversNoClip("full-camera region", startMs / 1000, endMs / 1000, document);
}
const blindMove = noCameraUnderSpan(document, landing.startSec, landing.endSec);
if (blindMove) return blindMove;
const base = applyLayoutSpan(
document,
existing.start,
existing.end,
runs[index - 1]?.member.preset ?? defaultLayoutPreset(document),
);
const next = applyLayoutSpan(base, landing.startSec, landing.endSec, "camera-only");
return {
ok: true,
document: next,
resultJson: JSON.stringify({
cameraFullscreenId: layoutRuns(next).find(
(r) => r.member.preset === "camera-only" && r.end > landing.startSec,
)?.ids[0],
...landingReport(landing, startMs / 1000, endMs / 1000),
}),
summary:
`moved full-camera to ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "addAudio": {
const parsed = addAudioArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { assetId, kind, offsetSec, gainDb, fadeInMs, fadeOutMs } = parsed.data;
const asset = document.assets.find((a) => a.id === assetId);
// Two distinct refusals, because they need two different corrections: an
// unknown id is a hallucinated asset, a video id is the model reaching for
// footage. Naming the audio the project HAS is what stops the retry loop.
if (!asset) {
const available = document.assets.filter((a) => a.kind === "audio");
return failure(
`Unknown asset: ${assetId}.` +
(available.length
? ` Imported audio in this project: ${available.map((a) => `${a.id} (${a.label})`).join(", ")}.`
: " This project has no imported audio: a file has to be imported into it first (importMedia, where it is offered)."),
);
}
if (asset.kind !== "audio") {
return failure(
`Asset ${assetId} is video, not audio. addAudio plays an imported audio file over the recording; to place footage on the timeline use insertClip.`,
);
}
const durationSec = asset.durationSec ?? 0;
// "Start the file at offsetSec" is only answerable when there is file left
// there. Past the end it yields a track that plays silence, which the model
// then reports as having placed audio. Unknown duration is not a refusal: an
// import whose probe failed carries 0 until the renderer re-probes it.
if (durationSec > 0 && offsetSec >= durationSec) {
return failure(
`offsetSec ${offsetSec}s is at or past the end of ${assetId} (${durationSec}s), so the track would play nothing. Pick an offset inside the file.`,
);
}
// No endSec means "as long as the file is" — the natural span, and the one
// the editor's own add uses, so the model never has to compute it.
const startSec = parsed.data.startSec;
const endSec =
parsed.data.endSec ??
startSec + Math.max(0.1, (durationSec || DEFAULT_AGENT_AUDIO_SEC) - offsetSec);
const startMs = toMs(Math.min(startSec, endSec));
const endMs = toMs(Math.max(startSec, endSec));
const trackId = createId("audio");
const withTrack = placeAudioTrackInDocument(
document,
{
id: trackId,
trackId,
startMs,
endMs,
assetId,
kind,
durationSec,
offsetMs: toMs(offsetSec),
gainDb,
loop: false,
fadeInMs,
fadeOutMs,
muted: false,
label: asset.label,
origin: "agent",
} as AxcutDocument["audioTracks"][number],
() => createId("audio"),
"create",
);
if (withTrack === document) {
return coversNoClip("audio", startMs / 1000, endMs / 1000, document);
}
const placed = withTrack.audioTracks.filter((t) => trackGroupId(t) === trackId);
const next: AxcutDocument = withTrack;
const landing = landingOf(placed, document);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
audioId: trackId,
...landingReport(landing, startMs / 1000, endMs / 1000),
}),
summary:
`added ${kind} "${asset.label}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
landingSuffix(landing, startMs / 1000, endMs / 1000),
};
}
case "setAudio": {
const parsed = setAudioArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { audioId } = parsed.data;
const pill = collapseTracksToPills(document.audioTracks).find(
(t) => trackGroupId(t) === audioId,
);
if (!pill) return failure(`Unknown audio track: ${audioId}`);
if (parsed.data.offsetSec !== undefined) {
const asset = document.assets.find((a) => a.id === pill.assetId);
const durationSec = asset?.durationSec ?? 0;
if (durationSec > 0 && parsed.data.offsetSec >= durationSec) {
return failure(
`offsetSec ${parsed.data.offsetSec}s is at or past the end of ${pill.assetId} (${durationSec}s), so the track would play nothing.`,
);
}
}
// Payload first, through the helper that keeps every fragment of the group in
// agreement — gain, mute, loop and the offset are all track-wide, and a patch
// that reached only one fragment would split the pill in two.
let next = patchAudioTrack(document, audioId, {
...(parsed.data.gainDb !== undefined ? { gainDb: parsed.data.gainDb } : {}),
...(parsed.data.muted !== undefined ? { muted: parsed.data.muted } : {}),
...(parsed.data.loop !== undefined ? { loop: parsed.data.loop } : {}),
...(parsed.data.fadeInMs !== undefined ? { fadeInMs: parsed.data.fadeInMs } : {}),
...(parsed.data.fadeOutMs !== undefined ? { fadeOutMs: parsed.data.fadeOutMs } : {}),
...(parsed.data.offsetSec !== undefined ? { offsetMs: toMs(parsed.data.offsetSec) } : {}),
});
// A span or lane change re-anchors: drop the group and lay it down again, so
// the fragments are re-cut against the clips the new span covers rather than
// patched in place against the old ones.
const wantsRespan =
parsed.data.startSec !== undefined ||
parsed.data.endSec !== undefined ||
parsed.data.kind !== undefined;
if (wantsRespan) {
const current =
collapseTracksToPills(next.audioTracks).find((t) => trackGroupId(t) === audioId) ?? pill;
const { startMs, endMs } = resolveSpanMs(current, parsed.data.startSec, parsed.data.endSec);
// A `kind` flip re-clamps against the DESTINATION lane's neighbours, not the
// one it is leaving — moving a take onto the music row must respect what is
// already on the music row (issue #560).
const moved = placeAudioTrackInDocument(
next,
{
...current,
id: audioId,
trackId: audioId,
startMs,
endMs,
...(parsed.data.kind !== undefined ? { kind: parsed.data.kind } : {}),
},
() => createId("audio"),
"move",
);
if (moved === next) {
return coversNoClip("audio", startMs / 1000, endMs / 1000, document);
}
next = moved;
}
const after = collapseTracksToPills(next.audioTracks).find(
(t) => trackGroupId(t) === audioId,
);
return {
ok: true,
document: next,
resultJson: JSON.stringify({
audioId,
startSec: roundSec(after?.startMs ?? pill.startMs),
endSec: roundSec(after?.endMs ?? pill.endMs),
fadeInMs: after?.fadeInMs ?? pill.fadeInMs,
fadeOutMs: after?.fadeOutMs ?? pill.fadeOutMs,
}),
summary: `updated audio ${audioId} ${formatSec(roundSec(after?.startMs ?? pill.startMs))} – ${formatSec(roundSec(after?.endMs ?? pill.endMs))}`,
};
}
case "removeTrim": {
const parsed = removeTrimArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { trimRangeId } = parsed.data;
if (!document.timeline.trimRanges.some((s) => s.id === trimRangeId)) {
return failure(`Unknown trim range: ${trimRangeId}`);
}
const next = removeRegion(document, "trim", trimRangeId);
return {
ok: true,
document: next,
resultJson: JSON.stringify({ removed: trimRangeId, kind: "trim" }),
summary: `removed trim ${trimRangeId}`,
};
}
case "removeModifier": {
const parsed = removeModifierArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { id } = parsed.data;
const legacy = (document.legacyEditor as Record<string, unknown>) ?? {};
const speedRegions = (legacy.speedRegions as Array<{ id: string }> | undefined) ?? [];
const cameraFullscreenRegions =
(legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? [];
let kind: ModifierKind | null = null;
if (document.zoomRanges.some((z) => z.id === id)) kind = "zoom";
else if (document.annotations.some((a) => a.id === id)) kind = "annotation";
else if (speedRegions.some((s) => s.id === id)) kind = "speed";
else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen";
else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio";
else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect";
else if (layoutRuns(document).some((r) => r.ids.includes(id))) kind = "layout";
if (kind === "layout" && layoutRuns(document)[0]?.ids.includes(id)) {
return failure("The first layout cannot be removed. Change its preset instead.");
}
if (!kind) {
return failure(
`No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` +
`For a trim use removeTrim; for a clip use removeClip.`,
);
}
const beforeIds = modifierIds(document, kind);
const next = removeRegion(document, kind, id);
const remainingIds = new Set(modifierIds(next, kind));
const removedIds = beforeIds.filter((candidateId) => !remainingIds.has(candidateId));
return {
ok: true,
document: next,
resultJson: JSON.stringify({ removed: id, removedIds, kind }),
summary:
removedIds.length === 1
? `removed ${kind} ${id}`
: `removed ${removedIds.length} ${kind} rows: ${removedIds.join(", ")}`,
};
}
case "removeClip": {
const parsed = removeClipArgs.safeParse(args);
if (!parsed.success) return failure(parsed.error.message);
const { clipId } = parsed.data;
if (!document.timeline.clips.some((c) => c.id === clipId)) {
return failure(`Unknown clip: ${clipId}`);
}
const next = removeClip(document, clipId);
const dropped = droppedByEdit(document, next);
const casualties = dropped.droppedModifierIds.length + dropped.droppedTrimIds.length;
return {
ok: true,
document: next,
resultJson: JSON.stringify({
removed: clipId,
clipCount: next.timeline.clips.length,
...dropped,
}),
summary:
`removed clip ${clipId}` +
(casualties > 0
? ` — dropped ${[...dropped.droppedModifierIds, ...dropped.droppedTrimIds].join(", ")}`
: ""),
};
}
default:
return failure(`Unknown tool: ${name}`);
}
}