Open source CapCut alternative.
TypeScript
"use client";
import { formatBytes } from "@/lib/bytes";
import { isLinkedAssetType, isTimelineId, parkedTimeline, sanitizeTimelines, timelineAspect, TIMELINE_IDS, TIMELINE_LABELS, type TimelineBody, type TimelineId } from "./types";
import { FIND_MEDIA_SNAPSHOT_CAP } from "./findMedia";
import { GUIDE_PRESETS, guideFits, guideGeometry, safeAreaOf, sanitizeGuideLines, sanitizeGuides, type GuideId, type GuideLines } from "./guides";
import { hasOverlayAnim, retimeOf, speedCurveOf, WHEEL_LABELS, WHEEL_ZONES, type ClipSound, type OutputSpace, type SpeedNode } from "@donkeycut/effects-kit";
import { chatOwner } from "./chatAssets";
import { useGenerate } from "./generate";
import { matteBakesAvailable, useMatteBakes } from "./removal/bakeJobs";
import { cardMatteKey, resolveCardShape } from "./cameraCard";
import { assetIdsOnAnyTimeline, deriveTransitionFields, getClipSpans, normalizeDocState, overlayLayers, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";
import type { VideoProject } from "./genvideo/types";
import { playheadAt, skimAt } from "./playhead";
import { cueWordCount } from "./cueChunk";
import { laneCues, subtitleLaneCount } from "./subtitles";
import { emphasizedWords } from "./captionEmphasis";
import { watchSweepActive } from "./watch/sweep";
import { libraryFontId, listLibraryFonts, listLutChoices } from "./linkedLibrary";
import { savedGradesKnown } from "./gradePresets";
import { sourceProfileOf } from "./baseFrame";
import { groupCameras } from "./groupCamera";
import { isOffline } from "./network";
import {
clipZoom,
frameOf,
rectOf,
regionLabel,
uploadedFontId,
type Aspect,
type AudioClip,
type ClipSpan,
type MediaAsset,
type MediaFolder,
overlayName,
type Overlay,
type ProjectDoc,
type SubtitleCue,
type SubtitlesBlock,
type TimelineTransition,
type VideoClip,
} from "./types";
const r = (n: number) => Math.round(n * 100) / 100;
/** Element times to the millisecond: a 60 fps frame is 0.0167 s, which two
* decimals blur. editor_state and the tool results both read times through it. */
export const roundT = (n: number) => Math.round(n * 1000) / 1000;
/** Frame fractions to the 1/10,000: a pixel of a 1080 frame is 0.0009, which
* two decimals blur by up to five pixels. */
const rF = (n: number) => Math.round(n * 10000) / 10000;
/** A box as the chat reads and writes every box: x/y its centre and w/h its
* size, in frame fractions. The store keeps a top-left corner; the chat never
* sees one, so a measured box, a safe area and a layer's region all mean the
* same numbers. */
export function centreBox(b: { x: number; y: number; w: number; h: number }) {
return { x: rF(b.x + b.w / 2), y: rF(b.y + b.h / 2), w: rF(b.w), h: rF(b.h) };
}
/** The transition this clip applies into the next one, clamped to the live
* overlap the same way the preview and export are. Null when there's no
* transition or no next clip. */
function transitionToNext(sp: ClipSpan, index: number, spans: ClipSpan[]) {
const seconds = sp.clip.transition ?? 0;
if (seconds <= 0 || index >= spans.length - 1) return null;
const style = sp.clip.transitionStyle ?? "crossfade";
return {
style,
seconds: roundT(sp.transitionOut),
...(sp.clip.transitionFeather ? { feather: sp.clip.transitionFeather } : {}),
...(sp.clip.transitionBand ? { band: sp.clip.transitionBand } : {}),
};
}
/** The clip's own effects — entrance/exit animations, box styling, and its
* color state in the shape the grading tools take: the LUT and the preset
* refs plus a compact summary of the manual grade (nonzero sliders verbatim;
* curves as presence flags; the wheels by their tool names; the touched hue
* bands by name). */
function clipEffects(clip: VideoClip, live: boolean, frame?: CardFrame) {
const grade = clip.grade;
const sliders: Record<string, number> = {};
for (const [k, v] of Object.entries(grade ?? {})) {
if (typeof v === "number" && v !== 0) sliders[k] = v;
}
const color = grade
? {
...(grade.lut ? { lut: { id: grade.lut.id, amount: r(grade.lut.amount ?? 1) } } : {}),
...(grade.preset
? {
colorPreset: {
id: grade.preset.id,
amount: r(grade.preset.amount ?? 1),
...(grade.preset.skin ? { protectSkin: true } : {}),
},
}
: {}),
...(Object.keys(sliders).length ? { grade: sliders } : {}),
...(grade.curves ? { curves: Object.keys(grade.curves) } : {}),
...(grade.wheels
? { wheels: WHEEL_ZONES.filter((z) => grade.wheels?.[z]).map((z) => WHEEL_LABELS[z].toLowerCase()) }
: {}),
...(grade.hsl ? { hslBands: Object.keys(grade.hsl) } : {}),
}
: {};
return {
...(clip.animIn ? { animIn: { style: clip.animIn.style, seconds: roundT(clip.animIn.seconds) } } : {}),
...(clip.animOut
? { animOut: { style: clip.animOut.style, seconds: roundT(clip.animOut.seconds) } }
: {}),
...color,
...(clip.effects?.length ? { effects: clip.effects } : {}),
...(clip.boxStyle ? { boxStyle: clip.boxStyle } : {}),
...describeRemoval(clip, live),
...(clip.card ? { card: describeCameraCard(clip, live, frame) } : {}),
};
}
/** What a camera card's defaults read: the frame shape and the clip's
* source. */
interface CardFrame {
aspect: Aspect;
assetById: Map<string, MediaAsset>;
}
/** The clip's camera-card layout as set_camera_card speaks it — the side,
* top and width it shows at, with `auto` naming the ones that follow the
* frame shape and the source — with the head's person matte: "ready", a live bake's progress or error, "off" when
* the card hides the head, or owed until the editor bakes it — what a claim
* about the head showing must rest on. */
export function describeCameraCard(clip: VideoClip, live: boolean, frame?: CardFrame) {
const c = clip.card;
if (!c) return null;
const fr = frameOf(frame?.aspect ?? "9:16");
const rect = rectOf(clip);
const asset = frame?.assetById.get(clip.assetId);
const shape = resolveCardShape(c, rect.w * fr.w, rect.h * fr.h, asset?.width ?? 0, asset?.height ?? 0);
const auto = [
...(c.side === undefined ? ["side"] : []),
...(c.top === undefined ? ["top"] : []),
...(shape.side !== "bottom" && c.width === undefined ? ["width"] : []),
];
const job = live ? useMatteBakes.getState().jobs[cardMatteKey(clip.id)] : undefined;
const matte = !c.popOut
? "off"
: c.matte
? "ready"
: job?.status === "running"
? `baking (${Math.round(job.progress * 100)}%)`
: job?.status === "error"
? `error: ${job.error ?? "failed"}`
: live && matteBakesAvailable
? "starting"
: "owed: bakes when the project is open in the editor";
return {
side: shape.side,
top: r(shape.top),
...(shape.side !== "bottom" ? { width: r(shape.width) } : {}),
...(auto.length ? { auto } : {}),
radius: c.radius,
side_bleed: c.sideBleed,
pop_out: c.popOut,
feather: c.feather,
shadow: r(c.shadow),
...(c.scale !== undefined ? { scale: r(c.scale) } : {}),
...(c.offsetX !== undefined ? { offset_x: r(c.offsetX) } : {}),
...(c.offsetY !== undefined ? { offset_y: r(c.offsetY) } : {}),
matte,
};
}
/** The clip's cutout state: mode, whether its AI matte is ready (and at which
* tier), a live bake's progress, and the stroke/background riding on it —
* what set_removal claims must be grounded in. */
function describeRemoval(clip: VideoClip, live: boolean) {
const rm = clip.removal;
if (!rm) return {};
const job = live ? useMatteBakes.getState().jobs[clip.id] : undefined;
return {
removal: {
mode: rm.mode,
...(rm.off ? { off: true } : {}),
removes: rm.invert ? "subject" : "background",
...(rm.subject ? { subject: rm.subject } : {}),
matte: rm.matte ? `ready (${rm.matte.quality === "hq" ? "quality" : "quick"})` : "none",
...(job
? {
bake:
job.status === "running"
? `baking ${job.quality === "hq" ? "quality" : "quick"} (${Math.round(job.progress * 100)}%)`
: `error: ${job.error ?? "failed"}`,
}
: {}),
...(rm.stroke
? {
stroke: {
style: rm.stroke.style,
color: rm.stroke.color,
...(rm.stroke.width !== undefined ? { width: rm.stroke.width } : {}),
},
}
: {}),
...(rm.backdrop && rm.backdrop.kind !== "none" ? { background: rm.backdrop } : {}),
},
};
}
/** How much written record the per-message snapshot carries before it hands
* the rest to get_state. */
const NOTE_SNAPSHOT_BUDGET = 3000;
/** An asset's watch notes shaped for the context: whole when they fit (and
* whenever the caller asked for everything), otherwise each note shortened to
* an even share of the budget so every span still says what it holds, with
* `observedText` naming where the rest is. */
function noteFields(
notes: { from: number; to: number; text: string }[] | undefined,
full: boolean
): Record<string, unknown> {
if (!notes || notes.length === 0) return {};
const size = notes.reduce((sum, n) => sum + n.text.length, 0);
if (full || size <= NOTE_SNAPSHOT_BUDGET)
return { observed: notes.map((n) => ({ from: roundT(n.from), to: roundT(n.to), text: n.text })) };
const share = Math.max(120, Math.floor(NOTE_SNAPSHOT_BUDGET / notes.length));
let trimmed = false;
const observed = notes.map((n) => {
if (n.text.length <= share) return { from: roundT(n.from), to: roundT(n.to), text: n.text };
trimmed = true;
return { from: roundT(n.from), to: roundT(n.to), text: `${n.text.slice(0, share).trimEnd()}…` };
});
return { observed, ...(trimmed ? { observedText: "shortened here, whole in get_state" } : {}) };
}
/**
* Compact JSON snapshot of everything the assistant should know: the cut,
* the selection, what's on screen, and every user-facing setting. Sent with
* each message and served by the get_state tool.
*
* `fullCues` includes the entire transcript. The per-message snapshot leaves
* it off (a long transcript would inflate every turn's token cost, even ones
* that never touch captions); the get_state tool passes it so the model can
* pull every cue on demand.
*
* Each chat is independent: it sees the shared project, the Media panel, and
* the Library, but never another chat's private media or renders. `chatId`
* scopes that to the asking thread — an explicit id, else the live turn's (or
* open panel's) owner. Media and jobs a *different* chat still owns are left
* out; once media is placed on the timeline, filed into Media, or copied to
* the Library it stops being chat-private and shows to every chat.
*/
export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | null }) {
const s = useEditor.getState();
const chatId = opts?.chatId !== undefined ? opts.chatId : chatOwner();
const core = describeState(s, { fullCues: opts?.fullCues === true, chatId, live: true });
const spans = getClipSpans(s.clips, s.assets);
const assetById = new Map(s.assets.map((a) => [a.id, a]));
const selection = (() => {
if (!s.selection) return null;
const { kind, id } = s.selection;
if (kind === "clip") {
const sp = spans.find((x) => x.clip.id === id);
if (sp) {
return {
kind,
id,
asset: sp.asset.name,
start: roundT(sp.start),
len: roundT(sp.len),
muted: sp.clip.muted,
...(sp.clip.hidden ? { hidden: true } : {}),
...describeRate(sp.clip),
};
}
// A layer clip carries no span (spans are track 0); describe its
// compositing shape instead.
const c = s.clips.find((x) => x.id === id);
return c ? { kind, id, ...describeOverlayClip(c, assetById) } : { kind, id };
}
if (kind === "audio") {
const a = s.audioClips.find((x) => x.id === id);
return a ? { kind, id, ...describeAudio(a, assetById) } : { kind, id };
}
if (kind === "cue") {
const c = s.subtitles.cues.find((x) => x.id === id);
return c
? { kind, id, text: c.text, start: roundT(c.start), end: roundT(c.end), track: c.lane ?? 0, ...cueExtras(c) }
: { kind, id };
}
// The selection's kind stays "overlay" (what select/delete take); the
// element's own kind rides in from describeOverlay as `element`.
const o = s.overlays.find((x) => x.id === id);
if (!o) return { kind, id };
const { kind: element, ...rest } = describeOverlay(o);
return { kind, id, element, ...rest };
})();
return {
project: core.project,
playhead: roundT(playheadAt()),
skimmer: skimAt() === null ? null : roundT(skimAt()!),
playing: s.playing,
selection,
// Every font a title or caption can be set in beyond the built-in list:
// the account's own, off the Library shelf, plus any still living in this
// project from before fonts moved to the shelf.
fonts: [
...listLibraryFonts().map((f) => ({ id: libraryFontId(f.key), label: f.label })),
...s.assets
.filter((a) => a.type === "font")
.map((a) => ({ id: uploadedFontId(a.id), label: a.name })),
],
// Every LUT set_color_lut takes, by id: the built-in film looks and the
// user's own files in the Library. The grades saved there, by name, for
// apply_saved_grade.
luts: listLutChoices().map((l) => ({ id: l.id, label: l.label, ...(l.builtIn ? { builtIn: true } : {}) })),
savedGrades: savedGradesKnown().map((g) => ({ id: g.id, name: g.name })),
...(core.mediaFolders ? { mediaFolders: core.mediaFolders } : {}),
media: core.media,
mediaTruncated: core.mediaTruncated,
...(core.mediaInOtherChats ? { mediaInOtherChats: core.mediaInOtherChats, mediaInOtherChatsNote: core.mediaInOtherChatsNote } : {}),
// The browser has no connection: hosted work (renders, cloud reads,
// uploads) waits for it, and wait_for_uploads waits with it.
...(isOffline() ? { offline: true } : {}),
// AI video renders for this project, live from the job store — what
// "rendering" claims must be grounded in. A done render names the asset
// it landed as (already in `media`); a failed one carries its error.
// Renders another chat launched stay out; chat-less panel renders show to
// every chat as shared project activity.
renders: useGenerate
.getState()
.jobs.filter(
(j) =>
j.kind === "video" &&
j.projectId === s.projectId &&
!(j.chatId && j.chatId !== chatId)
)
.slice(0, 8)
.map((j) => ({
jobId: j.id,
prompt: j.prompt.length > 80 ? `${j.prompt.slice(0, 77)}…` : j.prompt,
status: j.status,
...(j.status === "running"
? { elapsedSec: Math.round((Date.now() - j.startedAt) / 1000) }
: {}),
...(j.status === "done" && j.assetId ? { assetId: j.assetId } : {}),
...(j.status === "error" && j.error ? { error: j.error } : {}),
})),
videoTrack: core.videoTrack,
overlayVideo: core.overlayVideo,
transitions: core.transitions,
soundtrack: core.soundtrack,
overlays: core.overlays,
subtitles: core.subtitles,
publish: s.publish,
// The Details tab's notes: where the cut keeps its source links.
...(s.notes.text || s.notes.publishedAt || s.notes.links.some(Boolean) ? { notes: s.notes } : {}),
view: {
pxPerSec: r(s.pxPerSec),
timelineH: s.timelineH,
exportDialogOpen: s.exportOpen,
},
};
}
/** A parked timeline's items, described the way the open one's are: what
* get_state reads with `timeline`, so the chat can copy an item another
* timeline holds without opening it. */
export function describeParkedTimeline(id: TimelineId) {
const s = useEditor.getState();
const body = parkedTimeline(s, id);
const core = describeState({ ...s, ...body, timeline: id }, { fullCues: true, chatId: chatOwner(), live: true });
return {
timeline: id,
aspect: body.aspect,
duration: roundT(projectDuration(body)),
videoTrack: core.videoTrack,
overlayVideo: core.overlayVideo,
transitions: core.transitions,
soundtrack: core.soundtrack,
overlays: core.overlays,
...("groupCameras" in core ? { groupCameras: core.groupCameras } : {}),
subtitles: core.subtitles,
};
}
/** A stored document described the way the open project is: the same
* shape the model reads in <editor_state>, built from the document alone.
* This is how another project is read as a reference — its edit, its
* sources with their notes and transcripts, its caption look — and it goes
* through the store's own normalization so an old document reads like a
* new one. Live-only fields (playhead, selection, renders, view) stay out. */
export function describeDoc(doc: ProjectDoc, opts?: { fullCues?: boolean; chatId?: string | null }) {
const assets: MediaAsset[] = (doc.assets ?? []).map((a) => ({ ...a, url: "" }));
const state = normalizeDocState(doc, assets);
const aspect = state.aspect ?? "9:16";
const core = describeState(
{
projectId: doc.id ?? null,
projectName: doc.name ?? "",
assets,
// The store writes each clip's blend into the clip as it lands; the
// description reads the same fields.
clips: deriveTransitionFields(state.clips, state.transitions),
transitions: state.transitions,
audioClips: state.audioClips,
overlays: state.overlays,
subtitles: state.subtitles,
subtitleLane: 0,
subtitleStatus: state.subtitles.cues.length > 0 ? "ready" : "idle",
aspect,
background: state.background,
colorSpace: state.colorSpace,
magnet: doc.magnet === true,
guides: sanitizeGuides(doc.guides),
guideLines: sanitizeGuideLines(doc.guideLines),
genvideo: doc.genvideo ?? undefined,
mediaFolders: doc.mediaFolders ?? [],
timeline: isTimelineId(doc.timeline) ? doc.timeline : "main",
timelines: sanitizeTimelines(doc.timelines, { aspect }) ?? {},
},
{ fullCues: opts?.fullCues === true, chatId: opts?.chatId ?? null, live: false }
);
const { mediaFolders, ...rest } = core;
return { ...rest, ...(mediaFolders ? { mediaFolders } : {}) };
}
/** The document-shaped part of the editor's state: what the store holds
* after opening a project, or a stored document normalized the same way. */
interface DescribedState {
projectId: string | null;
projectName: string;
assets: MediaAsset[];
clips: VideoClip[];
transitions: TimelineTransition[];
audioClips: AudioClip[];
overlays: Overlay[];
subtitles: SubtitlesBlock;
subtitleLane: number;
subtitleStatus: string;
aspect: Aspect;
background: string;
colorSpace: OutputSpace;
magnet: boolean;
guides: GuideId[];
guideLines: GuideLines;
genvideo?: VideoProject | null;
mediaFolders: MediaFolder[];
timeline: TimelineId;
timelines: Partial<Record<TimelineId, TimelineBody>>;
}
/** An asset still owned by another chat: chat-made, tagged to a different
* thread, and not yet moved onto the timeline. Placed media is project
* content, so it stays visible even when a chat made it. The same rule reads
* the open project and a reference, so a chat sees no more of a project by
* reading it as a reference than by opening it. */
export const hiddenFromChat = (
a: { origin?: string; chatId?: string; id: string },
chatId: string | null,
placed: Set<string>
): boolean => a.origin === "chat" && !!a.chatId && a.chatId !== chatId && !placed.has(a.id);
/** What a chat is told about media it can't see: that it exists, so it never
* says a file is missing from the project, and where it can be cleared. */
export const OTHER_CHATS_MEDIA_NOTE =
"Some project media belongs to another chat thread, which made it and hasn't placed it, so this chat can't see or delete it. Tell the user it is there: they can clear it from that thread, or delete that thread from the chat history, which removes the unplaced media it owns.";
/** `live` marks the open project: the sweep's progress, the matte bakes in
* flight, and another chat's private media are facts about this page, and
* a document read as a reference has none of them. */
function describeState(
s: DescribedState,
opts: { fullCues: boolean; chatId: string | null; live: boolean }
) {
const cueCap = opts.fullCues ? Infinity : 60;
const chatId = opts.chatId;
const placed = assetIdsOnAnyTimeline(s);
const ownedByOtherChat = (a: { origin?: string; chatId?: string; id: string }) =>
hiddenFromChat(a, chatId, placed);
// A font asset is a typeface, not footage: it is listed under `fonts` with
// the account's shelf fonts, where a font id is what the model needs.
const visibleAssets = s.assets.filter((a) => !isLinkedAssetType(a.type) && !ownedByOtherChat(a));
const otherChatsMedia = s.assets.filter((a) => !isLinkedAssetType(a.type) && ownedByOtherChat(a)).length;
// The cap trims the tail of a huge media list, and a source someone has
// watched and written up stays regardless of where it sits: the record is
// what later decisions are made from.
const mediaCap = opts.fullCues ? Infinity : FIND_MEDIA_SNAPSHOT_CAP;
const shownAssets =
visibleAssets.length <= mediaCap
? visibleAssets
: visibleAssets.filter((a, i) => i < mediaCap || (a.watch?.notes?.length ?? 0) > 0);
const spans = getClipSpans(s.clips, s.assets);
const duration = totalDuration(s.clips);
const assetById = new Map(s.assets.map((a) => [a.id, a]));
const subtitleTracks = subtitleLaneCount(s.subtitles);
// Scene-run lineage: which plan shot (1-based, what regenerate_shot takes)
// placed each clip, so "fix this clip" maps straight to a shot revision.
const shotByClip = new Map(
(s.genvideo?.shots ?? []).flatMap((sh, i) =>
sh.timelineClipId ? [[sh.timelineClipId, { n: i + 1, still: sh.status === "failed" }] as const] : []
)
);
const live = opts.live;
return {
project: {
id: s.projectId,
name: s.projectName,
duration: roundT(duration),
// The open timeline is what everything else here describes; the others
// are parked until switch_timeline opens one.
timeline: s.timeline,
timelines: TIMELINE_IDS.map((id) => {
const body = id === s.timeline ? s : s.timelines[id];
return {
id,
label: TIMELINE_LABELS[id],
...(id === s.timeline ? { open: true } : {}),
duration: roundT(body ? projectDuration(body) : 0),
clips: body?.clips.length ?? 0,
aspect: timelineAspect(s, id),
// A parked timeline's sound as it is mixed there, so "Main's music"
// is that clip with its trim and levels (copy_from_timeline).
...(id !== s.timeline && body?.audioClips.length
? { soundtrack: body.audioClips.map((a) => ({ id: a.id, ...describeAudio(a, assetById) })) }
: {}),
};
}),
aspect: s.aspect,
frame: `${frameOf(s.aspect).w}x${frameOf(s.aspect).h}`,
background: s.background,
// The delivery's color space (set_project_color): sdr, or hlg / pq
// for an HDR export and an HDR preview where the display carries it.
colorSpace: s.colorSpace,
// The magnet (set_magnet): on, track 0 stays closed up.
magnet: s.magnet,
// Every guide preset's safe area for this frame, whether or not it
// shows, so placement can respect a platform's UI without a guide on:
// safeArea and keepOut are boxes as the chat reads them (centreBox).
safeZones: Object.fromEntries(
GUIDE_PRESETS.filter((p) => p.id !== "custom" && guideFits(p.id, s.aspect)).flatMap((p) => {
const safeArea = safeAreaOf([p.id], s.aspect);
if (!safeArea) return [];
const keepOut = guideGeometry([p.id], s.aspect).boxes.map((b) => ({ label: b.label, ...centreBox(b) }));
return [[p.id, { safeArea: centreBox(safeArea), ...(keepOut.length ? { keepOut } : {}) }]];
})
),
// The guides showing on the preview and the user's own lines.
...(s.guides.length > 0
? {
guides: {
on: s.guides,
...(s.guides.includes("custom") ? { customLines: s.guideLines } : {}),
},
}
: {}),
},
// The Media panel's folders, when the project has any; each filed asset
// carries its folderId below (media_organize moves them).
mediaFolders:
s.mediaFolders.length > 0 ? s.mediaFolders.map((f) => ({ id: f.id, name: f.name })) : null,
// Every project asset visible to this chat, timeline-placed or not (media
// another chat still owns is filtered out above). `origin` marks Cut-made
// media (generated/voiceover/recording/stock/freeze/block); no origin = a user
// import shown in the Media panel.
media: shownAssets.map((a) => ({
id: a.id,
name: a.name,
type: a.type,
fileName: a.fileName,
...(a.sizeBytes !== undefined ? { sizeBytes: a.sizeBytes, fileSize: formatBytes(a.sizeBytes, "en") } : {}),
...(a.width !== undefined ? { width: a.width } : {}),
...(a.height !== undefined ? { height: a.height } : {}),
duration: roundT(a.duration),
...(a.origin ? { origin: a.origin } : {}),
...(a.folderId != null ? { folderId: a.folderId } : {}),
// A file on its way to storage plays and reads from the bytes in this
// browser; `upload` says how far it is, or what stopped it.
...(live && a.upload
? {
upload: a.upload.error
? { status: "failed", error: a.upload.error }
: { status: "uploading", percent: Math.round(a.upload.progress * 100) },
}
: {}),
// Where the picture sits inside a recorded frame (find_picture): the
// box the measuring tools read inside, centre and size as fractions.
...(a.picture && (a.picture.w < 1 || a.picture.h < 1) ? { picture: a.picture } : {}),
// The recording bound to this video (sync_audio): every clip of it
// plays that file, recording second = video second + offset.
...(a.soundFrom
? {
soundFrom: {
assetId: a.soundFrom.assetId,
name: s.assets.find((x) => x.id === a.soundFrom!.assetId)?.name ?? null,
offset: Math.round(a.soundFrom.offset * 1000) / 1000,
},
}
: {}),
// What the file's code values mean, for footage that is not plain
// Rec.709: the profile in force and, when the person overrode it, what
// the header read (set_source_color changes it).
...(sourceProfileOf(a) !== "rec709" || (a.color && a.color.detected !== "rec709")
? {
sourceColor: {
profile: sourceProfileOf(a),
...(a.color && a.colorProfile && a.colorProfile !== a.color.detected ? { detected: a.color.detected } : {}),
},
}
: {}),
// Source spans whose frame map exists (persisted with the project):
// distinct-moment times and cut candidates recorded by an earlier watch
// or by the background sweep. The map aims a watch — it is not seen
// footage; only sheets returned in this chat are. `watching` marks the
// sweep still filling the rest; the spans grow as segments land.
...(a.watch && a.watch.ranges.length > 0
? { mapped: a.watch.ranges.map((rg) => ({ from: roundT(rg.from), to: roundT(rg.to) })) }
: {}),
...(live && watchSweepActive(a.id) ? { watching: true } : {}),
// The written record of this source — what someone looked at these
// frames and wrote down (note_source). Unlike `mapped`, this IS seen
// footage, and it survives the contact sheets that showed it, so a
// source too long to hold in one conversation is decided from here.
// The snapshot rides every model call, so it carries the notes while
// they are small and hands the rest to get_state.
...noteFields(a.watch?.notes, opts.fullCues),
// The source's own transcript (built quietly by the sweep; no subtitle
// track involved). The snapshot carries the verdict; get_state carries
// the segments — read those for the words instead of inlining audio.
...(a.speech
? {
speech: a.speech.noSpeech ? "none" : "transcribed",
...(opts.fullCues && a.speech.segments.length > 0
? {
transcript: a.speech.segments.map((sg) => ({
start: roundT(sg.start),
end: roundT(sg.end),
text: sg.text,
})),
}
: {}),
}
: {}),
})),
mediaTruncated: shownAssets.length < visibleAssets.length,
...(live && otherChatsMedia > 0 ? { mediaInOtherChats: otherChatsMedia, mediaInOtherChatsNote: OTHER_CHATS_MEDIA_NOTE } : {}),
videoTrack: spans.map((sp, index) => ({
index,
id: sp.clip.id,
...(sp.clip.groupId ? { groupId: sp.clip.groupId } : {}),
asset: sp.asset.name,
...(sp.clip.name ? { name: sp.clip.name } : {}),
start: roundT(sp.start),
len: roundT(sp.len),
in: roundT(sp.clip.in),
out: roundT(sp.clip.out),
// A still has no source length; report its placed length instead of 0.
sourceDuration: roundT(sp.asset.type === "image" ? sp.len : sp.asset.duration),
// A blocked-out shot holds a slot and draws a flat colour — there is no
// picture here yet, and its label lives on the chip. Without this the
// label reads as a clip's name and a shell of a cut looks finished.
...(sp.asset.block ? { emptyShot: true } : {}),
muted: sp.clip.muted,
...(sp.clip.hidden ? { hidden: true } : {}),
// The clip's own equalizer/compressor/limiter, set through
// set_clip_sound; absent when its sound is untouched.
...(sp.clip.sound ? { sound: sp.clip.sound } : {}),
...splitEditOf(sp.clip),
framing: sp.clip.fit ?? "fit",
...describeRate(sp.clip),
// The generated scene shot this clip came from — sceneShot is the
// 1-based number regenerate_shot takes; heldStill marks a render that
// fell back to its keyframe.
...(() => {
const sh = shotByClip.get(sp.clip.id);
return sh ? { sceneShot: sh.n, ...(sh.still ? { sceneShotHeldStill: true } : {}) } : {};
})(),
// Track 0 is free-positioned: empty stretches play black.
...(() => {
const prevEnd = index === 0 ? 0 : spans[index - 1].start + spans[index - 1].len;
return sp.start - prevEnd > 0.005 ? { gapBefore: roundT(sp.start - prevEnd) } : {};
})(),
...(() => {
const t = transitionToNext(sp, index, spans);
return t ? { transitionToNext: t } : {};
})(),
...(sp.clip.fit === "fill" || clipZoom(sp.clip) > 1
? {
...(clipZoom(sp.clip) > 1 ? { zoom: r(clipZoom(sp.clip)) } : {}),
panX: r(sp.clip.panX ?? 0),
panY: r(sp.clip.panY ?? 0),
}
: {}),
...(sp.clip.flipH ? { flipH: true } : {}),
...(sp.clip.flipV ? { flipV: true } : {}),
...(sp.clip.rotation ? { rotation: sp.clip.rotation } : {}),
...((sp.clip.opacity ?? 1) < 1 ? { opacity: r(sp.clip.opacity ?? 1) } : {}),
...(sp.clip.grade ? { colorGrade: sp.clip.grade } : {}),
...(sp.clip.mask ? { mask: sp.clip.mask } : {}),
...(sp.clip.kf?.length
? { keyframes: sp.clip.kf.map((k) => ({ ...k, t: roundT(k.t), x: rF(k.x), y: rF(k.y) })) }
: {}),
...(sp.clip.motionBlur ? { motionBlur: r(sp.clip.motionBlur) } : {}),
...clipEffects(sp.clip, live, { aspect: s.aspect, assetById }),
})),
// Video layers composited over track 0 in track order (the topmost
// full-frame clip covers the rest). Each track carries its own
// transitions, reported per clip like track 0's.
overlayVideo: [...new Set(overlayLayers(s.clips).map((c) => c.track))].flatMap((track) => {
const trackSpans = getClipSpans(s.clips, s.assets, track);
return trackSpans.map((sp, i) => ({
id: sp.clip.id,
...(sp.clip.groupId ? { groupId: sp.clip.groupId } : {}),
...describeOverlayClip(sp.clip, assetById),
...(() => {
const t = transitionToNext(sp, i, trackSpans);
return t ? { transitionToNext: t } : {};
})(),
...clipEffects(sp.clip, live, { aspect: s.aspect, assetById }),
}));
}),
// Every transition bar in the doc. A bar is its own object on the
// transitions row: it plays whatever cut or clip edge its window lines up
// with, and one lining up with nothing sits parked, playing nothing. The
// clip-level transitionToNext above only shows bars that landed on a cut,
// so this is where a parked leftover — the debris a move or a retime
// leaves — is visible and addressable by id.
transitions: (() => {
const roles = resolveTransitions(s.clips, s.transitions);
return [...s.transitions]
.sort((a, b) => a.start - b.start)
.map((t) => {
const plays = roles.get(t.id) ?? [];
return {
id: t.id,
...(t.groupId ? { groupId: t.groupId } : {}),
start: roundT(t.start),
seconds: roundT(t.seconds),
style: t.style,
...(t.feather ? { feather: t.feather } : {}),
...(t.band ? { band: t.band } : {}),
...(t.hidden ? { hidden: true } : {}),
...(plays.length > 0
? { plays: plays.map((p) => ({ at: p.kind, clipId: p.clipId })) }
: { parked: true }),
};
});
})(),
soundtrack: s.audioClips.map((a) => ({ id: a.id, ...describeAudio(a, assetById) })),
// Every overlay element on the title lanes: titles, shapes, stickers —
// each entry carries its `kind`.
overlays: s.overlays.map((o) => ({ id: o.id, ...describeOverlay(o) })),
// Each group's camera once, keys in seconds from the group's start
// (set_group_camera); the members carry only their groupId.
...(() => {
const cams = groupCameras(s.overlays);
return cams.length
? {
groupCameras: cams.map((c) => ({
groupId: c.groupId,
start: roundT(c.start),
keys: c.keys.map((k) => ({ ...k, t: roundT(k.t), x: rF(k.x), y: rF(k.y), scale: r(k.scale), rotation: r(k.rotation) })),
...(c.motionBlur ? { motionBlur: r(c.motionBlur) } : {}),
})),
}
: {};
})(),
subtitles: {
count: s.subtitles.cues.length,
showOnVideo: s.subtitles.showOnVideo,
showOnTimeline: s.subtitles.showOnTimeline,
// One language per track; the active track is what the panel edits and
// what generation/translation writes to.
activeTrack: s.subtitleLane,
tracks: Array.from({ length: subtitleTracks }, (_, i) => ({
track: i,
locale:
s.subtitles.tracks?.[i]?.locale ??
(i === 0 ? s.subtitles.locale ?? "en-US" : undefined),
cues: laneCues(s.subtitles, i).length,
...(s.subtitles.tracks?.[i]?.hidden ? { hidden: true } : {}),
})),
// How the captions READ: how many words one holds at a time. Changing it
// re-cuts every track (set_caption_look words_per_cue).
wordsPerCue: cueWordCount(s.subtitles),
// The look the captions wear, so a reference's caption treatment is
// legible without the cues: style preset, font, size, anchor, and the
// word effect.
...(s.subtitles.style ? { style: s.subtitles.style } : {}),
...(s.subtitles.font ? { font: s.subtitles.font } : {}),
...(s.subtitles.size !== undefined ? { size: s.subtitles.size } : {}),
...(s.subtitles.x !== undefined || s.subtitles.y !== undefined
? { x: rF(s.subtitles.x ?? 0.5), y: rF(s.subtitles.y ?? 0.5) }
: {}),
...(s.subtitles.wordHighlight ? { wordHighlight: true } : {}),
...(s.subtitles.accentMode ? { accentMode: s.subtitles.accentMode } : {}),
...(s.subtitles.accentColor ? { accentColor: s.subtitles.accentColor } : {}),
// The emphasis style, where it departs from the default
// (set_caption_look emphasis_*); the words wearing it ride on each cue.
...(s.subtitles.emphasisFont ? { emphasisFont: s.subtitles.emphasisFont } : {}),
...(s.subtitles.emphasisColor ? { emphasisColor: s.subtitles.emphasisColor } : {}),
...(s.subtitles.emphasisItalic !== undefined ? { emphasisItalic: s.subtitles.emphasisItalic } : {}),
...(s.subtitles.emphasisWeight !== undefined ? { emphasisWeight: s.subtitles.emphasisWeight } : {}),
...(s.subtitles.emphasisScale !== undefined ? { emphasisScale: s.subtitles.emphasisScale } : {}),
status: s.subtitleStatus,
// A window of cues by default; when truncated the model calls get_state
// for the whole transcript (e.g. "clean up all the captions").
cues: s.subtitles.cues.slice(0, cueCap).map((c) => ({
id: c.id,
...(c.groupId ? { groupId: c.groupId } : {}),
start: roundT(c.start),
end: roundT(c.end),
text: c.text,
...(subtitleTracks > 1 ? { track: c.lane ?? 0 } : {}),
...cueExtras(c),
})),
cuesTruncated: s.subtitles.cues.length > cueCap,
},
};
}
/** What a cue carries beyond its words and times: its emphasized words as
* [index, word] pairs (set_caption_emphasis), and its own x/y when it sits
* apart from its track (update_cue). Absent when it has neither. */
function cueExtras(c: SubtitleCue) {
const emphasis = emphasizedWords(c);
return {
...(emphasis.length > 0 ? { emphasis } : {}),
...(c.x !== undefined && c.y !== undefined ? { x: rF(c.x), y: rF(c.y) } : {}),
};
}
/** A clip's rate as the model reads it: one number, or the average plus the
* curve's nodes as [sourceSeconds, rate] when the rate changes through the
* footage. */
function describeRate(c: { speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean; smoothSlow?: boolean; in: number; out: number }) {
const nodes = speedCurveOf(c);
const reverse = { ...(c.reverse ? { reverse: true } : {}), ...(c.smoothSlow ? { smooth: true } : {}) };
if (!nodes) return { speed: r(c.speed ?? 1), ...reverse };
return {
speed: r(retimeOf(c).rate),
speedCurve: nodes.map(([at, rate]) => [r(at), r(rate)]),
...reverse,
};
}
function describeAudio(
a: { groupId?: string; assetId: string; name?: string; start: number; in: number; out: number; volume: number; fadeIn?: number; fadeOut?: number; speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean; sound?: ClipSound; duck?: number; lane?: number; hidden?: boolean },
assets: Map<string, { name: string }>
) {
const rt = retimeOf(a);
const speed = rt.rate;
return {
...(a.groupId ? { groupId: a.groupId } : {}),
asset: assets.get(a.assetId)?.name ?? a.assetId,
...(a.name ? { name: a.name } : {}),
start: roundT(a.start),
len: roundT(rt.len),
in: roundT(a.in),
out: roundT(a.out),
volume: r(a.volume),
fadeIn: r(a.fadeIn ?? 0),
fadeOut: r(a.fadeOut ?? 0),
...(a.sound ? { sound: a.sound } : {}),
...(speed !== 1 || a.speedCurve || a.reverse ? describeRate(a) : {}),
...(a.lane ? { lane: a.lane } : {}),
...(a.hidden ? { hidden: true } : {}),
// A voiceover: while it plays, other audio ducks to this gain.
...(a.duck !== undefined ? { duck: r(a.duck) } : {}),
};
}
/** A clip's split edit as set_split_edit wrote it: seconds its sound leads
* the picture and carries past it. Absent when it has none. */
function splitEditOf(c: VideoClip) {
return c.audioLead || c.audioTail ? { audioLead: c.audioLead ?? 0, audioTail: c.audioTail ?? 0 } : {};
}
function describeOverlayClip(c: VideoClip, assets: Map<string, { name: string }>) {
const rect = rectOf(c);
return {
asset: assets.get(c.assetId)?.name ?? c.assetId,
...(c.name ? { name: c.name } : {}),
track: c.track,
start: roundT(c.start),
len: roundT(retimeOf(c).len),
in: roundT(c.in),
out: roundT(c.out),
muted: c.muted,
...splitEditOf(c),
...(c.hidden ? { hidden: true } : {}),
...(c.sound ? { sound: c.sound } : {}),
// The frame region this layer occupies: Full covers the frame; Top/Bottom/
// Left/Right split it; PiP floats inside it.
layout: regionLabel(rect),
region: centreBox(rect),
fit: c.fit ?? "fit",
...(clipZoom(c) > 1
? { zoom: r(clipZoom(c)), panX: r(c.panX ?? 0), panY: r(c.panY ?? 0) }
: c.fit === "fill"
? { panX: r(c.panX ?? 0), panY: r(c.panY ?? 0) }
: {}),
...(c.flipH ? { flipH: true } : {}),
...(c.flipV ? { flipV: true } : {}),
...(c.rotation ? { rotation: c.rotation } : {}),
...((c.opacity ?? 1) < 1 ? { opacity: r(c.opacity ?? 1) } : {}),
...(retimeOf(c).rate !== 1 || c.speedCurve || c.reverse || c.smoothSlow ? describeRate(c) : {}),
...(c.grade ? { colorGrade: c.grade } : {}),
...(c.mask ? { mask: c.mask } : {}),
...(c.boxStyle ? { boxStyle: c.boxStyle } : {}),
...(c.kf?.length
? { keyframes: c.kf.map((k) => ({ ...k, t: roundT(k.t), x: rF(k.x), y: rF(k.y) })) }
: {}),
...(c.motionBlur ? { motionBlur: r(c.motionBlur) } : {}),
};
}
function describeOverlay(o: Overlay) {
const base = {
kind: o.kind ?? "text",
...(o.groupId ? { groupId: o.groupId } : {}),
name: overlayName(o),
start: roundT(o.start),
end: roundT(o.end),
x: rF(o.x),
y: rF(o.y),
...(o.rotation ? { rotation: o.rotation } : {}),
...(o.opacity !== undefined ? { opacity: r(o.opacity) } : {}),
...(o.blur ? { blur: r(o.blur) } : {}),
...(o.motionBlur ? { motionBlur: r(o.motionBlur) } : {}),
...(hasOverlayAnim(o.anim) ? { anim: o.anim } : {}),
...(o.kf?.length
? { keyframes: o.kf.map((k) => ({ ...k, t: roundT(k.t), x: rF(k.x), y: rF(k.y) })) }
: {}),
...(o.mask ? { mask: o.mask } : {}),
...(o.lane ? { lane: o.lane } : {}),
...(o.hidden ? { hidden: true } : {}),
};
if (o.kind === "shape") {
return {
...base,
shape: o.shape,
w: rF(o.w),
h: rF(o.h),
fill: o.fill,
...(o.fillOpacity !== undefined ? { fillOpacity: r(o.fillOpacity) } : {}),
...(o.radius ? { radius: o.radius } : {}),
...(o.stroke ? { stroke: o.stroke } : {}),
...(o.pattern ? { pattern: o.pattern } : {}),
...(o.shadow ? { shadow: o.shadow } : {}),
};
}
if (o.kind === "sticker") {
return { ...base, assetId: o.assetId, w: rF(o.w), ...(o.h ? { h: rF(o.h) } : {}) };
}
if (o.kind === "effect") {
return {
...base,
effect: o.effect,
amount: r(o.amount ?? 0.5),
...(o.effect === "zoom"
? { focus: { x: r(o.focus?.x ?? 0.5), y: r(o.focus?.y ?? 0.5) } }
: {}),
...(o.effect === "lightleak" && o.leak ? { leak: o.leak } : {}),
...(o.effect === "glitch" && o.glitch ? { glitch: o.glitch } : {}),
...(o.effect === "flash" && o.tone ? { tone: o.tone } : {}),
...(o.effect === "flash" && o.rate ? { rate: r(o.rate), rhythm: o.rhythm ?? "strobe" } : {}),
...(o.effect === "negative" && o.rate ? { rate: r(o.rate) } : {}),
...(o.effect === "flash" && o.fade ? { fade: o.fade, ...(o.fadeEase ? { fade_ease: o.fadeEase } : {}) } : {}),
};
}
return {
...base,
text: o.text,
size: o.size,
font: o.font,
weight: o.weight,
color: o.color,
shadow: o.shadow,
plate: o.plate,
...(o.plateRadius !== undefined && { plateRadius: r(o.plateRadius) }),
...(o.wrapWidth !== undefined ? { wrapWidth: r(o.wrapWidth) } : {}),
...(o.texture ? { texture: o.texture } : {}),
};
}