Skip to content

Open source CapCut alternative.

TypeScript

feat(site): the composer writes a moment in the cut

David Le1c4161d16b20
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsx
index bb4c5496..05eb4237 100644
--- a/site/src/cut/components/AiPanel.tsx
+++ b/site/src/cut/components/AiPanel.tsx
@@ -97,6 +97,7 @@ import { useCreditsRecheck, useOutOfCredits } from "@/cut/lib/hosted";
import { cutChatLive, dropPiSession, foldIntoCutChat, hydratePiSession, judgeEngineSkill, readPiSession, streamCutChat, triageQueuedMessages } from "@/cut/lib/pi/cutAgent";
import { dueForRetriage, toolProgress } from "@/cut/lib/queueTriage";
import { registerQueueSink } from "@/cut/lib/chatQueue";
+import { syncTimes, type TimeMark } from "@/cut/lib/timeMark";
import { productionDeps } from "@/cut/lib/pi/prodDeps";
import { withChatProject } from "@/cut/lib/projectChatTools";
import { runAiTool } from "@/cut/lib/aiTools";
@@ -241,6 +242,14 @@ function liveRefs(refs: AssetRef[]): AssetRef[] {
);
}
+/** A waiting row's text with its `@1:34` moments read against the cut as it
+ * stands this instant — what a dispatch sends, whichever render it fires on. */
+function dueText(m: QueuedMessage): string {
+ if (!m.times?.length) return m.text;
+ const { clips, audioClips } = useEditor.getState();
+ return syncTimes(m.text, m.times, { clips, audioClips }).text;
+}
+
function readThreads(projectId: string): ChatThread[] {
return readRawThreads(projectId).filter((t) => isStoredThread(t) && !t.deleted) as ChatThread[];
}
@@ -391,6 +400,11 @@ function readStoredQueue(
.map((m) => ({
...m,
attachments: Array.isArray(m.attachments) ? m.attachments : [],
+ // A stored mark with no token or no time would format as nonsense;
+ // the row's written time stands on its own without it.
+ times: (Array.isArray(m.times) ? m.times : []).filter(
+ (t) => typeof t?.token === "string" && Number.isFinite(t?.at),
+ ),
}));
} catch {
return [];
@@ -926,6 +940,10 @@ function ChatSession({
micWasActive.current = mic.state !== "idle";
}, [mic.state]);
const [attachments, setAttachments] = useState<AssetRef[]>([]);
+ // Moments the composer's `@` tokens point at, in the order they were
+ // written. They ride with the message so the time can be re-read against
+ // the cut the message actually runs on.
+ const [times, setTimes] = useState<TimeMark[]>([]);
const candidates = useRefCandidates(visible);
// Any OS file drag over the window hints the composer as a drop target;
// hovering it (dropActive below) strengthens the ring and shows the label.
@@ -1533,14 +1551,15 @@ function ChatSession({
)
continue;
const live = liveRefs(row.attachments);
+ const text = dueText(current);
if (verdict === "spawn") {
- const id = onSpawn({ text: row.text, attachments: live, model });
+ const id = onSpawn({ text, attachments: live, model });
setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "spawned", threadId: id } : m)));
} else if (verdict === "fold" && provider(model) === "gemini" && cutChatLive(threadId)) {
setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "folding" } : m)));
const folded = await foldIntoCutChat({
threadId,
- text: row.text,
+ text,
attachments: live,
deps: productionDeps(projectId, turnSignalRef.current),
});
@@ -1553,7 +1572,7 @@ function ChatSession({
const user: UIMessage = {
id: crypto.randomUUID(),
role: "user",
- parts: [{ type: "text", text: row.text }],
+ parts: [{ type: "text", text }],
...(live.length > 0 && { metadata: { attachments: live } }),
};
const tail = msgs.at(-1);
@@ -1607,6 +1626,27 @@ function ChatSession({
),
[threadId],
);
+ // A waiting row keeps pointing at the same moment of the video while the
+ // running turn edits the cut under it. Each row's marks are re-read against
+ // the live clips and its `@1:34` tokens rewritten, so the tray shows — and
+ // the dispatch sends — the time that moment sits at now. The row being
+ // edited is left alone; its text belongs to the editor until it commits.
+ const docClips = useEditor((s) => s.clips);
+ const docAudio = useEditor((s) => s.audioClips);
+ useEffect(() => {
+ // eslint-disable-next-line react-hooks/set-state-in-effect -- the rows react to the cut moving; the write is skipped unless a token's time actually changed
+ setQueue((q) => {
+ let moved = false;
+ const next = q.map((m) => {
+ if (!m.times?.length || m.status !== "queued" || m.id === queueEditing) return m;
+ const synced = syncTimes(m.text, m.times, { clips: docClips, audioClips: docAudio });
+ if (synced.text === m.text) return m;
+ moved = true;
+ return { ...m, text: synced.text, times: synced.marks };
+ });
+ return moved ? next : q;
+ });
+ }, [docClips, docAudio, queueEditing]);
// Height of the floating stack above the composer (warning tabs + queue
// tray); the messages pad their bottom by it so the newest message can
// scroll out from behind the stack.
@@ -1632,7 +1672,14 @@ function ChatSession({
// Inline @mentions attach their assets alongside the dropped chips. The
// message keeps the raw tokens — they render as interactive chips and the
// model reads the handle↔asset mapping from <attached_assets>.
- const body = text.trim();
+ // Moments are re-read here too: the cut can have moved since the token was
+ // written, and a token typed by hand picks up its mark now.
+ const doc = useEditor.getState();
+ const synced = syncTimes(text.trim(), times, {
+ clips: doc.clips,
+ audioClips: doc.audioClips,
+ });
+ const body = synced.text;
const { refs: all } = collectRefs(body, attachments, candidates);
if (!body && all.length === 0) return;
// An unavailable provider surfaces as an error in the thread at send time
@@ -1645,10 +1692,17 @@ function ChatSession({
// A paused queue stays paused: parked rows fire only from the tray's
// resume button, never as a side effect of submitting something new.
// With the queue live, the triage places the row right away.
- const row: QueuedMessage = { id: crypto.randomUUID(), text: body, attachments: all, status: "queued" };
+ const row: QueuedMessage = {
+ id: crypto.randomUUID(),
+ text: body,
+ attachments: all,
+ status: "queued",
+ ...(synced.marks.length > 0 && { times: synced.marks }),
+ };
setQueue((q) => [...q, row]);
setInput("");
setAttachments([]);
+ setTimes([]);
pinnedRef.current = true;
if (!queuePaused) void triageRef.current([row]);
return;
@@ -1674,6 +1728,7 @@ function ChatSession({
}));
setInput("");
setAttachments([]);
+ setTimes([]);
};
// A turn the page lost picks itself back up. With an engine journal the
@@ -1774,9 +1829,10 @@ function ChatSession({
// parked (or at file blobs that died with the last page); dropping the
// dead refs keeps the message from claiming media it can't deliver.
const live = liveRefs(next.attachments);
- runningAskRef.current = next.text;
+ const text = dueText(next);
+ runningAskRef.current = text;
dispatchChat(() => sendMessage({
- text: next.text,
+ text,
...(live.length > 0 && { metadata: { attachments: live } }),
}));
// The rows behind it are placed against the turn that just started:
@@ -1972,9 +2028,16 @@ function ChatSession({
}}
onCommitEdit={(id, text) =>
setQueue((q) =>
- q.map((m) =>
- m.id === id ? { ...m, text: text.trim() || m.text } : m,
- ),
+ q.map((m) => {
+ if (m.id !== id) return m;
+ // A rewrite can drop a moment or type a new one; re-read
+ // the row's marks off the text it now carries.
+ const synced = syncTimes(text.trim() || m.text, m.times ?? [], {
+ clips: docClips,
+ audioClips: docAudio,
+ });
+ return { ...m, text: synced.text, times: synced.marks };
+ }),
)
}
onRemove={(id) => setQueue((q) => q.filter((m) => m.id !== id))}
@@ -2061,6 +2124,7 @@ function ChatSession({
onUpsertRef={(ref) => setAttachments((p) => upsertRef(p, ref))}
onPasteFiles={attachFiles}
onRemoveLastRef={() => setAttachments((p) => p.slice(0, -1))}
+ onInsertTime={(mark) => setTimes((p) => [...p, mark])}
/>
<div className="flex items-center gap-1 px-1.5 pb-1.5">
<ModelSelector
diff --git a/site/src/cut/components/AssetRefs.tsx b/site/src/cut/components/AssetRefs.tsx
index e33ad41f..eb1f4780 100644
--- a/site/src/cut/components/AssetRefs.tsx
+++ b/site/src/cut/components/AssetRefs.tsx
@@ -1,7 +1,7 @@
"use client";
import { useEffect, useLayoutEffect, useMemo, useRef, useState, type ReactNode, type RefObject } from "react";
-import { Check, Copy, FileText, X } from "lucide-react";
+import { Check, Clock, Copy, FileText, X } from "lucide-react";
import {
highlightMentions,
liveRefUrl,
@@ -19,7 +19,16 @@ import { useInView } from "@/cut/hooks/useInView";
import { revealRef } from "@/cut/lib/refReveal";
import { formatTime } from "@/cut/lib/time";
import { clipLen, getClipSpans, useEditor } from "@/cut/lib/store";
-import { playheadAt } from "@/cut/lib/playhead";
+import { playheadAt, previewAt, usePreviewTimeEvery } from "@/cut/lib/playhead";
+import {
+ formatMark,
+ markAt,
+ parseMark,
+ PLAYHEAD_HANDLE,
+ PLAYHEAD_NAME,
+ splitTimes,
+ type TimeMark,
+} from "@/cut/lib/timeMark";
import { AudioPillSurface } from "@/cut/components/AudioPanel";
import { entityGlyph } from "@/cut/components/entityIcons";
import {
@@ -345,6 +354,38 @@ export function RefTokenChip({
);
}
+/**
+ * An inline `@1:34` moment token: a place in the cut the message points at.
+ * Clicking it takes the playhead there. Written messages hold the moment they
+ * ran against; a message still waiting in the queue holds the moment it will
+ * run against, re-read as the cut moves under it.
+ */
+export function TimeTokenChip({
+ text,
+ at,
+ onDark,
+}: {
+ text: string;
+ at: number;
+ onDark?: boolean;
+}) {
+ return (
+ <button
+ type="button"
+ className={cn(
+ "ref-token inline-flex items-center gap-0.5 rounded-md px-1 align-middle font-mono text-[11px] transition-colors",
+ onDark
+ ? "bg-white/15 text-[#8ec7ff] hover:bg-white/25"
+ : "bg-[#0a84ff]/10 text-[#0a84ff] hover:bg-[#0a84ff]/20"
+ )}
+ onClick={() => useEditor.getState().seek(at)}
+ >
+ <Clock className="size-2.5 shrink-0" />
+ {text.slice(1)}
+ </button>
+ );
+}
+
/** Message text with its resolved `@` mentions drawn as token chips. One
* renderer for every place a written message is shown — the chat thread and
* the queue tray above the composer — so a message reads the same before it
@@ -362,8 +403,13 @@ export function MentionedText({
onDark?: boolean;
}) {
const candidates = useRefCandidates();
+ // Mentions first, then the moments inside what is left over: an asset's
+ // name never looks like a time, so the two passes never fight over a token.
const parts = useMemo(
- () => splitMentions(text, [...attachments, ...candidates]),
+ () =>
+ splitMentions(text, [...attachments, ...candidates]).flatMap<
+ string | AssetRef | { text: string; at: number }
+ >((p) => (typeof p === "string" ? splitTimes(p) : [p])),
[text, attachments, candidates]
);
return (
@@ -371,6 +417,8 @@ export function MentionedText({
{parts.map((p, i) =>
typeof p === "string" ? (
<span key={i}>{p}</span>
+ ) : "at" in p ? (
+ <TimeTokenChip key={i} text={p.text} at={p.at} onDark={onDark} />
) : (
<RefTokenChip key={i} item={p} onDark={onDark} />
)
@@ -939,6 +987,18 @@ function mentionAtCaret(value: string, caret: number): { start: number; query: s
return { start: at, query };
}
+/** The preview's moment on the `@` menu's playhead row. Mounted only while
+ * the menu is open, and reading the clock a few times a second: the composer
+ * itself never re-renders on the playhead. */
+function PlayheadRowTime() {
+ const at = usePreviewTimeEvery(10) / 10;
+ return (
+ <span className="shrink-0 font-mono text-[10.5px] text-muted-foreground tabular-nums">
+ {formatMark(at)}
+ </span>
+ );
+}
+
/** One resolved mention in the composer mirror. Entity pills paint over the
* raw token with the entity's icon (or the sticker's own art) and name — the
* token glyphs underneath keep their exact layout, so the textarea's caret
@@ -1100,6 +1160,7 @@ export function MentionTextarea({
onUpsertRef,
onPasteFiles,
onRemoveLastRef,
+ onInsertTime,
}: {
value: string;
onChange: (v: string) => void;
@@ -1130,6 +1191,11 @@ export function MentionTextarea({
/** Backspace with the caret at the very start (nothing selected, nothing to
* delete leftward) removes the newest attachment chip instead. */
onRemoveLastRef?: () => void;
+ /** Take a moment in the cut the user picked off the `@` menu. Providing it
+ * puts the playhead at the top of that menu, so `@here` writes the time the
+ * preview is sitting on; the mark comes with it so the caller can re-read
+ * the moment if the cut moves before the message goes out. */
+ onInsertTime?: (mark: TimeMark) => void;
}) {
const taRef = useRef<HTMLTextAreaElement>(null);
const rootRef = useRef<HTMLDivElement>(null);
@@ -1183,16 +1249,29 @@ export function MentionTextarea({
// Mirror segments with their character offsets, so a pill can tell when
// the caret sits inside its token.
+ // Moments pill only where they mean something — the surface that takes them.
+ const takesTime = !!onInsertTime;
const mirrorSegs = useMemo(
() =>
- highlightMentions(value, pool).reduce<
- { text: string; ref: AssetRef | null; start: number }[]
- >((acc, seg) => {
- const prev = acc[acc.length - 1];
- acc.push({ ...seg, start: prev ? prev.start + prev.text.length : 0 });
- return acc;
- }, []),
- [value, pool]
+ highlightMentions(value, pool)
+ // Moments live in the runs no mention claimed.
+ .flatMap((seg) =>
+ seg.ref || !takesTime
+ ? [seg]
+ : splitTimes(seg.text).map((part) =>
+ typeof part === "string"
+ ? { text: part, ref: null, time: false }
+ : { text: part.text, ref: null, time: true }
+ )
+ )
+ .reduce<
+ { text: string; ref: AssetRef | null; time?: boolean; start: number }[]
+ >((acc, seg) => {
+ const prev = acc[acc.length - 1];
+ acc.push({ ...seg, start: prev ? prev.start + prev.text.length : 0 });
+ return acc;
+ }, []),
+ [value, pool, takesTime]
);
// A pill cover is one box, so its token must move between lines whole —
@@ -1218,15 +1297,19 @@ export function MentionTextarea({
}, [mirrorSegs, value, onChange]);
const mention = useMemo(() => mentionAtCaret(value, caret), [value, caret]);
+ // A caret strictly inside an intact, resolved token is a click on the pill —
+ // the mention is complete, and offering lookalike suggestions for its
+ // half-read query would misdirect it.
+ const inResolved = useMemo(
+ () =>
+ mirrorSegs.some(
+ (seg) =>
+ (seg.ref || seg.time) && caret > seg.start && caret < seg.start + seg.text.length
+ ),
+ [mirrorSegs, caret]
+ );
const matches = useMemo(() => {
- if (!mention || dismissed === mention.start) return [];
- // A caret strictly inside an intact, resolved token is a click on the
- // pill — the mention is complete, and offering lookalike suggestions for
- // its half-read query would misdirect it.
- const inResolved = mirrorSegs.some(
- (seg) => seg.ref && caret > seg.start && caret < seg.start + seg.text.length
- );
- if (inResolved) return [];
+ if (!mention || dismissed === mention.start || inResolved) return [];
const q = mention.query.toLowerCase();
// Best match first, not list order: a typed handle prefix ("c" → c1, c2)
// beats a name prefix, which beats a substring hit anywhere in the name —
@@ -1248,14 +1331,24 @@ export function MentionTextarea({
.sort((a, b) => a.r - b.r)
.slice(0, 8)
.map((x) => x.c);
- }, [mention, dismissed, candidates, mirrorSegs, caret]);
- const open = matches.length > 0;
+ }, [mention, dismissed, inResolved, candidates]);
+ // The playhead leads the menu wherever moments are taken: a bare `@`, or a
+ // query that starts the row's handle or name. Matched the way a candidate
+ // is, so `@here` and `@play` both land on it.
+ const timeRow =
+ !!onInsertTime &&
+ !!mention &&
+ dismissed !== mention.start &&
+ !inResolved &&
+ [PLAYHEAD_HANDLE, PLAYHEAD_NAME].some((n) => n.startsWith(mention.query.toLowerCase()));
+ const rowCount = matches.length + (timeRow ? 1 : 0);
+ const open = rowCount > 0;
// Each keystroke re-ranks the list, so a highlight chosen under a previous
// query derives back to the best (first) match instead of holding a stale
// arrow/hover position.
const sel = selState.q === mention?.query ? selState.i : 0;
const setSel = (i: number) => setSelState({ q: mention?.query, i });
- const selIndex = Math.min(sel, matches.length - 1);
+ const selIndex = Math.min(sel, rowCount - 1);
const syncCaret = () => {
const el = taRef.current;
@@ -1335,9 +1428,9 @@ export function MentionTextarea({
});
};
- const pick = (ref: AssetRef) => {
+ /** Write `token` over the mention being typed and leave the caret after it. */
+ const writeToken = (token: string) => {
if (!mention) return;
- const token = refToken(ref) + " ";
const next = value.slice(0, mention.start) + token + value.slice(caret);
const newCaret = mention.start + token.length;
onChange(next);
@@ -1351,6 +1444,28 @@ export function MentionTextarea({
});
};
+ const pick = (ref: AssetRef) => writeToken(refToken(ref) + " ");
+
+ /** Write the preview's moment as a token, and hand the caller the mark that
+ * holds what it points at, so the time can be re-read if the cut moves. */
+ const pickTime = () => {
+ if (!onInsertTime) return;
+ const { clips, audioClips } = useEditor.getState();
+ const mark = markAt(previewAt(), { clips, audioClips });
+ onInsertTime(mark);
+ writeToken(mark.token + " ");
+ };
+
+ /** Take the menu row at `i`: the playhead leads the list when it is shown. */
+ const pickRow = (i: number) => {
+ if (timeRow && i === 0) {
+ pickTime();
+ return;
+ }
+ const ref = matches[timeRow ? i - 1 : i];
+ if (ref) pick(ref);
+ };
+
// Auto-grow: reset to natural height, then match the content. The caller's
// `max-h-*` caps it and the textarea scrolls internally past that point.
useLayoutEffect(() => {
@@ -1370,26 +1485,50 @@ export function MentionTextarea({
menuSide === "top" ? "bottom-full mb-1" : "top-full mt-1"
)}
>
- {matches.map((c, i) => (
+ {timeRow && (
<button
- key={`${c.scope}:${c.id}`}
type="button"
className={cn(
"flex w-full items-center gap-2 rounded-md px-1.5 py-1 text-left",
- i === selIndex ? "bg-muted" : "hover:bg-muted/60"
+ selIndex === 0 ? "bg-muted" : "hover:bg-muted/60"
)}
- onMouseEnter={() => setSel(i)}
- // mousedown, not click: keep focus (and the mention state) in the textarea.
+ onMouseEnter={() => setSel(0)}
onMouseDown={(e) => {
e.preventDefault();
- pick(c);
+ pickTime();
}}
>
- <RefThumb item={c} className="size-8" />
- {c.handle && <RefHandleBadge handle={c.handle} className="shrink-0" />}
- <span className="min-w-0 flex-1 truncate text-[11.5px]">{c.name}</span>
+ <span className="grid size-8 shrink-0 place-items-center rounded-lg border border-border bg-muted text-[#0a84ff]">
+ <Clock className="size-4" />
+ </span>
+ <RefHandleBadge handle={PLAYHEAD_HANDLE} className="shrink-0" />
+ <span className="min-w-0 flex-1 truncate text-[11.5px]">{PLAYHEAD_NAME}</span>
+ <PlayheadRowTime />
</button>
- ))}
+ )}
+ {matches.map((c, i) => {
+ const row = timeRow ? i + 1 : i;
+ return (
+ <button
+ key={`${c.scope}:${c.id}`}
+ type="button"
+ className={cn(
+ "flex w-full items-center gap-2 rounded-md px-1.5 py-1 text-left",
+ row === selIndex ? "bg-muted" : "hover:bg-muted/60"
+ )}
+ onMouseEnter={() => setSel(row)}
+ // mousedown, not click: keep focus (and the mention state) in the textarea.
+ onMouseDown={(e) => {
+ e.preventDefault();
+ pick(c);
+ }}
+ >
+ <RefThumb item={c} className="size-8" />
+ {c.handle && <RefHandleBadge handle={c.handle} className="shrink-0" />}
+ <span className="min-w-0 flex-1 truncate text-[11.5px]">{c.name}</span>
+ </button>
+ );
+ })}
</div>
)}
<textarea
@@ -1517,14 +1656,12 @@ export function MentionTextarea({
if (open) {
if (e.key === "ArrowDown" || e.key === "ArrowUp") {
e.preventDefault();
- setSel(
- (selIndex + (e.key === "ArrowDown" ? 1 : matches.length - 1)) % matches.length
- );
+ setSel((selIndex + (e.key === "ArrowDown" ? 1 : rowCount - 1)) % rowCount);
return;
}
if (e.key === "Enter" || e.key === "Tab") {
e.preventDefault();
- pick(matches[selIndex]);
+ pickRow(selIndex);
return;
}
if (e.key === "Escape") {
@@ -1561,14 +1698,33 @@ export function MentionTextarea({
)}
>
{mirrorSegs.map((seg, i) => {
+ // A moment pills over its own glyphs — the raw token keeps its
+ // exact layout, so the caret math never drifts — and takes a click
+ // to put the playhead back where the token points.
+ if (seg.time)
+ return (
+ <span
+ key={i}
+ className="pointer-events-auto cursor-pointer rounded-[4px] bg-[#0a84ff]/12 hover:bg-[#0a84ff]/25"
+ onMouseDown={(e) => {
+ e.preventDefault();
+ const at = parseMark(seg.text.slice(1));
+ if (at !== null) useEditor.getState().seek(at);
+ }}
+ >
+ {seg.text}
+ </span>
+ );
if (!seg.ref) return <span key={i}>{seg.text}</span>;
// A side facing another pill across a single space keeps its bleed
// off — two bleeds would swallow the only gap between the pills.
+ // A moment counts as a pill here too.
const facing = (gap: number, other: number) =>
!!mirrorSegs[gap] &&
!mirrorSegs[gap].ref &&
+ !mirrorSegs[gap].time &&
/^\s$/.test(mirrorSegs[gap].text) &&
- !!mirrorSegs[other]?.ref;
+ !!(mirrorSegs[other]?.ref || mirrorSegs[other]?.time);
const end = seg.start + seg.text.length;
// Raw text shows only while a range endpoint rests strictly inside
// the token; entity endpoints normalize to the token edges, so an
diff --git a/site/src/cut/components/ComposerQueue.tsx b/site/src/cut/components/ComposerQueue.tsx
index e85c7957..31a4c9ec 100644
--- a/site/src/cut/components/ComposerQueue.tsx
+++ b/site/src/cut/components/ComposerQueue.tsx
@@ -17,6 +17,7 @@ import { ScrollArea } from "@/components/ui/scroll-area";
import { useDragSort } from "@/components/Sortable";
import { MentionedText, RefThumb } from "./AssetRefs";
import type { AssetRef } from "@/cut/lib/assetRef";
+import type { TimeMark } from "@/cut/lib/timeMark";
/** Small icon control on a tray row or the tray header. */
const queueIconButton =
@@ -34,6 +35,10 @@ export interface QueuedMessage {
attachments: AssetRef[];
status: "queued" | "folding" | "spawned" | "running" | "done";
threadId?: string;
+ /** What the row's `@1:34` moments point at. A waiting row's times are
+ * re-read as the running turn moves the cut, so the text always names the
+ * moment the row will actually run against. */
+ times?: TimeMark[];
}
/** The queue tray: a folder tab on top of the composer holding the messages
diff --git a/site/src/cut/lib/timeMark.test.ts b/site/src/cut/lib/timeMark.test.ts
new file mode 100644
index 00000000..5e1ed123
--- /dev/null
+++ b/site/src/cut/lib/timeMark.test.ts
@@ -0,0 +1,222 @@
+import { describe, expect, test } from "bun:test";
+import { retimeOf } from "@donkeycut/effects-kit";
+import { formatMark, markAt, markTime, parseMark, splitTimes, syncTimes } from "./timeMark";
+import { splitTimelineItems } from "./timelineItems";
+import type { AudioClip, VideoClip } from "./types";
+
+const clip = (id: string, start: number, patch: Partial<VideoClip> = {}): VideoClip => ({
+ id,
+ assetId: id,
+ track: 0,
+ start,
+ in: 0,
+ out: 10,
+ muted: false,
+ ...patch,
+});
+
+const sound = (id: string, start: number, patch: Partial<AudioClip> = {}): AudioClip => ({
+ id,
+ assetId: id,
+ start,
+ in: 0,
+ out: 20,
+ volume: 1,
+ ...patch,
+});
+
+/** a: 0–10, b: 10–20 on track 0. */
+const cut = () => ({ clips: [clip("a", 0), clip("b", 10)] });
+
+/** The source second the cut actually shows at `t`. */
+function frameAt(s: { clips: VideoClip[] }, t: number): number | null {
+ const c = s.clips.find((x) => t >= x.start && t <= x.start + retimeOf(x).len);
+ return c ? retimeOf(c).srcAt(t - c.start) : null;
+}
+
+describe("time tokens", () => {
+ test("format drops a zero tenth and grows an hour field", () => {
+ expect(formatMark(94)).toBe("1:34");
+ expect(formatMark(94.24)).toBe("1:34.2");
+ expect(formatMark(8.02)).toBe("0:08");
+ expect(formatMark(3723)).toBe("1:02:03");
+ expect(formatMark(-5)).toBe("0:00");
+ });
+
+ test("parse reads both shapes", () => {
+ expect(parseMark("1:34")).toBe(94);
+ expect(parseMark("1:34.5")).toBe(94.5);
+ expect(parseMark("1:02:03")).toBe(3723);
+ expect(parseMark("nope")).toBe(null);
+ });
+
+ test("split keeps the written token and leaves the rest of the text", () => {
+ expect(splitTimes("zoom at @1:34 please")).toEqual([
+ "zoom at ",
+ { text: "@1:34", at: 94 },
+ " please",
+ ]);
+ // A handle that merely starts with digits is not a moment.
+ expect(splitTimes("@v2 is loud")).toEqual(["@v2 is loud"]);
+ expect(splitTimes("@1:34:56:78")).toEqual(["@1:34:56:78"]);
+ });
+});
+
+describe("a moment holds its frame", () => {
+ test("through a ripple", () => {
+ const s = cut();
+ const mark = markAt(12, s);
+ expect(mark.clipId).toBe("b");
+ expect(mark.src).toBeCloseTo(2);
+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 13)] })).toBeCloseTo(15);
+ });
+
+ test("through a head trim", () => {
+ const mark = markAt(12, cut());
+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 10, { in: 1 })] })).toBeCloseTo(11);
+ });
+
+ test("through a split, on either side of the cut", () => {
+ const s = cut();
+ const early = markAt(12, s);
+ const late = markAt(17, s);
+ let n = 0;
+ const after = splitTimelineItems(
+ { ...s, audioClips: [], overlays: [], subtitles: { cues: [] } } as never,
+ [{ kind: "clip", id: "b" }],
+ 15,
+ () => `new${++n}`,
+ ).next as { clips: VideoClip[] };
+ // The right half is a clip the mark has never seen.
+ expect(after.clips.map((c) => c.id)).toEqual(["a", "b", "new1"]);
+ expect(markTime(early, after)).toBeCloseTo(12);
+ expect(markTime(late, after)).toBeCloseTo(17);
+ // And it follows that new half when the turn moves it.
+ const moved = { clips: after.clips.map((c) => (c.id === "new1" ? { ...c, start: 18 } : c)) };
+ expect(markTime(late, moved)).toBeCloseTo(20);
+ });
+
+ test("through a speed change, a speed curve and a reverse", () => {
+ const s = cut();
+ const mark = markAt(17, s);
+ expect(mark.src).toBeCloseTo(7);
+ for (const patch of [
+ { speed: 2 },
+ { speed: 0.5 },
+ { speedCurve: [[0, 0.5], [0.5, 2], [1, 1]] as [number, number][] },
+ { reverse: true },
+ { speed: 2, reverse: true },
+ ]) {
+ const next = { clips: [clip("a", 0), clip("b", 10, patch)] };
+ const at = markTime(mark, next);
+ // Wherever the moment lands, the cut shows the frame it was written on.
+ expect(frameAt(next, at)).toBeCloseTo(7, 4);
+ }
+ });
+
+ test("through a retime that also moves the clip", () => {
+ const mark = markAt(17, cut());
+ const next = { clips: [clip("a", 0), clip("b", 30, { speed: 4 })] };
+ expect(frameAt(next, markTime(mark, next))).toBeCloseTo(7, 4);
+ });
+
+ test("and clamps into the clip when the frame is trimmed away", () => {
+ const mark = markAt(17, cut());
+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 10, { out: 5 })] })).toBeCloseTo(15);
+ });
+
+ test("on a cut belongs to the incoming clip", () => {
+ const mark = markAt(10, cut());
+ expect(mark.clipId).toBe("b");
+ expect(mark.src).toBeCloseTo(0);
+ // Trimming the outgoing clip's tail pulls the cut back, and with it the moment.
+ expect(markTime(mark, { clips: [clip("a", 0, { out: 8 }), clip("b", 8)] })).toBeCloseTo(8);
+ });
+
+ test("and picks the nearer copy when the source is on the timeline twice", () => {
+ const mark = markAt(17, cut());
+ const twice = { clips: [clip("a", 0), clip("b", 10), clip("b2", 20, { assetId: "b" })] };
+ expect(markTime(mark, twice)).toBeCloseTo(17);
+ });
+});
+
+describe("a moment off the clips", () => {
+ test("in a gap rides the head of the clip ahead of it", () => {
+ const s = { clips: [clip("a", 0), clip("b", 14)] };
+ const mark = markAt(12, s);
+ expect(mark.clipId).toBe("b");
+ expect(mark.lead).toBeCloseTo(-2);
+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 19)] })).toBeCloseTo(17);
+ });
+
+ test("past the end rides the head of the last clip", () => {
+ const s = { clips: [clip("a", 0)] };
+ const mark = markAt(13, s);
+ expect(mark.clipId).toBe("a");
+ expect(mark.lead).toBeCloseTo(13);
+ expect(markTime(mark, { clips: [clip("a", 4)] })).toBeCloseTo(17);
+ });
+
+ test("in a gap finds the head again after the clip ahead is split", () => {
+ const s = { clips: [clip("a", 0), clip("b", 14)] };
+ const mark = markAt(12, s);
+ // The split leaves b's head where it was; the moment stays two before it.
+ let n = 0;
+ const after = splitTimelineItems(
+ { ...s, audioClips: [], overlays: [], subtitles: { cues: [] } } as never,
+ [{ kind: "clip", id: "b" }],
+ 18,
+ () => `new${++n}`,
+ ).next as { clips: VideoClip[] };
+ expect(markTime(mark, after)).toBeCloseTo(12);
+ });
+
+ test("with no clips at all keeps its second", () => {
+ const mark = markAt(4, { clips: [] });
+ expect(mark.clipId).toBeUndefined();
+ expect(markTime(mark, cut())).toBe(4);
+ });
+
+ test("whose clip is gone keeps its second", () => {
+ const mark = markAt(12, cut());
+ expect(markTime(mark, { clips: [clip("a", 0)] })).toBe(12);
+ });
+});
+
+describe("a project with no picture", () => {
+ test("anchors on the soundtrack and rides its ripple", () => {
+ const s = { clips: [], audioClips: [sound("vo", 0)] };
+ const mark = markAt(12, s);
+ expect(mark.clipId).toBe("vo");
+ expect(mark.src).toBeCloseTo(12);
+ expect(markTime(mark, { clips: [], audioClips: [sound("vo", 5)] })).toBeCloseTo(17);
+ });
+
+ test("but the picture wins wherever there is one", () => {
+ const mark = markAt(12, { ...cut(), audioClips: [sound("vo", 0)] });
+ expect(mark.clipId).toBe("b");
+ });
+});
+
+describe("syncTimes", () => {
+ test("rewrites every token to where its moment sits now", () => {
+ const s = cut();
+ const written = syncTimes("trim @0:05 and title @0:12", [], s);
+ expect(written.text).toBe("trim @0:05 and title @0:12");
+ expect(written.marks.map((m) => m.clipId)).toEqual(["a", "b"]);
+ const moved = { clips: [clip("a", 0, { out: 14 }), clip("b", 14)] };
+ expect(syncTimes(written.text, written.marks, moved).text).toBe("trim @0:05 and title @0:16");
+ });
+
+ test("a token typed by hand picks up a mark on the way out", () => {
+ const { marks } = syncTimes("cut at @0:12", [], cut());
+ expect(marks).toHaveLength(1);
+ expect(marks[0].clipId).toBe("b");
+ });
+
+ test("a rewrite drops the marks its tokens left behind", () => {
+ const s = cut();
+ const first = syncTimes("@0:05 and @0:12", [], s);
+ expect(syncTimes("@0:12 only", first.marks, s).marks).toHaveLength(1);
+ });
+});
diff --git a/site/src/cut/lib/timeMark.ts b/site/src/cut/lib/timeMark.ts
new file mode 100644
index 00000000..a587589c
--- /dev/null
+++ b/site/src/cut/lib/timeMark.ts
@@ -0,0 +1,232 @@
+"use client";
+
+import { retimeOf } from "@donkeycut/effects-kit";
+import { clipLen } from "./store";
+import type { AudioClip, VideoClip } from "./types";
+
+/**
+ * Moments a message points at.
+ *
+ * A chat message can name a place in the cut — "@1:34 zoom in here". The time
+ * is written in the text, so the model reads it the way the user wrote it, and
+ * a mark beside the text records the frame it was written on: the clip under
+ * the moment, the source it plays, and the second inside that source.
+ *
+ * Between writing a message and running it the cut can move. The queue holds
+ * messages while a turn runs, and that turn ripples, splits, retimes, trims
+ * and deletes. So a queued message re-reads its marks against the live
+ * document before it goes out, and the token in the text is rewritten to the
+ * second that frame has reached.
+ *
+ * A source second is what makes that hold. Trim is in source seconds, so a
+ * speed, a curve, a reverse and a preset all leave it alone and the clip's own
+ * map puts it back on the timeline; a split copies it to both halves, so the
+ * frame is found again through the source even on the half that took a new id.
+ * A moment sitting in a gap has no frame, so it holds its distance from the
+ * head of the clip beside it — a head is the edge an insert lands against, and
+ * the one a split leaves where it was. A moment whose clip is gone altogether
+ * keeps the second it was written at.
+ */
+
+/** A time token in message text, e.g. `@1:34` or `@0:02.5` or `@1:02:30`. */
+export const TIME_TOKEN_RE = /@(\d{1,3}:[0-5]\d(?::[0-5]\d)?(?:\.\d{1,2})?)(?![\d:])/g;
+
+/** What `@` offers for the playhead: the row's handle and its name, matched
+ * by the picker the same way an asset's are. */
+export const PLAYHEAD_HANDLE = "here";
+export const PLAYHEAD_NAME = "playhead";
+
+/** How far outside a clip's trim a source second may sit and still count as
+ * inside it — a rounding tolerance, not a reach. */
+const TRIM_EPS = 1e-6;
+
+/** A moment a message points at, and what it hangs off in the cut. */
+export interface TimeMark {
+ /** The token as it stands in the text right now. */
+ token: string;
+ /** The timeline second the token reads as. */
+ at: number;
+ /** The clip the moment hangs off; absent when nothing carries it. */
+ clipId?: string;
+ /** The source that clip plays, so the frame is findable even after the
+ * clip it was written over is split, replaced or renumbered. */
+ assetId?: string;
+ /** The source second under the moment, for a moment over the clip itself. */
+ src?: number;
+ /** Seconds from the clip's head, for a moment sitting off the clip — the
+ * head is the stable edge: splitting a clip leaves its head where it was. */
+ lead?: number;
+}
+
+/** What the marks read against: everything on the timeline that carries time. */
+export interface TimeMarkState {
+ clips: VideoClip[];
+ audioClips?: AudioClip[];
+}
+
+/** `94.2` -> `"1:34.2"`, `94` -> `"1:34"`, `3723` -> `"1:02:03"`. */
+export function formatMark(t: number): string {
+ const tenths = Math.round(Math.max(0, t) * 10);
+ const whole = Math.floor(tenths / 10);
+ const tenth = tenths % 10;
+ const h = Math.floor(whole / 3600);
+ const m = Math.floor((whole % 3600) / 60);
+ const s = whole % 60;
+ const pad = (n: number) => String(n).padStart(2, "0");
+ const base = h > 0 ? `${h}:${pad(m)}:${pad(s)}` : `${m}:${pad(s)}`;
+ return tenth > 0 ? `${base}.${tenth}` : base;
+}
+
+/** `"1:34.2"` -> `94.2`. Null for anything that is not a time. */
+export function parseMark(text: string): number | null {
+ const parts = text.split(":");
+ if (parts.length < 2 || parts.length > 3) return null;
+ let total = 0;
+ for (const part of parts) {
+ const n = Number(part);
+ if (!Number.isFinite(n) || n < 0) return null;
+ total = total * 60 + n;
+ }
+ return total;
+}
+
+/** The token text for a moment: `@1:34`. */
+export const markToken = (t: number) => `@${formatMark(t)}`;
+
+/** Anything on the timeline that can carry a moment. */
+type Carrier = VideoClip | AudioClip;
+
+/** Everything on the timeline a moment can hang off, in the order a moment
+ * picks one: the spine first, then the pictures over it, then the sound. A
+ * project with no picture at all — a voiceover being assembled — anchors on
+ * its audio, and rides the same ripples. */
+function carriers(s: TimeMarkState): Carrier[] {
+ const video = s.clips
+ .filter((c) => Number.isFinite(c.start))
+ .sort((a, b) => a.track - b.track || a.start - b.start);
+ return [...video, ...(s.audioClips ?? [])];
+}
+
+/** Is `src` a source second this clip plays? Trim is in source seconds, so
+ * this survives every retime — a speed, a curve, a reverse, a preset all
+ * leave `in` and `out` where they are. */
+function playsSource(c: Carrier, src: number): boolean {
+ const lo = Math.min(c.in, c.out) - TRIM_EPS;
+ const hi = Math.max(c.in, c.out) + TRIM_EPS;
+ return src >= lo && src <= hi;
+}
+
+/** Where a clip puts a source second on the timeline, clamped to its span. */
+function timeOfSource(c: Carrier, src: number): number {
+ const len = clipLen(c);
+ return c.start + Math.min(Math.max(retimeOf(c).tAt(src), 0), len);
+}
+
+/**
+ * Hang a moment off the cut. Over a clip it holds the source second under it —
+ * the frame — so any retime, trim or split still finds it. Off every clip it
+ * holds its distance from the head of the clip it sits next to, the coming one
+ * where there is one, since a head is what an insert lands against.
+ */
+export function markAt(at: number, s: TimeMarkState): TimeMark {
+ const token = markToken(at);
+ const list = carriers(s);
+ // On a cut the incoming clip owns the moment — its first frame is what the
+ // preview shows there — so a clip's own span is checked before the tail its
+ // neighbour starts on.
+ const over =
+ list.find((c) => at >= c.start && at < c.start + clipLen(c)) ??
+ list.find((c) => at >= c.start && at <= c.start + clipLen(c));
+ if (over)
+ return {
+ token,
+ at,
+ clipId: over.id,
+ assetId: over.assetId,
+ src: retimeOf(over).srcAt(at - over.start),
+ };
+ const ahead = list
+ .filter((c) => c.start > at)
+ .sort((a, b) => a.start - b.start)[0];
+ const behind = list
+ .filter((c) => c.start < at)
+ .sort((a, b) => b.start - a.start)[0];
+ const near = ahead ?? behind;
+ return near
+ ? { token, at, clipId: near.id, assetId: near.assetId, lead: at - near.start }
+ : { token, at };
+}
+
+/** Where a mark's moment sits now. */
+export function markTime(mark: TimeMark, s: TimeMarkState): number {
+ if (!mark.clipId) return mark.at;
+ const list = carriers(s);
+ const holder = list.find((c) => c.id === mark.clipId);
+ // Off a clip: the distance from that clip's head is the whole of it. A head
+ // the split moved to another id is still findable through the source.
+ if (mark.lead !== undefined) {
+ const heads = (holder ? [holder] : list.filter((c) => c.assetId === mark.assetId))
+ .map((c) => c.start + mark.lead!)
+ .sort((a, b) => Math.abs(a - mark.at) - Math.abs(b - mark.at));
+ return heads.length > 0 ? Math.max(0, heads[0]) : mark.at;
+ }
+ const src = mark.src ?? 0;
+ // The frame, wherever it plays now. A split hands half the clip a new id, a
+ // trim can push the frame into the neighbour, and the same source can be on
+ // the timeline more than once — so every clip over that source is a
+ // candidate, and the one nearest where the moment last was wins.
+ const covering = list
+ .filter((c) => (c.id === mark.clipId || c.assetId === mark.assetId) && playsSource(c, src))
+ .map((c) => timeOfSource(c, src))
+ .sort((a, b) => Math.abs(a - mark.at) - Math.abs(b - mark.at));
+ if (covering.length > 0) return Math.max(0, covering[0]);
+ // The frame was trimmed away: the closest the clip still reaches.
+ if (holder) return Math.max(0, timeOfSource(holder, src));
+ return mark.at;
+}
+
+/**
+ * Re-read a message's moments against the live cut: every time token is
+ * rewritten to where its moment sits now, and a token that arrived without a
+ * mark — typed by hand, or restored from storage — gets one here so it moves
+ * with the cut from this point on.
+ */
+export function syncTimes(
+ text: string,
+ marks: TimeMark[],
+ s: TimeMarkState
+): { text: string; marks: TimeMark[] } {
+ const pool = [...marks];
+ const out: TimeMark[] = [];
+ let next = "";
+ let last = 0;
+ for (const m of text.matchAll(TIME_TOKEN_RE)) {
+ const written = parseMark(m[1]);
+ if (written === null) continue;
+ const held = pool.findIndex((p) => p.token === m[0]);
+ const mark = held >= 0 ? pool.splice(held, 1)[0] : markAt(written, s);
+ const at = held >= 0 ? markTime(mark, s) : written;
+ const fresh: TimeMark = { ...mark, token: markToken(at), at };
+ out.push(fresh);
+ next += text.slice(last, m.index) + fresh.token;
+ last = m.index + m[0].length;
+ }
+ return { text: next + text.slice(last), marks: out };
+}
+
+/** Split text into literal runs and time tokens, for drawing each moment as a
+ * chip. The token text is kept exactly as written, so a caller that mirrors
+ * the string character for character stays aligned. */
+export function splitTimes(text: string): (string | { text: string; at: number })[] {
+ const parts: (string | { text: string; at: number })[] = [];
+ let last = 0;
+ for (const m of text.matchAll(TIME_TOKEN_RE)) {
+ const at = parseMark(m[1]);
+ if (at === null) continue;
+ if (m.index > last) parts.push(text.slice(last, m.index));
+ parts.push({ text: m[0], at });
+ last = m.index + m[0].length;
+ }
+ if (last < text.length) parts.push(text.slice(last));
+ return parts;
+}