Open source CapCut alternative.
TypeScript
feat(site): the composer writes a moment in the cut
David Le1c4161d16b20
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsxindex bb4c5496..05eb4237 100644--- a/site/src/cut/components/AiPanel.tsx+++ b/site/src/cut/components/AiPanel.tsx@@ -97,6 +97,7 @@ import { useCreditsRecheck, useOutOfCredits } from "@/cut/lib/hosted";import { cutChatLive, dropPiSession, foldIntoCutChat, hydratePiSession, judgeEngineSkill, readPiSession, streamCutChat, triageQueuedMessages } from "@/cut/lib/pi/cutAgent";import { dueForRetriage, toolProgress } from "@/cut/lib/queueTriage";import { registerQueueSink } from "@/cut/lib/chatQueue";+import { syncTimes, type TimeMark } from "@/cut/lib/timeMark";import { productionDeps } from "@/cut/lib/pi/prodDeps";import { withChatProject } from "@/cut/lib/projectChatTools";import { runAiTool } from "@/cut/lib/aiTools";@@ -241,6 +242,14 @@ function liveRefs(refs: AssetRef[]): AssetRef[] {);}+/** A waiting row's text with its `@1:34` moments read against the cut as it+ * stands this instant — what a dispatch sends, whichever render it fires on. */+function dueText(m: QueuedMessage): string {+ if (!m.times?.length) return m.text;+ const { clips, audioClips } = useEditor.getState();+ return syncTimes(m.text, m.times, { clips, audioClips }).text;+}+function readThreads(projectId: string): ChatThread[] {return readRawThreads(projectId).filter((t) => isStoredThread(t) && !t.deleted) as ChatThread[];}@@ -391,6 +400,11 @@ function readStoredQueue(.map((m) => ({...m,attachments: Array.isArray(m.attachments) ? m.attachments : [],+ // A stored mark with no token or no time would format as nonsense;+ // the row's written time stands on its own without it.+ times: (Array.isArray(m.times) ? m.times : []).filter(+ (t) => typeof t?.token === "string" && Number.isFinite(t?.at),+ ),}));} catch {return [];@@ -926,6 +940,10 @@ function ChatSession({micWasActive.current = mic.state !== "idle";}, [mic.state]);const [attachments, setAttachments] = useState<AssetRef[]>([]);+ // Moments the composer's `@` tokens point at, in the order they were+ // written. They ride with the message so the time can be re-read against+ // the cut the message actually runs on.+ const [times, setTimes] = useState<TimeMark[]>([]);const candidates = useRefCandidates(visible);// Any OS file drag over the window hints the composer as a drop target;// hovering it (dropActive below) strengthens the ring and shows the label.@@ -1533,14 +1551,15 @@ function ChatSession({)continue;const live = liveRefs(row.attachments);+ const text = dueText(current);if (verdict === "spawn") {- const id = onSpawn({ text: row.text, attachments: live, model });+ const id = onSpawn({ text, attachments: live, model });setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "spawned", threadId: id } : m)));} else if (verdict === "fold" && provider(model) === "gemini" && cutChatLive(threadId)) {setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "folding" } : m)));const folded = await foldIntoCutChat({threadId,- text: row.text,+ text,attachments: live,deps: productionDeps(projectId, turnSignalRef.current),});@@ -1553,7 +1572,7 @@ function ChatSession({const user: UIMessage = {id: crypto.randomUUID(),role: "user",- parts: [{ type: "text", text: row.text }],+ parts: [{ type: "text", text }],...(live.length > 0 && { metadata: { attachments: live } }),};const tail = msgs.at(-1);@@ -1607,6 +1626,27 @@ function ChatSession({),[threadId],);+ // A waiting row keeps pointing at the same moment of the video while the+ // running turn edits the cut under it. Each row's marks are re-read against+ // the live clips and its `@1:34` tokens rewritten, so the tray shows — and+ // the dispatch sends — the time that moment sits at now. The row being+ // edited is left alone; its text belongs to the editor until it commits.+ const docClips = useEditor((s) => s.clips);+ const docAudio = useEditor((s) => s.audioClips);+ useEffect(() => {+ // eslint-disable-next-line react-hooks/set-state-in-effect -- the rows react to the cut moving; the write is skipped unless a token's time actually changed+ setQueue((q) => {+ let moved = false;+ const next = q.map((m) => {+ if (!m.times?.length || m.status !== "queued" || m.id === queueEditing) return m;+ const synced = syncTimes(m.text, m.times, { clips: docClips, audioClips: docAudio });+ if (synced.text === m.text) return m;+ moved = true;+ return { ...m, text: synced.text, times: synced.marks };+ });+ return moved ? next : q;+ });+ }, [docClips, docAudio, queueEditing]);// Height of the floating stack above the composer (warning tabs + queue// tray); the messages pad their bottom by it so the newest message can// scroll out from behind the stack.@@ -1632,7 +1672,14 @@ function ChatSession({// Inline @mentions attach their assets alongside the dropped chips. The// message keeps the raw tokens — they render as interactive chips and the// model reads the handle↔asset mapping from <attached_assets>.- const body = text.trim();+ // Moments are re-read here too: the cut can have moved since the token was+ // written, and a token typed by hand picks up its mark now.+ const doc = useEditor.getState();+ const synced = syncTimes(text.trim(), times, {+ clips: doc.clips,+ audioClips: doc.audioClips,+ });+ const body = synced.text;const { refs: all } = collectRefs(body, attachments, candidates);if (!body && all.length === 0) return;// An unavailable provider surfaces as an error in the thread at send time@@ -1645,10 +1692,17 @@ function ChatSession({// A paused queue stays paused: parked rows fire only from the tray's// resume button, never as a side effect of submitting something new.// With the queue live, the triage places the row right away.- const row: QueuedMessage = { id: crypto.randomUUID(), text: body, attachments: all, status: "queued" };+ const row: QueuedMessage = {+ id: crypto.randomUUID(),+ text: body,+ attachments: all,+ status: "queued",+ ...(synced.marks.length > 0 && { times: synced.marks }),+ };setQueue((q) => [...q, row]);setInput("");setAttachments([]);+ setTimes([]);pinnedRef.current = true;if (!queuePaused) void triageRef.current([row]);return;@@ -1674,6 +1728,7 @@ function ChatSession({}));setInput("");setAttachments([]);+ setTimes([]);};// A turn the page lost picks itself back up. With an engine journal the@@ -1774,9 +1829,10 @@ function ChatSession({// parked (or at file blobs that died with the last page); dropping the// dead refs keeps the message from claiming media it can't deliver.const live = liveRefs(next.attachments);- runningAskRef.current = next.text;+ const text = dueText(next);+ runningAskRef.current = text;dispatchChat(() => sendMessage({- text: next.text,+ text,...(live.length > 0 && { metadata: { attachments: live } }),}));// The rows behind it are placed against the turn that just started:@@ -1972,9 +2028,16 @@ function ChatSession({}}onCommitEdit={(id, text) =>setQueue((q) =>- q.map((m) =>- m.id === id ? { ...m, text: text.trim() || m.text } : m,- ),+ q.map((m) => {+ if (m.id !== id) return m;+ // A rewrite can drop a moment or type a new one; re-read+ // the row's marks off the text it now carries.+ const synced = syncTimes(text.trim() || m.text, m.times ?? [], {+ clips: docClips,+ audioClips: docAudio,+ });+ return { ...m, text: synced.text, times: synced.marks };+ }),)}onRemove={(id) => setQueue((q) => q.filter((m) => m.id !== id))}@@ -2061,6 +2124,7 @@ function ChatSession({onUpsertRef={(ref) => setAttachments((p) => upsertRef(p, ref))}onPasteFiles={attachFiles}onRemoveLastRef={() => setAttachments((p) => p.slice(0, -1))}+ onInsertTime={(mark) => setTimes((p) => [...p, mark])}/><div className="flex items-center gap-1 px-1.5 pb-1.5"><ModelSelectordiff --git a/site/src/cut/components/AssetRefs.tsx b/site/src/cut/components/AssetRefs.tsxindex e33ad41f..eb1f4780 100644--- a/site/src/cut/components/AssetRefs.tsx+++ b/site/src/cut/components/AssetRefs.tsx@@ -1,7 +1,7 @@"use client";import { useEffect, useLayoutEffect, useMemo, useRef, useState, type ReactNode, type RefObject } from "react";-import { Check, Copy, FileText, X } from "lucide-react";+import { Check, Clock, Copy, FileText, X } from "lucide-react";import {highlightMentions,liveRefUrl,@@ -19,7 +19,16 @@ import { useInView } from "@/cut/hooks/useInView";import { revealRef } from "@/cut/lib/refReveal";import { formatTime } from "@/cut/lib/time";import { clipLen, getClipSpans, useEditor } from "@/cut/lib/store";-import { playheadAt } from "@/cut/lib/playhead";+import { playheadAt, previewAt, usePreviewTimeEvery } from "@/cut/lib/playhead";+import {+ formatMark,+ markAt,+ parseMark,+ PLAYHEAD_HANDLE,+ PLAYHEAD_NAME,+ splitTimes,+ type TimeMark,+} from "@/cut/lib/timeMark";import { AudioPillSurface } from "@/cut/components/AudioPanel";import { entityGlyph } from "@/cut/components/entityIcons";import {@@ -345,6 +354,38 @@ export function RefTokenChip({);}+/**+ * An inline `@1:34` moment token: a place in the cut the message points at.+ * Clicking it takes the playhead there. Written messages hold the moment they+ * ran against; a message still waiting in the queue holds the moment it will+ * run against, re-read as the cut moves under it.+ */+export function TimeTokenChip({+ text,+ at,+ onDark,+}: {+ text: string;+ at: number;+ onDark?: boolean;+}) {+ return (+ <button+ type="button"+ className={cn(+ "ref-token inline-flex items-center gap-0.5 rounded-md px-1 align-middle font-mono text-[11px] transition-colors",+ onDark+ ? "bg-white/15 text-[#8ec7ff] hover:bg-white/25"+ : "bg-[#0a84ff]/10 text-[#0a84ff] hover:bg-[#0a84ff]/20"+ )}+ onClick={() => useEditor.getState().seek(at)}+ >+ <Clock className="size-2.5 shrink-0" />+ {text.slice(1)}+ </button>+ );+}+/** Message text with its resolved `@` mentions drawn as token chips. One* renderer for every place a written message is shown — the chat thread and* the queue tray above the composer — so a message reads the same before it@@ -362,8 +403,13 @@ export function MentionedText({onDark?: boolean;}) {const candidates = useRefCandidates();+ // Mentions first, then the moments inside what is left over: an asset's+ // name never looks like a time, so the two passes never fight over a token.const parts = useMemo(- () => splitMentions(text, [...attachments, ...candidates]),+ () =>+ splitMentions(text, [...attachments, ...candidates]).flatMap<+ string | AssetRef | { text: string; at: number }+ >((p) => (typeof p === "string" ? splitTimes(p) : [p])),[text, attachments, candidates]);return (@@ -371,6 +417,8 @@ export function MentionedText({{parts.map((p, i) =>typeof p === "string" ? (<span key={i}>{p}</span>+ ) : "at" in p ? (+ <TimeTokenChip key={i} text={p.text} at={p.at} onDark={onDark} />) : (<RefTokenChip key={i} item={p} onDark={onDark} />)@@ -939,6 +987,18 @@ function mentionAtCaret(value: string, caret: number): { start: number; query: sreturn { start: at, query };}+/** The preview's moment on the `@` menu's playhead row. Mounted only while+ * the menu is open, and reading the clock a few times a second: the composer+ * itself never re-renders on the playhead. */+function PlayheadRowTime() {+ const at = usePreviewTimeEvery(10) / 10;+ return (+ <span className="shrink-0 font-mono text-[10.5px] text-muted-foreground tabular-nums">+ {formatMark(at)}+ </span>+ );+}+/** One resolved mention in the composer mirror. Entity pills paint over the* raw token with the entity's icon (or the sticker's own art) and name — the* token glyphs underneath keep their exact layout, so the textarea's caret@@ -1100,6 +1160,7 @@ export function MentionTextarea({onUpsertRef,onPasteFiles,onRemoveLastRef,+ onInsertTime,}: {value: string;onChange: (v: string) => void;@@ -1130,6 +1191,11 @@ export function MentionTextarea({/** Backspace with the caret at the very start (nothing selected, nothing to* delete leftward) removes the newest attachment chip instead. */onRemoveLastRef?: () => void;+ /** Take a moment in the cut the user picked off the `@` menu. Providing it+ * puts the playhead at the top of that menu, so `@here` writes the time the+ * preview is sitting on; the mark comes with it so the caller can re-read+ * the moment if the cut moves before the message goes out. */+ onInsertTime?: (mark: TimeMark) => void;}) {const taRef = useRef<HTMLTextAreaElement>(null);const rootRef = useRef<HTMLDivElement>(null);@@ -1183,16 +1249,29 @@ export function MentionTextarea({// Mirror segments with their character offsets, so a pill can tell when// the caret sits inside its token.+ // Moments pill only where they mean something — the surface that takes them.+ const takesTime = !!onInsertTime;const mirrorSegs = useMemo(() =>- highlightMentions(value, pool).reduce<- { text: string; ref: AssetRef | null; start: number }[]- >((acc, seg) => {- const prev = acc[acc.length - 1];- acc.push({ ...seg, start: prev ? prev.start + prev.text.length : 0 });- return acc;- }, []),- [value, pool]+ highlightMentions(value, pool)+ // Moments live in the runs no mention claimed.+ .flatMap((seg) =>+ seg.ref || !takesTime+ ? [seg]+ : splitTimes(seg.text).map((part) =>+ typeof part === "string"+ ? { text: part, ref: null, time: false }+ : { text: part.text, ref: null, time: true }+ )+ )+ .reduce<+ { text: string; ref: AssetRef | null; time?: boolean; start: number }[]+ >((acc, seg) => {+ const prev = acc[acc.length - 1];+ acc.push({ ...seg, start: prev ? prev.start + prev.text.length : 0 });+ return acc;+ }, []),+ [value, pool, takesTime]);// A pill cover is one box, so its token must move between lines whole —@@ -1218,15 +1297,19 @@ export function MentionTextarea({}, [mirrorSegs, value, onChange]);const mention = useMemo(() => mentionAtCaret(value, caret), [value, caret]);+ // A caret strictly inside an intact, resolved token is a click on the pill —+ // the mention is complete, and offering lookalike suggestions for its+ // half-read query would misdirect it.+ const inResolved = useMemo(+ () =>+ mirrorSegs.some(+ (seg) =>+ (seg.ref || seg.time) && caret > seg.start && caret < seg.start + seg.text.length+ ),+ [mirrorSegs, caret]+ );const matches = useMemo(() => {- if (!mention || dismissed === mention.start) return [];- // A caret strictly inside an intact, resolved token is a click on the- // pill — the mention is complete, and offering lookalike suggestions for- // its half-read query would misdirect it.- const inResolved = mirrorSegs.some(- (seg) => seg.ref && caret > seg.start && caret < seg.start + seg.text.length- );- if (inResolved) return [];+ if (!mention || dismissed === mention.start || inResolved) return [];const q = mention.query.toLowerCase();// Best match first, not list order: a typed handle prefix ("c" → c1, c2)// beats a name prefix, which beats a substring hit anywhere in the name —@@ -1248,14 +1331,24 @@ export function MentionTextarea({.sort((a, b) => a.r - b.r).slice(0, 8).map((x) => x.c);- }, [mention, dismissed, candidates, mirrorSegs, caret]);- const open = matches.length > 0;+ }, [mention, dismissed, inResolved, candidates]);+ // The playhead leads the menu wherever moments are taken: a bare `@`, or a+ // query that starts the row's handle or name. Matched the way a candidate+ // is, so `@here` and `@play` both land on it.+ const timeRow =+ !!onInsertTime &&+ !!mention &&+ dismissed !== mention.start &&+ !inResolved &&+ [PLAYHEAD_HANDLE, PLAYHEAD_NAME].some((n) => n.startsWith(mention.query.toLowerCase()));+ const rowCount = matches.length + (timeRow ? 1 : 0);+ const open = rowCount > 0;// Each keystroke re-ranks the list, so a highlight chosen under a previous// query derives back to the best (first) match instead of holding a stale// arrow/hover position.const sel = selState.q === mention?.query ? selState.i : 0;const setSel = (i: number) => setSelState({ q: mention?.query, i });- const selIndex = Math.min(sel, matches.length - 1);+ const selIndex = Math.min(sel, rowCount - 1);const syncCaret = () => {const el = taRef.current;@@ -1335,9 +1428,9 @@ export function MentionTextarea({});};- const pick = (ref: AssetRef) => {+ /** Write `token` over the mention being typed and leave the caret after it. */+ const writeToken = (token: string) => {if (!mention) return;- const token = refToken(ref) + " ";const next = value.slice(0, mention.start) + token + value.slice(caret);const newCaret = mention.start + token.length;onChange(next);@@ -1351,6 +1444,28 @@ export function MentionTextarea({});};+ const pick = (ref: AssetRef) => writeToken(refToken(ref) + " ");++ /** Write the preview's moment as a token, and hand the caller the mark that+ * holds what it points at, so the time can be re-read if the cut moves. */+ const pickTime = () => {+ if (!onInsertTime) return;+ const { clips, audioClips } = useEditor.getState();+ const mark = markAt(previewAt(), { clips, audioClips });+ onInsertTime(mark);+ writeToken(mark.token + " ");+ };++ /** Take the menu row at `i`: the playhead leads the list when it is shown. */+ const pickRow = (i: number) => {+ if (timeRow && i === 0) {+ pickTime();+ return;+ }+ const ref = matches[timeRow ? i - 1 : i];+ if (ref) pick(ref);+ };+// Auto-grow: reset to natural height, then match the content. The caller's// `max-h-*` caps it and the textarea scrolls internally past that point.useLayoutEffect(() => {@@ -1370,26 +1485,50 @@ export function MentionTextarea({menuSide === "top" ? "bottom-full mb-1" : "top-full mt-1")}>- {matches.map((c, i) => (+ {timeRow && (<button- key={`${c.scope}:${c.id}`}type="button"className={cn("flex w-full items-center gap-2 rounded-md px-1.5 py-1 text-left",- i === selIndex ? "bg-muted" : "hover:bg-muted/60"+ selIndex === 0 ? "bg-muted" : "hover:bg-muted/60")}- onMouseEnter={() => setSel(i)}- // mousedown, not click: keep focus (and the mention state) in the textarea.+ onMouseEnter={() => setSel(0)}onMouseDown={(e) => {e.preventDefault();- pick(c);+ pickTime();}}>- <RefThumb item={c} className="size-8" />- {c.handle && <RefHandleBadge handle={c.handle} className="shrink-0" />}- <span className="min-w-0 flex-1 truncate text-[11.5px]">{c.name}</span>+ <span className="grid size-8 shrink-0 place-items-center rounded-lg border border-border bg-muted text-[#0a84ff]">+ <Clock className="size-4" />+ </span>+ <RefHandleBadge handle={PLAYHEAD_HANDLE} className="shrink-0" />+ <span className="min-w-0 flex-1 truncate text-[11.5px]">{PLAYHEAD_NAME}</span>+ <PlayheadRowTime /></button>- ))}+ )}+ {matches.map((c, i) => {+ const row = timeRow ? i + 1 : i;+ return (+ <button+ key={`${c.scope}:${c.id}`}+ type="button"+ className={cn(+ "flex w-full items-center gap-2 rounded-md px-1.5 py-1 text-left",+ row === selIndex ? "bg-muted" : "hover:bg-muted/60"+ )}+ onMouseEnter={() => setSel(row)}+ // mousedown, not click: keep focus (and the mention state) in the textarea.+ onMouseDown={(e) => {+ e.preventDefault();+ pick(c);+ }}+ >+ <RefThumb item={c} className="size-8" />+ {c.handle && <RefHandleBadge handle={c.handle} className="shrink-0" />}+ <span className="min-w-0 flex-1 truncate text-[11.5px]">{c.name}</span>+ </button>+ );+ })}</div>)}<textarea@@ -1517,14 +1656,12 @@ export function MentionTextarea({if (open) {if (e.key === "ArrowDown" || e.key === "ArrowUp") {e.preventDefault();- setSel(- (selIndex + (e.key === "ArrowDown" ? 1 : matches.length - 1)) % matches.length- );+ setSel((selIndex + (e.key === "ArrowDown" ? 1 : rowCount - 1)) % rowCount);return;}if (e.key === "Enter" || e.key === "Tab") {e.preventDefault();- pick(matches[selIndex]);+ pickRow(selIndex);return;}if (e.key === "Escape") {@@ -1561,14 +1698,33 @@ export function MentionTextarea({)}>{mirrorSegs.map((seg, i) => {+ // A moment pills over its own glyphs — the raw token keeps its+ // exact layout, so the caret math never drifts — and takes a click+ // to put the playhead back where the token points.+ if (seg.time)+ return (+ <span+ key={i}+ className="pointer-events-auto cursor-pointer rounded-[4px] bg-[#0a84ff]/12 hover:bg-[#0a84ff]/25"+ onMouseDown={(e) => {+ e.preventDefault();+ const at = parseMark(seg.text.slice(1));+ if (at !== null) useEditor.getState().seek(at);+ }}+ >+ {seg.text}+ </span>+ );if (!seg.ref) return <span key={i}>{seg.text}</span>;// A side facing another pill across a single space keeps its bleed// off — two bleeds would swallow the only gap between the pills.+ // A moment counts as a pill here too.const facing = (gap: number, other: number) =>!!mirrorSegs[gap] &&!mirrorSegs[gap].ref &&+ !mirrorSegs[gap].time &&/^\s$/.test(mirrorSegs[gap].text) &&- !!mirrorSegs[other]?.ref;+ !!(mirrorSegs[other]?.ref || mirrorSegs[other]?.time);const end = seg.start + seg.text.length;// Raw text shows only while a range endpoint rests strictly inside// the token; entity endpoints normalize to the token edges, so andiff --git a/site/src/cut/components/ComposerQueue.tsx b/site/src/cut/components/ComposerQueue.tsxindex e85c7957..31a4c9ec 100644--- a/site/src/cut/components/ComposerQueue.tsx+++ b/site/src/cut/components/ComposerQueue.tsx@@ -17,6 +17,7 @@ import { ScrollArea } from "@/components/ui/scroll-area";import { useDragSort } from "@/components/Sortable";import { MentionedText, RefThumb } from "./AssetRefs";import type { AssetRef } from "@/cut/lib/assetRef";+import type { TimeMark } from "@/cut/lib/timeMark";/** Small icon control on a tray row or the tray header. */const queueIconButton =@@ -34,6 +35,10 @@ export interface QueuedMessage {attachments: AssetRef[];status: "queued" | "folding" | "spawned" | "running" | "done";threadId?: string;+ /** What the row's `@1:34` moments point at. A waiting row's times are+ * re-read as the running turn moves the cut, so the text always names the+ * moment the row will actually run against. */+ times?: TimeMark[];}/** The queue tray: a folder tab on top of the composer holding the messagesdiff --git a/site/src/cut/lib/timeMark.test.ts b/site/src/cut/lib/timeMark.test.tsnew file mode 100644index 00000000..5e1ed123--- /dev/null+++ b/site/src/cut/lib/timeMark.test.ts@@ -0,0 +1,222 @@+import { describe, expect, test } from "bun:test";+import { retimeOf } from "@donkeycut/effects-kit";+import { formatMark, markAt, markTime, parseMark, splitTimes, syncTimes } from "./timeMark";+import { splitTimelineItems } from "./timelineItems";+import type { AudioClip, VideoClip } from "./types";++const clip = (id: string, start: number, patch: Partial<VideoClip> = {}): VideoClip => ({+ id,+ assetId: id,+ track: 0,+ start,+ in: 0,+ out: 10,+ muted: false,+ ...patch,+});++const sound = (id: string, start: number, patch: Partial<AudioClip> = {}): AudioClip => ({+ id,+ assetId: id,+ start,+ in: 0,+ out: 20,+ volume: 1,+ ...patch,+});++/** a: 0–10, b: 10–20 on track 0. */+const cut = () => ({ clips: [clip("a", 0), clip("b", 10)] });++/** The source second the cut actually shows at `t`. */+function frameAt(s: { clips: VideoClip[] }, t: number): number | null {+ const c = s.clips.find((x) => t >= x.start && t <= x.start + retimeOf(x).len);+ return c ? retimeOf(c).srcAt(t - c.start) : null;+}++describe("time tokens", () => {+ test("format drops a zero tenth and grows an hour field", () => {+ expect(formatMark(94)).toBe("1:34");+ expect(formatMark(94.24)).toBe("1:34.2");+ expect(formatMark(8.02)).toBe("0:08");+ expect(formatMark(3723)).toBe("1:02:03");+ expect(formatMark(-5)).toBe("0:00");+ });++ test("parse reads both shapes", () => {+ expect(parseMark("1:34")).toBe(94);+ expect(parseMark("1:34.5")).toBe(94.5);+ expect(parseMark("1:02:03")).toBe(3723);+ expect(parseMark("nope")).toBe(null);+ });++ test("split keeps the written token and leaves the rest of the text", () => {+ expect(splitTimes("zoom at @1:34 please")).toEqual([+ "zoom at ",+ { text: "@1:34", at: 94 },+ " please",+ ]);+ // A handle that merely starts with digits is not a moment.+ expect(splitTimes("@v2 is loud")).toEqual(["@v2 is loud"]);+ expect(splitTimes("@1:34:56:78")).toEqual(["@1:34:56:78"]);+ });+});++describe("a moment holds its frame", () => {+ test("through a ripple", () => {+ const s = cut();+ const mark = markAt(12, s);+ expect(mark.clipId).toBe("b");+ expect(mark.src).toBeCloseTo(2);+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 13)] })).toBeCloseTo(15);+ });++ test("through a head trim", () => {+ const mark = markAt(12, cut());+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 10, { in: 1 })] })).toBeCloseTo(11);+ });++ test("through a split, on either side of the cut", () => {+ const s = cut();+ const early = markAt(12, s);+ const late = markAt(17, s);+ let n = 0;+ const after = splitTimelineItems(+ { ...s, audioClips: [], overlays: [], subtitles: { cues: [] } } as never,+ [{ kind: "clip", id: "b" }],+ 15,+ () => `new${++n}`,+ ).next as { clips: VideoClip[] };+ // The right half is a clip the mark has never seen.+ expect(after.clips.map((c) => c.id)).toEqual(["a", "b", "new1"]);+ expect(markTime(early, after)).toBeCloseTo(12);+ expect(markTime(late, after)).toBeCloseTo(17);+ // And it follows that new half when the turn moves it.+ const moved = { clips: after.clips.map((c) => (c.id === "new1" ? { ...c, start: 18 } : c)) };+ expect(markTime(late, moved)).toBeCloseTo(20);+ });++ test("through a speed change, a speed curve and a reverse", () => {+ const s = cut();+ const mark = markAt(17, s);+ expect(mark.src).toBeCloseTo(7);+ for (const patch of [+ { speed: 2 },+ { speed: 0.5 },+ { speedCurve: [[0, 0.5], [0.5, 2], [1, 1]] as [number, number][] },+ { reverse: true },+ { speed: 2, reverse: true },+ ]) {+ const next = { clips: [clip("a", 0), clip("b", 10, patch)] };+ const at = markTime(mark, next);+ // Wherever the moment lands, the cut shows the frame it was written on.+ expect(frameAt(next, at)).toBeCloseTo(7, 4);+ }+ });++ test("through a retime that also moves the clip", () => {+ const mark = markAt(17, cut());+ const next = { clips: [clip("a", 0), clip("b", 30, { speed: 4 })] };+ expect(frameAt(next, markTime(mark, next))).toBeCloseTo(7, 4);+ });++ test("and clamps into the clip when the frame is trimmed away", () => {+ const mark = markAt(17, cut());+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 10, { out: 5 })] })).toBeCloseTo(15);+ });++ test("on a cut belongs to the incoming clip", () => {+ const mark = markAt(10, cut());+ expect(mark.clipId).toBe("b");+ expect(mark.src).toBeCloseTo(0);+ // Trimming the outgoing clip's tail pulls the cut back, and with it the moment.+ expect(markTime(mark, { clips: [clip("a", 0, { out: 8 }), clip("b", 8)] })).toBeCloseTo(8);+ });++ test("and picks the nearer copy when the source is on the timeline twice", () => {+ const mark = markAt(17, cut());+ const twice = { clips: [clip("a", 0), clip("b", 10), clip("b2", 20, { assetId: "b" })] };+ expect(markTime(mark, twice)).toBeCloseTo(17);+ });+});++describe("a moment off the clips", () => {+ test("in a gap rides the head of the clip ahead of it", () => {+ const s = { clips: [clip("a", 0), clip("b", 14)] };+ const mark = markAt(12, s);+ expect(mark.clipId).toBe("b");+ expect(mark.lead).toBeCloseTo(-2);+ expect(markTime(mark, { clips: [clip("a", 0), clip("b", 19)] })).toBeCloseTo(17);+ });++ test("past the end rides the head of the last clip", () => {+ const s = { clips: [clip("a", 0)] };+ const mark = markAt(13, s);+ expect(mark.clipId).toBe("a");+ expect(mark.lead).toBeCloseTo(13);+ expect(markTime(mark, { clips: [clip("a", 4)] })).toBeCloseTo(17);+ });++ test("in a gap finds the head again after the clip ahead is split", () => {+ const s = { clips: [clip("a", 0), clip("b", 14)] };+ const mark = markAt(12, s);+ // The split leaves b's head where it was; the moment stays two before it.+ let n = 0;+ const after = splitTimelineItems(+ { ...s, audioClips: [], overlays: [], subtitles: { cues: [] } } as never,+ [{ kind: "clip", id: "b" }],+ 18,+ () => `new${++n}`,+ ).next as { clips: VideoClip[] };+ expect(markTime(mark, after)).toBeCloseTo(12);+ });++ test("with no clips at all keeps its second", () => {+ const mark = markAt(4, { clips: [] });+ expect(mark.clipId).toBeUndefined();+ expect(markTime(mark, cut())).toBe(4);+ });++ test("whose clip is gone keeps its second", () => {+ const mark = markAt(12, cut());+ expect(markTime(mark, { clips: [clip("a", 0)] })).toBe(12);+ });+});++describe("a project with no picture", () => {+ test("anchors on the soundtrack and rides its ripple", () => {+ const s = { clips: [], audioClips: [sound("vo", 0)] };+ const mark = markAt(12, s);+ expect(mark.clipId).toBe("vo");+ expect(mark.src).toBeCloseTo(12);+ expect(markTime(mark, { clips: [], audioClips: [sound("vo", 5)] })).toBeCloseTo(17);+ });++ test("but the picture wins wherever there is one", () => {+ const mark = markAt(12, { ...cut(), audioClips: [sound("vo", 0)] });+ expect(mark.clipId).toBe("b");+ });+});++describe("syncTimes", () => {+ test("rewrites every token to where its moment sits now", () => {+ const s = cut();+ const written = syncTimes("trim @0:05 and title @0:12", [], s);+ expect(written.text).toBe("trim @0:05 and title @0:12");+ expect(written.marks.map((m) => m.clipId)).toEqual(["a", "b"]);+ const moved = { clips: [clip("a", 0, { out: 14 }), clip("b", 14)] };+ expect(syncTimes(written.text, written.marks, moved).text).toBe("trim @0:05 and title @0:16");+ });++ test("a token typed by hand picks up a mark on the way out", () => {+ const { marks } = syncTimes("cut at @0:12", [], cut());+ expect(marks).toHaveLength(1);+ expect(marks[0].clipId).toBe("b");+ });++ test("a rewrite drops the marks its tokens left behind", () => {+ const s = cut();+ const first = syncTimes("@0:05 and @0:12", [], s);+ expect(syncTimes("@0:12 only", first.marks, s).marks).toHaveLength(1);+ });+});diff --git a/site/src/cut/lib/timeMark.ts b/site/src/cut/lib/timeMark.tsnew file mode 100644index 00000000..a587589c--- /dev/null+++ b/site/src/cut/lib/timeMark.ts@@ -0,0 +1,232 @@+"use client";++import { retimeOf } from "@donkeycut/effects-kit";+import { clipLen } from "./store";+import type { AudioClip, VideoClip } from "./types";++/**+ * Moments a message points at.+ *+ * A chat message can name a place in the cut — "@1:34 zoom in here". The time+ * is written in the text, so the model reads it the way the user wrote it, and+ * a mark beside the text records the frame it was written on: the clip under+ * the moment, the source it plays, and the second inside that source.+ *+ * Between writing a message and running it the cut can move. The queue holds+ * messages while a turn runs, and that turn ripples, splits, retimes, trims+ * and deletes. So a queued message re-reads its marks against the live+ * document before it goes out, and the token in the text is rewritten to the+ * second that frame has reached.+ *+ * A source second is what makes that hold. Trim is in source seconds, so a+ * speed, a curve, a reverse and a preset all leave it alone and the clip's own+ * map puts it back on the timeline; a split copies it to both halves, so the+ * frame is found again through the source even on the half that took a new id.+ * A moment sitting in a gap has no frame, so it holds its distance from the+ * head of the clip beside it — a head is the edge an insert lands against, and+ * the one a split leaves where it was. A moment whose clip is gone altogether+ * keeps the second it was written at.+ */++/** A time token in message text, e.g. `@1:34` or `@0:02.5` or `@1:02:30`. */+export const TIME_TOKEN_RE = /@(\d{1,3}:[0-5]\d(?::[0-5]\d)?(?:\.\d{1,2})?)(?![\d:])/g;++/** What `@` offers for the playhead: the row's handle and its name, matched+ * by the picker the same way an asset's are. */+export const PLAYHEAD_HANDLE = "here";+export const PLAYHEAD_NAME = "playhead";++/** How far outside a clip's trim a source second may sit and still count as+ * inside it — a rounding tolerance, not a reach. */+const TRIM_EPS = 1e-6;++/** A moment a message points at, and what it hangs off in the cut. */+export interface TimeMark {+ /** The token as it stands in the text right now. */+ token: string;+ /** The timeline second the token reads as. */+ at: number;+ /** The clip the moment hangs off; absent when nothing carries it. */+ clipId?: string;+ /** The source that clip plays, so the frame is findable even after the+ * clip it was written over is split, replaced or renumbered. */+ assetId?: string;+ /** The source second under the moment, for a moment over the clip itself. */+ src?: number;+ /** Seconds from the clip's head, for a moment sitting off the clip — the+ * head is the stable edge: splitting a clip leaves its head where it was. */+ lead?: number;+}++/** What the marks read against: everything on the timeline that carries time. */+export interface TimeMarkState {+ clips: VideoClip[];+ audioClips?: AudioClip[];+}++/** `94.2` -> `"1:34.2"`, `94` -> `"1:34"`, `3723` -> `"1:02:03"`. */+export function formatMark(t: number): string {+ const tenths = Math.round(Math.max(0, t) * 10);+ const whole = Math.floor(tenths / 10);+ const tenth = tenths % 10;+ const h = Math.floor(whole / 3600);+ const m = Math.floor((whole % 3600) / 60);+ const s = whole % 60;+ const pad = (n: number) => String(n).padStart(2, "0");+ const base = h > 0 ? `${h}:${pad(m)}:${pad(s)}` : `${m}:${pad(s)}`;+ return tenth > 0 ? `${base}.${tenth}` : base;+}++/** `"1:34.2"` -> `94.2`. Null for anything that is not a time. */+export function parseMark(text: string): number | null {+ const parts = text.split(":");+ if (parts.length < 2 || parts.length > 3) return null;+ let total = 0;+ for (const part of parts) {+ const n = Number(part);+ if (!Number.isFinite(n) || n < 0) return null;+ total = total * 60 + n;+ }+ return total;+}++/** The token text for a moment: `@1:34`. */+export const markToken = (t: number) => `@${formatMark(t)}`;++/** Anything on the timeline that can carry a moment. */+type Carrier = VideoClip | AudioClip;++/** Everything on the timeline a moment can hang off, in the order a moment+ * picks one: the spine first, then the pictures over it, then the sound. A+ * project with no picture at all — a voiceover being assembled — anchors on+ * its audio, and rides the same ripples. */+function carriers(s: TimeMarkState): Carrier[] {+ const video = s.clips+ .filter((c) => Number.isFinite(c.start))+ .sort((a, b) => a.track - b.track || a.start - b.start);+ return [...video, ...(s.audioClips ?? [])];+}++/** Is `src` a source second this clip plays? Trim is in source seconds, so+ * this survives every retime — a speed, a curve, a reverse, a preset all+ * leave `in` and `out` where they are. */+function playsSource(c: Carrier, src: number): boolean {+ const lo = Math.min(c.in, c.out) - TRIM_EPS;+ const hi = Math.max(c.in, c.out) + TRIM_EPS;+ return src >= lo && src <= hi;+}++/** Where a clip puts a source second on the timeline, clamped to its span. */+function timeOfSource(c: Carrier, src: number): number {+ const len = clipLen(c);+ return c.start + Math.min(Math.max(retimeOf(c).tAt(src), 0), len);+}++/**+ * Hang a moment off the cut. Over a clip it holds the source second under it —+ * the frame — so any retime, trim or split still finds it. Off every clip it+ * holds its distance from the head of the clip it sits next to, the coming one+ * where there is one, since a head is what an insert lands against.+ */+export function markAt(at: number, s: TimeMarkState): TimeMark {+ const token = markToken(at);+ const list = carriers(s);+ // On a cut the incoming clip owns the moment — its first frame is what the+ // preview shows there — so a clip's own span is checked before the tail its+ // neighbour starts on.+ const over =+ list.find((c) => at >= c.start && at < c.start + clipLen(c)) ??+ list.find((c) => at >= c.start && at <= c.start + clipLen(c));+ if (over)+ return {+ token,+ at,+ clipId: over.id,+ assetId: over.assetId,+ src: retimeOf(over).srcAt(at - over.start),+ };+ const ahead = list+ .filter((c) => c.start > at)+ .sort((a, b) => a.start - b.start)[0];+ const behind = list+ .filter((c) => c.start < at)+ .sort((a, b) => b.start - a.start)[0];+ const near = ahead ?? behind;+ return near+ ? { token, at, clipId: near.id, assetId: near.assetId, lead: at - near.start }+ : { token, at };+}++/** Where a mark's moment sits now. */+export function markTime(mark: TimeMark, s: TimeMarkState): number {+ if (!mark.clipId) return mark.at;+ const list = carriers(s);+ const holder = list.find((c) => c.id === mark.clipId);+ // Off a clip: the distance from that clip's head is the whole of it. A head+ // the split moved to another id is still findable through the source.+ if (mark.lead !== undefined) {+ const heads = (holder ? [holder] : list.filter((c) => c.assetId === mark.assetId))+ .map((c) => c.start + mark.lead!)+ .sort((a, b) => Math.abs(a - mark.at) - Math.abs(b - mark.at));+ return heads.length > 0 ? Math.max(0, heads[0]) : mark.at;+ }+ const src = mark.src ?? 0;+ // The frame, wherever it plays now. A split hands half the clip a new id, a+ // trim can push the frame into the neighbour, and the same source can be on+ // the timeline more than once — so every clip over that source is a+ // candidate, and the one nearest where the moment last was wins.+ const covering = list+ .filter((c) => (c.id === mark.clipId || c.assetId === mark.assetId) && playsSource(c, src))+ .map((c) => timeOfSource(c, src))+ .sort((a, b) => Math.abs(a - mark.at) - Math.abs(b - mark.at));+ if (covering.length > 0) return Math.max(0, covering[0]);+ // The frame was trimmed away: the closest the clip still reaches.+ if (holder) return Math.max(0, timeOfSource(holder, src));+ return mark.at;+}++/**+ * Re-read a message's moments against the live cut: every time token is+ * rewritten to where its moment sits now, and a token that arrived without a+ * mark — typed by hand, or restored from storage — gets one here so it moves+ * with the cut from this point on.+ */+export function syncTimes(+ text: string,+ marks: TimeMark[],+ s: TimeMarkState+): { text: string; marks: TimeMark[] } {+ const pool = [...marks];+ const out: TimeMark[] = [];+ let next = "";+ let last = 0;+ for (const m of text.matchAll(TIME_TOKEN_RE)) {+ const written = parseMark(m[1]);+ if (written === null) continue;+ const held = pool.findIndex((p) => p.token === m[0]);+ const mark = held >= 0 ? pool.splice(held, 1)[0] : markAt(written, s);+ const at = held >= 0 ? markTime(mark, s) : written;+ const fresh: TimeMark = { ...mark, token: markToken(at), at };+ out.push(fresh);+ next += text.slice(last, m.index) + fresh.token;+ last = m.index + m[0].length;+ }+ return { text: next + text.slice(last), marks: out };+}++/** Split text into literal runs and time tokens, for drawing each moment as a+ * chip. The token text is kept exactly as written, so a caller that mirrors+ * the string character for character stays aligned. */+export function splitTimes(text: string): (string | { text: string; at: number })[] {+ const parts: (string | { text: string; at: number })[] = [];+ let last = 0;+ for (const m of text.matchAll(TIME_TOKEN_RE)) {+ const at = parseMark(m[1]);+ if (at === null) continue;+ if (m.index > last) parts.push(text.slice(last, m.index));+ parts.push({ text: m[0], at });+ last = m.index + m[0].length;+ }+ if (last < text.length) parts.push(text.slice(last));+ return parts;+}