Open source CapCut alternative.
TypeScript
"use client";
/**
* Track a target through a clip's footage and drive an item with it — the
* one path the chat's track_motion tool and the inspector's Track control
* both run.
*
* It picks the footage (a named clip, a video clip's own picture for a mask
* or a read, or the clip under the item), samples the stretch the two share on the timeline's clock, reads
* it through the landmarkers (tracking.ts), settles and places the path
* (trackKeys.ts), and lands the result on the item as one undo step.
*/
import { MASK_FEATHER_MAX, poseAt, retimeOf, type MaskPoint } from "@donkeycut/effects-kit";
import { CUT_TRACKING } from "./chatRuntime";
import { clipLen, useEditor } from "./store";
import { trackFootage } from "./tracking";
import {
followKeys,
pictureToFrame,
settleTrack,
trackedMask,
type FollowMode,
type TrackItem,
type TrackSample,
} from "./trackKeys";
import { trackTargetOf, type TrackTarget, type TrackUse } from "./trackTargets";
import { clipPoseAt, isEffectOverlay, type MediaAsset, type VideoClip } from "./types";
import { UserError } from "./userError";
/** The shortest stretch worth a track: a couple of frames. */
const MIN_TRACK_SECONDS = 0.05;
/** The most in-view stretches and path points a track reports. */
const TRACK_SPANS_MAX = 12;
const TRACK_PATH_POINTS = 24;
export interface TrackRequest {
/** The item driven: a video clip or an overlay element. */
id: string;
target: TrackTarget;
use: TrackUse;
/** The video clip whose footage is read; see `trackSource`. */
sourceId?: string;
/** Timeline seconds bounding the track. */
from?: number;
to?: number;
follow?: FollowMode;
feather?: number;
invert?: boolean;
/** Share of frames read so far, 0..1. */
onProgress?: (share: number) => void;
}
export async function trackMotion(req: TrackRequest) {
const s = useEditor.getState();
const o = s.overlays.find((x) => x.id === req.id);
const own = o ? undefined : s.clips.find((c) => c.id === req.id);
if (!o && !own) {
throw new UserError("track_item_missing", "No overlay element or video clip with that id.");
}
if (o && req.use === "mask" && isEffectOverlay(o)) {
throw new UserError("track_effect_mask", "Effect elements have no pixels to mask.");
}
// The item's clock as the read starts: keys count from here, so a move
// during the read cannot slide them off the footage.
const span = o ? { start: o.start, end: o.end } : { start: own!.start, end: own!.start + clipLen(own!) };
const source = trackSource(s.clips, s.assets, req, own, span);
const asset = s.assets.find((a) => a.id === source.assetId);
const picture = asset?.type === "video" && asset.width && asset.height ? { width: asset.width, height: asset.height } : null;
if (!asset || !picture) {
throw new UserError("track_source_no_picture", "The source clip has no video picture to track.");
}
// The stretch tracked: where the item and the source overlap, narrowed by
// from/to, sampled evenly on the timeline's clock.
const srcEnd = source.start + clipLen(source);
const a = Math.max(span.start, source.start, req.from ?? -Infinity);
const b = Math.min(span.end, srcEnd, req.to ?? Infinity);
if (b - a < MIN_TRACK_SECONDS) {
throw new UserError(
"track_no_overlap",
`The item and the source clip share no stretch to track (${round2(source.start)}–${round2(srcEnd)}s holds the footage).`,
{ from: round2(source.start), to: round2(srcEnd) },
);
}
const tune = CUT_TRACKING;
const fps = Math.min(tune.sampleFps, tune.maxFrames / (b - a));
const n = Math.max(2, Math.floor((b - a) * fps) + 1);
const last = b - 1 / (2 * tune.sampleFps);
const times = Array.from({ length: n }, (_, k) => a + ((last - a) * k) / (n - 1));
const rt = retimeOf(source);
const srcTimes = times.map((t) => rt.srcAt(t - source.start));
// Read, settle, and place the path in the project frame.
const raw = await trackFootage(asset.url, picture, req.target, srcTimes, (done, total) => req.onProgress?.(done / total));
const settled = settleTrack(times, raw, tune);
const place = pictureToFrame(source, picture, s.aspect);
const framed = settled.map((x) =>
x ? { outline: x.outline.map(place), center: place(x.center), axis: [place(x.axis[0]), place(x.axis[1])] as [MaskPoint, MaskPoint] } : null
);
const found = framed.filter((x) => x !== null).length;
const summary = {
target: req.target,
source_id: source.id,
from: round2(a),
to: round2(b),
frames: n,
found_share: round2(found / n),
in_view: inViewSpans(times, framed),
};
if (req.use === "read") return { ...summary, path: trackPath(times, framed) };
const label = trackTargetOf(req.target).label.toLowerCase();
if (found === 0) {
throw new UserError("track_target_not_found", `No ${label} found in the source between ${round2(a)}s and ${round2(b)}s.`, {
target: req.target,
from: round2(a),
to: round2(b),
});
}
// The item as it stands now: the read took a while, and the person may
// have moved it meanwhile.
const now = useEditor.getState();
const liveO = o && now.overlays.find((x) => x.id === o.id);
const liveClip = own && now.clips.find((c) => c.id === own.id);
if (!liveO && !liveClip) {
throw new UserError("track_item_deleted", "The item was deleted while it was being tracked.");
}
const item: TrackItem = liveO ? { kind: "overlay", overlay: liveO } : { kind: "clip", clip: liveClip! };
const local = times.map((t) => t - span.start);
// A keyed pen mask that traces the target.
if (req.use === "mask") {
const feather = Math.max(0, Math.min(MASK_FEATHER_MAX, req.feather ?? 0));
const invert = req.invert === true;
const mask = trackedMask(item, now.aspect, local, framed, { feather, invert }, tune.keyTolerance)!;
if (liveO) now.updateOverlay(liveO.id, { mask });
else now.updateClip(liveClip!.id, { mask });
return { ...summary, id: req.id, mask: { kind: mask.kind, keys: mask.kf!.length, feather, invert } };
}
// Pose keys that carry the item with the target.
const follow = req.follow ?? "move_scale";
const rest = liveO ? poseAt(liveO, a - span.start) : clipPoseAt(liveClip!, a - span.start);
const kf = followKeys(now.aspect, local, framed, rest, follow, tune.keyTolerance);
if (liveO) now.updateOverlay(liveO.id, { kf });
else now.updateClip(liveClip!.id, { kf });
return { ...summary, id: req.id, follow_mode: follow, keys: kf.length };
}
/**
* The footage a track reads: the clip `source_id` names; for a mask or a read
* on a video clip, the clip's own picture; else the video clip under the
* item. A follow always rides other footage — a clip carried along with its
* own subject would move that subject twice.
*/
function trackSource(
clips: VideoClip[],
assets: MediaAsset[],
req: TrackRequest,
own: VideoClip | undefined,
span: { start: number; end: number }
): VideoClip {
const isVideo = (c: VideoClip) => assets.find((x) => x.id === c.assetId)?.type === "video";
if (req.sourceId !== undefined) {
const named = clips.find((c) => c.id === req.sourceId);
if (!named) {
throw new UserError("track_source_missing", "No video clip with that source_id.");
}
if (req.use === "follow" && named.id === own?.id) {
throw new UserError(
"track_follow_self",
"A clip cannot follow its own footage — pass the source_id of the clip it should ride.",
);
}
return named;
}
if (own && req.use !== "follow" && isVideo(own)) return own;
const under = clipUnder(clips, span, req.id, isVideo);
if (!under) {
throw new UserError(
"track_source_none_under",
`No other video clip plays under that ${own ? "clip" : "element"} — pass source_id.`,
{ kind: own ? "clip" : "element" },
);
}
return under;
}
/** The video clip that covers most of `span` — the footage under an item.
* A tie goes to the clip nearest track 0, the shot itself. */
function clipUnder(
clips: VideoClip[],
span: { start: number; end: number },
itemId: string,
isVideo: (c: VideoClip) => boolean
): VideoClip | undefined {
const shared = (c: VideoClip) => Math.min(span.end, c.start + clipLen(c)) - Math.max(span.start, c.start);
return clips
.filter((c) => c.id !== itemId && !c.hidden && isVideo(c) && shared(c) > 0)
.sort((p, q) => shared(q) - shared(p) || Math.abs(p.track ?? 0) - Math.abs(q.track ?? 0))[0];
}
/** The stretches, in timeline seconds, where the target was found. */
function inViewSpans(times: number[], frame: (TrackSample | null)[]): { from: number; to: number }[] {
const spans: { from: number; to: number }[] = [];
frame.forEach((x, i) => {
if (!x) return;
const open = spans[spans.length - 1];
if (open && frame[i - 1]) open.to = round2(times[i]);
else spans.push({ from: round2(times[i]), to: round2(times[i]) });
});
return spans.slice(0, TRACK_SPANS_MAX);
}
/** Where the target goes, evenly sampled: its center and its outline's box,
* in frame fractions, at timeline seconds. */
function trackPath(times: number[], frame: (TrackSample | null)[]) {
const step = Math.max(1, Math.ceil(frame.length / TRACK_PATH_POINTS));
const out: { t: number; x: number; y: number; w: number; h: number }[] = [];
for (let i = 0; i < frame.length; i += step) {
const x = frame[i];
if (!x) continue;
const xs = x.outline.map((p) => p.x);
const ys = x.outline.map((p) => p.y);
out.push({
t: round2(times[i]),
x: round3(x.center.x),
y: round3(x.center.y),
w: round3(Math.max(...xs) - Math.min(...xs)),
h: round3(Math.max(...ys) - Math.min(...ys)),
});
}
return out;
}
const round2 = (n: number) => Math.round(n * 100) / 100;
const round3 = (n: number) => Math.round(n * 1000) / 1000;