Open source CapCut alternative.
TypeScript
"use client";
import { scanBeats, scanLevel, scanSilence, scanSpeech, type BeatScan, type LevelScan, type PcmChunk, type SpeechScan } from "./audioScan";
import { syncSound, type SyncAnswer } from "./soundSync";
import { LoudnessMeter, type LoudnessMeasure } from "./loudness";
import { apiFetch, apiJson, getBackend, type CutBackend } from "./backend";
import { HTTP_PAYLOAD_TOO_LARGE, type StorageFull } from "./backend/types";
import { registerBlobFile } from "./backend/browser/registry";
import { cloudBackend } from "./backend/cloud";
import { pollCloudJob } from "./cloudJob";
import { creatorNames } from "./defaultNames";
import { errorFromBody, replyError, UserError } from "./userError";
import { encodeWav } from "./cloudTranscribe";
import { allowance, canvasBytes, holdMemory } from "./memoryBudget";
import { normalizeLink } from "./link";
import { ConnectionLostError } from "./network";
import { downloadFromUrl } from "./download";
import { FONT_RE, LUT_RE } from "./libraryFileType";
import { markUploadStored, startUpload, uploadInFlight } from "./importQueue";
import { emitImportWall } from "./storageQuota";
import { convertProjectFile, dropProjectFile, mp4DisplayName } from "./mediaConvert";
import { awaitsCompat, ensureProxy } from "./mediaProxy";
import { noteUndecoded } from "./codecSupport";
import { compatCovers, type CompatCover } from "./proxyNames";
import {
audioChunks,
audioPeaks,
audioTrackOf,
decodeAudioSpan,
frameSink,
frameSinkIsCustom,
openMedia,
openMediaShared,
probeMediaColor,
probeMediaFile,
readMediaFileSize,
UnreadableMediaError,
videoTrackOf,
withMedia,
type IdleGate,
type MediaHandle,
type MediaProbe,
} from "./mediaRead";
import type { WrappedCanvas } from "mediabunny";
import {
clearPendingUpload,
dropLocalMedia,
localMediaFile,
localMediaUrl,
pendingUploadsFor,
renameLocalMedia,
stashCloudMedia,
stashUnclaimedMedia,
storedMediaUrl,
} from "./mediaSync";
import {
createRasterCanvas,
decodeRasterImage,
decodeRasterImageUrl,
rasterCanvasToBlob,
rasterCanvasToDataUrl,
type RasterSurface,
} from "./raster";
import { pickThumbTimes, type ThumbProbe } from "./filmstrip";
import { reportSwallowed } from "./report";
import { cloudMediaGone, cloudMediaStatus } from "./chunkCache";
import { landDerived, liveAsset } from "./derivedWrites";
import { baseFrameAt, baseFramesAt, baseFrameSink, colorReadAt } from "./baseFrame";
import { colorRecord } from "./colorProbe";
import { previewFile, sameColorRead } from "./sourceColor";
import { assetIdsInUse, useEditor } from "./store";
import type { AssetBeats, AssetType, AudioClip, MediaAsset, ProjectSummary, StoredAsset, VideoClip, WatchDetail, WatchKeepReason } from "./types";
import { contentRect, IMAGE_CLIP_SECONDS, mediaUrl } from "./types";
import { createDedupSelector, DEDUP_TUNING } from "./watch/dedupSelector";
import { diffCellPct, frameSig } from "./watch/signatures";
import { SIGNATURE_SIZE } from "./watch/types";
const uid = () => crypto.randomUUID().slice(0, 8);
/** True when a dropped OS file is something Cut can turn into a project. */
export function isMediaFile(file: File) {
return (
file.type.startsWith("video/") ||
file.type.startsWith("audio/") ||
file.type.startsWith("image/") ||
isVideoFile(file) ||
isAudioFile(file) ||
isImageFile(file)
);
}
/** Create a fresh project seeded from a single desktop file: upload the media,
* lay it on the timeline, and persist — no editor round-trip. Returns the new
* project's id, or null if the file isn't video/audio. */
export async function createProjectFromFile(
file: File,
folderId: string | null
): Promise<string | null> {
if (!isMediaFile(file)) return null;
const name = file.name.replace(/\.[^./]+$/, "") || creatorNames().untitled;
const res = await apiFetch("/api/cut/projects", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name, folderId }),
});
const project = await apiJson<ProjectSummary>(res);
if (!res.ok || !project.id) throw errorFromBody(project, new UserError("project_creation_failed", "Could not create the project."));
const asset = await importFileToProject(project.id, file);
// Media the engine rejects leaves an empty project rather than a dangling id.
if (!asset) return project.id;
const stored: StoredAsset = {
id: asset.id,
fileName: asset.fileName,
name: asset.name,
type: asset.type,
duration: asset.duration,
...(asset.width !== undefined ? { width: asset.width } : {}),
...(asset.height !== undefined ? { height: asset.height } : {}),
};
const doc: Partial<{ assets: StoredAsset[]; clips: VideoClip[]; audioClips: AudioClip[] }> = {
assets: [stored],
};
if (asset.type === "video" || asset.type === "image") {
const out = asset.type === "image" ? IMAGE_CLIP_SECONDS : asset.duration;
doc.clips = [{ id: uid(), assetId: asset.id, track: 0, start: 0, in: 0, out, muted: false }];
} else {
doc.audioClips = [
{ id: uid(), assetId: asset.id, start: 0, in: 0, out: asset.duration, volume: 1 },
];
}
await apiFetch(`/api/cut/projects/${project.id}`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(doc),
});
return project.id;
}
/** Save a project media file to the user's Downloads folder — the everywhere
* counterpart to revealing it in Finder.
*
* What lands is the source file itself: import stores the bytes it was handed
* and every derivative (proxy, ladder, filmstrip, export) is a separate file,
* so the media route is already the original. An asset whose bytes are still
* uploading has no server copy to serve, so it has nothing to download yet.
* `backend` pins the shelf the asset lives on when the ambient one may have
* moved on. */
export function downloadMedia(projectId: string, asset: MediaAsset, backend?: CutBackend) {
if (asset.upload) return;
downloadFromUrl(mediaUrl(projectId, asset.fileName, backend), asset.fileName);
}
/** Reveal a project media file in Finder (local engine only). */
export async function revealMedia(projectId: string, fileName: string) {
await apiFetch(
`/api/cut/projects/${projectId}/media/${encodeURIComponent(fileName)}/reveal`,
{ method: "POST" }
);
}
const VIDEO_EXTENSIONS = ["mp4", "mov", "m4v", "webm", "mkv"];
const AUDIO_EXTENSIONS = ["mp3", "m4a", "aac", "wav", "ogg", "flac"];
const IMAGE_EXTENSIONS = ["png", "jpg", "jpeg", "webp", "gif", "avif", "bmp"];
const hasExtension = (file: File, extensions: string[]) =>
new RegExp(`\\.(${extensions.join("|")})$`, "i").test(file.name);
export function isVideoFile(file: File) {
return file.type.startsWith("video/") || hasExtension(file, VIDEO_EXTENSIONS);
}
export function isAudioFile(file: File) {
return file.type.startsWith("audio/") || hasExtension(file, AUDIO_EXTENSIONS);
}
export function isImageFile(file: File) {
return file.type.startsWith("image/") || hasExtension(file, IMAGE_EXTENSIONS);
}
/** What a media file input's `accept` lists: every kind `isMediaFile` takes,
* by MIME family and by extension, so the OS picker offers the same files a
* drop does. */
export const MEDIA_ACCEPT = [
"video/*",
"audio/*",
"image/*",
...[...VIDEO_EXTENSIONS, ...AUDIO_EXTENSIONS, ...IMAGE_EXTENSIONS].map((ext) => `.${ext}`),
].join(",");
export function isTextFile(file: File) {
return file.type.startsWith("text/") || /\.(txt|md|markdown|srt|vtt|csv|json)$/i.test(file.name);
}
/** A font file the Library takes. Browsers often report an empty type for
* .otf, so the extension has the final say. */
export function isFontFile(file: File) {
return file.type.startsWith("font/") || FONT_RE.test(file.name);
}
/** A colour LUT file the Library takes. Browsers report no type for .cube or
* .3dl, so the extension is the whole test. */
export function isLutFile(file: File) {
return LUT_RE.test(file.name);
}
/** What a file picker offers for LUTs. */
export const LUT_ACCEPT = ".cube,.3dl";
/** The kind mark a card wears for a file: its extension, uppercased. Empty for
* a name that carries none. */
export const fileKind = (fileName: string) =>
(/\.([a-z0-9]+)$/i.exec(fileName)?.[1] ?? "").toUpperCase();
/** Fonts are small; a file past this is a mistake worth catching before the
* bytes move. The storage quota is the real ceiling. */
export const MAX_FONT_BYTES = 10 * 1024 ** 2;
/** Claim a name and mint a direct-to-R2 PUT for one file. The name is deduped
* and reserved server-side before the URL comes back, so callers can build the
* asset's final identity before a single byte moves. */
export async function presignUpload(
presignPath: string,
file: Blob,
name: string,
backend: CutBackend = getBackend(),
extra: Record<string, unknown> = {},
storageFull: StorageFull = "wall"
): Promise<{ key: string; url: string; fileName: string }> {
const mime = file.type || "application/octet-stream";
const res = await backend.fetch(presignPath, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ fileName: name, mime, bytes: file.size, ...extra }),
storageFull,
});
const body = await apiJson<{ key?: string; url?: string; fileName?: string }>(res);
if (!res.ok || !body.key || !body.url) {
const err = replyError(res.status, body, new UserError("upload_failed", "Upload failed.")) as Error & { status?: number };
err.status = res.status;
throw err;
}
return { key: body.key, url: body.url, fileName: body.fileName ?? name };
}
/** Presign a direct-to-R2 upload, PUT the bytes, and return the object key
* for the follow-up complete call. Shared by project media, the library, and
* export overlays; cloud backend only. */
export async function presignedUpload(
presignPath: string,
file: Blob,
name: string,
backend: CutBackend = getBackend()
): Promise<string> {
const signed = await presignUpload(presignPath, file, name, backend);
await putSigned(signed.url, file, file.type || "application/octet-stream");
return signed.key;
}
/** PUT a blob to a presigned R2 URL. XHR because it is the only way to watch
* the bytes leave, which is what an upload running behind the editor has to
* report. A headless runtime has no XHR and nobody watching; fetch covers it. */
export function putSigned(
url: string,
file: Blob,
mime?: string,
opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }
): Promise<void> {
if (typeof XMLHttpRequest === "undefined") {
return fetch(url, {
method: "PUT",
headers: { "Content-Type": mime ?? (file.type || "application/octet-stream") },
body: file,
signal: opts?.signal,
}).then((res) => {
if (!res.ok) throw new UserError("upload_failed", "Upload failed.");
opts?.onProgress?.(1);
});
}
return new Promise((resolve, reject) => {
if (opts?.signal?.aborted) return reject(new DOMException("Upload cancelled.", "AbortError"));
const xhr = new XMLHttpRequest();
xhr.open("PUT", url);
xhr.setRequestHeader("Content-Type", mime ?? (file.type || "application/octet-stream"));
if (opts?.onProgress) {
xhr.upload.onprogress = (e) => {
if (e.lengthComputable && e.total > 0) opts.onProgress!(e.loaded / e.total);
};
}
xhr.onload = () =>
xhr.status >= 200 && xhr.status < 300 ? resolve() : reject(new UserError("upload_failed", "Upload failed."));
xhr.onerror = () => reject(new ConnectionLostError("upload_failed", "Upload failed."));
xhr.onabort = () => reject(new DOMException("Upload cancelled.", "AbortError"));
opts?.signal?.addEventListener("abort", () => xhr.abort(), { once: true });
xhr.send(file);
});
}
/** A multipart POST with upload progress and a cancel, answering the stored
* name the engine replies with. */
function postFormWatched(
url: string,
form: FormData,
opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }
): Promise<string> {
return new Promise((resolve, reject) => {
if (opts?.signal?.aborted) return reject(new DOMException("Upload cancelled.", "AbortError"));
const xhr = new XMLHttpRequest();
xhr.open("POST", url);
if (opts?.onProgress) {
xhr.upload.onprogress = (e) => {
if (e.lengthComputable && e.total > 0) opts.onProgress!(e.loaded / e.total);
};
}
xhr.onload = () => {
let body: { fileName?: string; error?: string; code?: string } = {};
try {
body = JSON.parse(xhr.responseText) as typeof body;
} catch {
// A reply that never reached a handler is plain text.
}
if (xhr.status >= 200 && xhr.status < 300 && body.fileName) resolve(body.fileName);
else reject(errorFromBody(body, new UserError("upload_failed", "Upload failed.")));
};
xhr.onerror = () => reject(new ConnectionLostError("upload_failed", "Upload failed."));
xhr.onabort = () => reject(new DOMException("Upload cancelled.", "AbortError"));
opts?.signal?.addEventListener("abort", () => xhr.abort(), { once: true });
xhr.send(form);
});
}
/** Upload raw media bytes into a project. Local: the engine's multipart POST,
* byte-identical to the pre-seam request. Cloud: presign -> direct R2 PUT ->
* complete. Returns the stored (deduped) file name. */
export function uploadProjectMedia(projectId: string, file: Blob, name: string): Promise<string> {
return uploadProjectMediaTo(getBackend(), projectId, file, name);
}
/** `uploadProjectMedia` against an explicit backend — cross-residency copies
* upload to a backend that is not the globally bound one. */
export async function uploadProjectMediaTo(
backend: CutBackend,
projectId: string,
file: Blob,
name: string,
opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal; storageFull?: StorageFull }
): Promise<string> {
if (backend.kind !== "cloud") {
const form = new FormData();
form.append("file", file, name);
const path = `/api/cut/projects/${projectId}/media`;
// The engine's copy is watched the way a cloud PUT is: the bytes leaving
// are the progress, and a cancel stops them. The in-page browser router
// answers fetch only, and writes fast enough to need neither.
if (backend.kind === "local" && typeof XMLHttpRequest !== "undefined") {
return postFormWatched(backend.url(path), form, opts);
}
const res = await backend.fetch(path, { method: "POST", body: form, signal: opts?.signal });
const body = await apiJson<{ fileName?: string }>(res);
if (!res.ok || !body.fileName) throw errorFromBody(body, new UserError("upload_failed", "Upload failed."));
return body.fileName;
}
const signed = await presignUpload(
`/api/cut/projects/${projectId}/media/presign`,
file,
name,
backend,
{},
opts?.storageFull
);
await putSigned(signed.url, file, file.type || "application/octet-stream", opts);
const key = signed.key;
const res = await backend.fetch(`/api/cut/projects/${projectId}/media/complete`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ key }),
});
const body = await apiJson<{ fileName?: string }>(res);
if (!res.ok || !body.fileName) throw errorFromBody(body, new UserError("upload_failed", "Upload failed."));
return body.fileName;
}
// The cloud /image mirror only takes inline multipart bodies below ~3.5MB;
// larger images ride the presign path.
const IMAGE_INLINE_MAX = 3 * 1024 * 1024;
/** Store an image blob as a first-class project image asset. Local (and small
* cloud payloads): the engine's /image multipart route, byte-identical to the
* pre-seam request. Large cloud payloads: presign -> R2 PUT -> complete, with
* the dimensions probed here instead of by the server. */
export async function uploadProjectImage(
projectId: string,
file: Blob,
fileName: string,
opts?: { name?: string; origin?: "generated" | "sticker"; failMessage?: UserError | string; backend?: CutBackend }
): Promise<MediaAsset> {
// Callers whose work outlives navigation (finishing AI generations) pin the
// backend they started on; everyone else rides the active one.
const backend = opts?.backend ?? getBackend();
const failMessage = opts?.failMessage ?? new UserError("image_add_failed", "Could not add the image.");
if (backend.kind !== "cloud" || file.size < IMAGE_INLINE_MAX) {
const form = new FormData();
form.append("file", file, fileName);
if (opts?.name !== undefined) form.append("name", opts.name);
if (opts?.origin) form.append("origin", opts.origin);
const res = await backend.fetch(`/api/cut/projects/${projectId}/image`, {
method: "POST",
body: form,
});
const body = await apiJson<MediaAsset>(res);
if (!res.ok || !body.fileName) {
throw replyError(res.status, body, failMessage);
}
return { ...body, url: await storedMediaUrl(projectId, body.fileName, backend) };
}
const stored = await uploadProjectMediaTo(backend, projectId, file, fileName);
const url = await storedMediaUrl(projectId, stored, backend);
const dims = await loadImageMeta(url);
return {
id: uid(),
type: "image",
name: opts?.name?.trim() || fileName,
fileName: stored,
duration: 0,
width: dims.width,
height: dims.height,
...(opts?.origin ? { origin: opts.origin } : {}),
url,
};
}
/** The kind of asset a dropped file becomes, or null if Cut can't use it.
* MIME wins over extension: recordings are .webm for both video and audio.
* Linked kinds (fonts, LUTs) are not project assets: they land on the Library
* shelf through `linkedLibrary`, so they answer null here. */
export function assetTypeOf(file: File): AssetType | null {
if (file.type.startsWith("video/")) return "video";
if (file.type.startsWith("audio/")) return "audio";
if (file.type.startsWith("image/")) return "image";
if (isVideoFile(file)) return "video";
if (isAudioFile(file)) return "audio";
if (isImageFile(file)) return "image";
return null;
}
type ProbedMeta = {
type: AssetType;
duration: number;
sizeBytes?: number;
width?: number;
height?: number;
/** What a video's code values mean, from its header. */
color?: StoredAsset["color"];
};
/** The asset's `color` field from a header probe. */
const assetColor = (probe: MediaProbe["color"]): StoredAsset["color"] | undefined =>
probe ? colorRecord(probe) : undefined;
/**
* Write what a header probe learned about an asset.
*
* Every asset write re-renders what reads the assets, and on a slow machine
* that is a long beat of the main thread. A probe lands a few reads after the
* drop, which is often inside the play that followed it, so a write that
* leaves the picture as it is — the size the shelf already said, the color
* the preview already draws — waits for the play to stop, and one that
* changes what is drawn lands now.
*
* The probe writes the header's facts into `color`; the person's profile
* override lives beside it in `colorProfile`, so a header write never
* touches it. The decision reads the live asset, so an override set while
* the probe or the play ran is weighed with it.
*/
export async function landProbedFacts(id: string, patch: Pick<Partial<MediaAsset>, "width" | "height" | "color">): Promise<void> {
const live = () => useEditor.getState().assets.find((a) => a.id === id);
let asset = live();
if (!asset) return;
if (!probeChangesPicture(asset, patch)) {
await idlePlayback();
asset = live();
if (!asset) return;
}
useEditor.getState().updateAsset(id, patch);
}
/** Whether a probe's facts change what `asset` draws: its size, or the read
* its color record makes under the asset's profile override. */
export function probeChangesPicture(
asset: MediaAsset,
patch: Pick<Partial<MediaAsset>, "width" | "height" | "color">
): boolean {
const resized =
(patch.width !== undefined && patch.width !== asset.width) ||
(patch.height !== undefined && patch.height !== asset.height);
return (
resized ||
(patch.color !== undefined &&
!sameColorRead(asset, { color: patch.color, colorProfile: asset.colorProfile }, previewFile(asset)))
);
}
/** Read a media source's kind/duration/dimensions from its container. The
* probe can correct the guessed kind: a "video" container with no video track
* is really audio, and an "audio" one that turns out to carry picture is video.
*
* The duration is exact, which matters — it is a placed clip's length, and the
* metadata a player reports overestimates MP3s badly enough to leave a clip
* running past its own audio. */
async function probeMedia(type: AssetType, src: string | Blob): Promise<ProbedMeta> {
if (type === "image") {
// Images have no intrinsic duration; the timeline clip carries its length.
// Stills are the one kind read through the browser's image decoder rather
// than the container reader, which does not do them.
if (typeof Image === "undefined") {
// Headless runtime: decode through the raster seam (skia in the worker).
const blob = typeof src === "string" ? await (await fetch(src)).blob() : src;
const img = await decodeRasterImage(blob);
if (!img) throw new UnreadableMediaError();
return { type, duration: 0, sizeBytes: blob.size, width: img.width, height: img.height };
}
const url = typeof src === "string" ? src : URL.createObjectURL(src);
try {
const dims = await loadImageMeta(url);
return { type, duration: 0, sizeBytes: await readMediaFileSize(src), width: dims.width, height: dims.height };
} finally {
if (typeof src !== "string") URL.revokeObjectURL(url);
}
}
const meta = await probeMediaFile(src);
// The waveform is a decode of the whole file, and it comes after the asset
// has landed: enrichAsset reads the peaks behind the editor, from the bytes
// the import kept, so a song is on the timeline the moment its container
// has been read.
if (!meta.hasVideo) return { type: "audio", duration: meta.duration, sizeBytes: meta.sizeBytes };
return {
type: "video",
duration: meta.duration,
sizeBytes: meta.sizeBytes,
width: meta.width,
height: meta.height,
color: assetColor(meta.color),
};
}
/** Probe a media file's kind/duration/dimensions from the bytes in hand — for
* backends that can't probe server-side (cloud library complete). */
export async function probeFileMeta(file: File): Promise<{
type: AssetType;
duration: number;
width?: number;
height?: number;
}> {
const meta = await probeMedia(assetTypeOf(file) ?? "video", file);
return { type: meta.type, duration: meta.duration, width: meta.width, height: meta.height };
}
/** An import that is on screen before its bytes have left the browser. */
export type PendingImport = {
/** Ready to add to the project: plays from `localUrl`, carries its final
* `fileName`, and is marked `upload` until the bytes land. */
asset: MediaAsset;
/** The URL the asset plays from until the stored file takes over — a local
* object URL for a dropped file, the source URL for remote media. */
localUrl: string;
/** Put the bytes where a reload finds them — the browser store under a
* claimed name, or the project's own storage — as soon as the import starts.
* The queue runs it ahead of its transfer window, so a long drop is held
* whole while its uploads wait their turn. Safe to call again. */
hold?: ImportStep;
/** Send the bytes and mark the object complete; resolves to the stored
* (deduped) file name the asset swaps to. */
send: (opts?: ImportOpts) => Promise<string>;
};
type ImportOpts = { onProgress?: (fraction: number) => void; signal?: AbortSignal };
type ImportStep = (opts?: ImportOpts) => Promise<void>;
/** Prepare a dropped file so it is on the timeline at the release: probed
* from its own header, playing from the bytes in hand, named after the file.
* Everything else — the claim on a stored name, the copy into the browser
* store or the engine, the upload — runs behind the editor from `send`, and
* the asset joins the saved document once its bytes are held somewhere a
* reload can find them.
*
* Returns null for footage this browser cannot decode (a phone's HEVC, a
* camera's ProRes): that import goes the plain way, where the project
* converts the file and the asset is built from what comes back. */
export async function prepareImport(
projectId: string,
file: File,
backend: CutBackend = getBackend()
): Promise<PendingImport | null> {
const type = assetTypeOf(file);
if (!type) return null;
let meta: Awaited<ReturnType<typeof probeMedia>>;
try {
// The header is the only thing the clip waits on.
meta = await probeMedia(type, file);
} catch (e) {
if (e instanceof UnreadableMediaError && e.undecodable) return null;
throw e;
}
const id = uid();
keepImportedBytes(id, file);
// The bytes in hand are registered like a store file: decoders resolve the
// File behind the URL and read it as a blob, since ranged fetches of a blob
// URL are unreliable, and the URL lives for the session the way every
// registered one does — the filmstrip sweep reads it long after the copy
// has landed.
const localUrl = registerBlobFile(`pending/${id}`, file);
const asset: MediaAsset = {
id,
// A name of its own until the store claims one: nothing may match this
// asset by file name — a pin's resume, a delete's shared-file guard —
// before the bytes are actually stored under a name.
fileName: `pending:${id}:${file.name}`,
name: file.name,
type: meta.type,
duration: meta.duration,
sizeBytes: file.size,
...(meta.width !== undefined ? { width: meta.width } : {}),
...(meta.height !== undefined ? { height: meta.height } : {}),
...(meta.color !== undefined ? { color: meta.color } : {}),
url: localUrl,
upload: { progress: 0 },
};
const steps =
backend.kind === "cloud"
? sendCloudImport(backend, projectId, file, id)
: sendDirectImport(backend, projectId, file);
return { asset, localUrl, ...steps };
}
/** A browser or engine project stores a dropped file by copying it in, on
* this machine: the copy is the hold, and the send hands back its name. The
* copy is the whole transfer, so the copies of a drop take turns: each starts
* as soon as the one before it lands, ahead of the queue's window, so a
* reload finds the drop stored in order and a long drop never swamps the
* engine socket or the browser store. A failed or cancelled copy is
* forgotten, so a retry copies again. */
function sendDirectImport(
backend: CutBackend,
projectId: string,
file: File
): Pick<PendingImport, "hold" | "send"> {
let copy: Promise<string> | null = null;
const send: PendingImport["send"] = (opts) => {
copy ??= copyInTurn(() => uploadProjectMediaTo(backend, projectId, file, file.name, opts)).catch(
(err: unknown) => {
copy = null;
throw err;
}
);
return copy;
};
return { hold: async (opts) => void (await send(opts)), send };
}
/** Copies onto this machine run one at a time, in the order they start. */
let copyChain: Promise<unknown> = Promise.resolve();
function copyInTurn<T>(work: () => Promise<T>): Promise<T> {
const next = copyChain.then(work, work);
copyChain = next.catch(() => {});
return next;
}
/** Claims run one at a time, each against the names already taken, so two
* files of one name in a drop never race for the same stored name. Only the
* claim waits its turn; the copies run side by side. */
let claimChain: Promise<unknown> = Promise.resolve();
function claimInTurn<T>(work: () => Promise<T>): Promise<T> {
const next = claimChain.then(work, work);
claimChain = next.catch(() => {});
return next;
}
type CloudClaim = {
/** The presigned PUT; null when the account was full and the bytes wait in
* the browser store for the ledger drain to claim a cloud name. */
signed: { key: string; url: string; fileName: string } | null;
fileName: string;
/** Whether the browser store holds the bytes under `fileName`. */
stored: boolean;
};
/** The drain for a dropped file on a cloud project. The hold claims the
* stored name and lands the bytes in the browser store under it — the asset
* joins the saved document there, and a reload resumes the upload from the
* ledger; the send PUTs them and completes. A browser that cannot hold the
* store keeps the import tab-scoped and uploads it directly; a full account
* (413) stashes the bytes under a local name, and the ledger drain claims a
* cloud name for them once there is room. The claim is made once: a retry
* drains what it holds. */
function sendCloudImport(
backend: CutBackend,
projectId: string,
file: File,
assetId: string
): Pick<PendingImport, "hold" | "send"> {
let claim: CloudClaim | null = null;
const live = () => {
const s = useEditor.getState();
return s.projectId === projectId && s.assets.some((a) => a.id === assetId);
};
const cancelled = () => new DOMException("Cancelled.", "AbortError");
const hold: ImportStep = async (opts) => {
if (claim) return;
// The store write rides the claim's turn: the next claim's steer sees this
// pin, and one write at a time keeps each room check true to the disk.
const made = await claimInTurn(async (): Promise<CloudClaim> => {
if (opts?.signal?.aborted || !live()) throw cancelled();
const avoid = (await pendingUploadsFor(projectId)).map((p) => p.fileName);
let signed: CloudClaim["signed"] = null;
try {
signed = await presignUpload(
`/api/cut/projects/${projectId}/media/presign`,
file,
file.name,
backend,
avoid.length > 0 ? { avoid } : {},
"caller"
);
} catch (err) {
if ((err as { status?: number }).status !== HTTP_PAYLOAD_TOO_LARGE) throw err;
emitImportWall({ source: "quota-413" });
}
if (signed) {
const storedUrl = await stashCloudMedia(projectId, file, signed.fileName);
return { signed, fileName: signed.fileName, stored: storedUrl !== null };
}
const stashed = await stashUnclaimedMedia(projectId, file);
if (!stashed) throw replyError(413, {}, new UserError("upload_failed", "Upload failed."));
return { signed: null, fileName: stashed.fileName, stored: true };
});
// Deleted, or left behind, while the claim ran: the copy comes back out
// and nothing is completed, so nothing counts against the account.
if (opts?.signal?.aborted || !live()) {
if (made.stored) await dropLocalMedia(projectId, made.fileName);
throw cancelled();
}
claim = made;
if (claim.stored) markUploadStored(assetId, claim.fileName);
};
const send: PendingImport["send"] = async (opts) => {
await hold(opts);
if (!claim) throw cancelled();
// Stored bytes drain from the store, own name excluded from the steer;
// that path also re-signs, so a retry never reuses a signature that aged.
if (claim.stored || !claim.signed) {
return sendStoredUpload(backend, projectId, claim.fileName)(opts);
}
await putSigned(claim.signed.url, file, file.type || "application/octet-stream", opts);
const res = await backend.fetch(`/api/cut/projects/${projectId}/media/complete`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ key: claim.signed.key }),
});
const body = await apiJson<{ fileName?: string }>(res);
if (!res.ok || !body.fileName) throw errorFromBody(body, new UserError("upload_failed", "Upload failed."));
return body.fileName;
};
return { hold, send };
}
/** The drain for a store-backed upload, built from the stored bytes alone so
* a reload can resume it: reclaim the cloud name, PUT the file, complete, and
* unpin. The cloud's answer is the final name; when it drifts from the local
* one (a 413-era stash claiming its first cloud name), the local copy is
* renamed to match and the new name flows back to the asset. */
export function sendStoredUpload(
backend: CutBackend,
projectId: string,
fileName: string
): PendingImport["send"] {
return async (opts) => {
const file = await localMediaFile(projectId, fileName);
if (!file) throw new UserError("local_copy_gone", "The local copy of this file is gone.");
const mime = file.type || "application/octet-stream";
// Steer the cloud's dedupe around the other pinned names, never this one:
// this drain is the pin's own claim.
const avoid = (await pendingUploadsFor(projectId))
.map((p) => p.fileName)
.filter((n) => n !== fileName);
const res = await backend.fetch(`/api/cut/projects/${projectId}/media/presign`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
fileName,
mime,
bytes: file.size,
resume: true,
...(avoid.length > 0 ? { avoid } : {}),
}),
storageFull: "caller",
});
// A drain the account has no room for: the drop it came from raises the
// wall once, and the asset carries the refusal for its retry.
if (res.status === HTTP_PAYLOAD_TOO_LARGE) emitImportWall({ source: "quota-413" });
const body = await apiJson<{ key?: string; url?: string; fileName?: string; done?: boolean }>(
res
);
if (!res.ok || !body.fileName || (!body.done && !(body.url && body.key))) {
throw replyError(res.status, body, new UserError("upload_failed", "Upload failed."));
}
let finalName = body.fileName;
if (!body.done) {
await putSigned(body.url!, file, mime, opts);
// A claim that resolved to a different name (a 413-era stash drifting)
// has to flow back into the open asset. If the user left the project,
// pause instead of completing: the rename would move the local copy out
// from under the saved document's reference. The claim stays pending
// and the next open of the project resumes it. A same-name claim can't
// drift, so it completes regardless.
if (body.fileName !== fileName && useEditor.getState().projectId !== projectId) {
throw Object.assign(new Error("Paused: the project closed."), { paused: true });
}
const cres = await backend.fetch(`/api/cut/projects/${projectId}/media/complete`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ key: body.key }),
});
const cbody = await apiJson<{ fileName?: string }>(cres);
if (!cres.ok || !cbody.fileName) throw errorFromBody(cbody, new UserError("upload_failed", "Upload failed."));
finalName = cbody.fileName;
}
await renameLocalMedia(projectId, fileName, finalName);
await clearPendingUpload(projectId, finalName);
return finalName;
};
}
/** Re-arm a cloud project's unsynced imports after a reload: every ledger pin
* whose asset is still in the document gets its blob URL back, is re-marked
* uploading, and re-enters the queue; a pin whose asset left the document
* releases its bytes; a pin whose bytes are gone releases itself. */
export async function resumePendingUploads(
projectId: string,
backend: CutBackend = getBackend()
): Promise<void> {
if (backend.kind !== "cloud") return;
for (const p of await pendingUploadsFor(projectId)) {
const s = useEditor.getState();
if (s.projectId !== projectId) return;
const asset = s.assets.find((a) => a.fileName === p.fileName);
if (!asset) {
void dropLocalMedia(projectId, p.fileName);
continue;
}
if (uploadInFlight(asset.id)) continue;
const url = await localMediaUrl(projectId, p.fileName);
if (!url) {
void clearPendingUpload(projectId, p.fileName);
continue;
}
const upload = { progress: 0, stored: true };
useEditor.getState().updateAsset(asset.id, { url, upload });
startUpload(projectId, {
asset: { ...asset, url, upload },
localUrl: url,
send: sendStoredUpload(backend, projectId, p.fileName),
});
// The open's sweep ran first and read the cloud, where these bytes have
// not landed yet. The strip and the waveform read the local copy instead.
void enrichAsset({ ...asset, url, upload });
}
}
/** Upload a raw file into the project folder, probe it, and return the asset.
* Thumbnails/waveform are filled in asynchronously via `enrichAsset`. */
export async function importFileToProject(
projectId: string,
file: File,
backend: CutBackend = getBackend(),
storageFull: StorageFull = "wall"
): Promise<MediaAsset | null> {
const type = assetTypeOf(file);
if (!type) return null;
// A file from a drop answers a storage refusal with the drop's one wall.
let fileName = await uploadProjectMediaTo(backend, projectId, file, file.name, { storageFull }).catch(
(err: unknown) => {
if (storageFull === "caller" && (err as { status?: number }).status === HTTP_PAYLOAD_TOO_LARGE) {
emitImportWall({ source: "quota-413" });
}
throw err;
}
);
const originalFileName = fileName;
let name = file.name;
// Probe the bytes in hand: the same file the upload just wrote, with no
// second network read — which also lets a headless process import media.
const meta = await probeMedia(type, file).catch(async (e: unknown) => {
// Footage this browser has no decoder for — a phone's HEVC, a camera's
// ProRes. The file is already in the project, so convert it there and
// import what comes back; the raw upload goes with the failure otherwise.
if (!(e instanceof UnreadableMediaError && e.undecodable)) throw e;
const made = await convertProjectFile(
projectId,
{ fileName, url: mediaUrl(projectId, fileName, backend), name: file.name },
{ backend }
).catch((convertError: unknown) => {
void dropProjectFile(projectId, fileName, backend).catch(() => {});
throw convertError;
});
if (made.fileName !== fileName) void dropProjectFile(projectId, fileName, backend).catch(() => {});
fileName = made.fileName;
// The asset answers to what it now is: a card reading .MOV over an MP4
// file is the card lying about the media.
name = mp4DisplayName(name);
return probeMedia(type, mediaUrl(projectId, fileName, backend));
});
const url = mediaUrl(projectId, fileName, backend);
const id = uid();
if (fileName === originalFileName) keepImportedBytes(id, file);
return {
id,
fileName,
name,
type: meta.type,
duration: meta.duration,
sizeBytes: meta.sizeBytes,
...(meta.width !== undefined ? { width: meta.width } : {}),
...(meta.height !== undefined ? { height: meta.height } : {}),
...(meta.color !== undefined ? { color: meta.color } : {}),
url,
};
}
/** Natural pixel size of an image URL, for framing on the timeline. */
async function loadImageMeta(url: string): Promise<{ width: number; height: number }> {
const img = await decodeRasterImageUrl(url);
return img ? { width: img.width, height: img.height } : { width: 0, height: 0 };
}
/** What `importRemote` needs to register a fetchable source as an asset the
* user can edit with right away. Duration and dimensions come from the
* caller's catalog entry, so nothing has to be read before the asset exists. */
type RemoteImportInit = {
url: string;
name: string;
/** Name to store the bytes under; defaults to the URL's basename. */
fileName?: string;
type: AssetType;
duration: number;
width?: number;
height?: number;
origin?: StoredAsset["origin"];
/** The copy runs server-side, so the save indicator reports a copy, not an
* upload. */
server?: boolean;
};
/** Copy attempts ride out transient blips before the queue marks the asset
* failed; an abort (asset deleted, project left) stops immediately. */
async function withRetries(
copy: (opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }) => Promise<string>,
opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }
): Promise<string> {
for (let attempt = 0; ; attempt++) {
try {
return await copy(opts);
} catch (err) {
if (opts?.signal?.aborted || attempt >= 2) throw err;
await new Promise((r) => setTimeout(r, 1500 * (attempt + 1)));
}
}
}
/** Register media that lives at a fetchable URL — a stock tile, a library
* file — as a project asset that is usable the moment this returns. The asset
* plays from the source URL while `copy` moves the bytes into project storage
* behind the editor (the import queue); when they land it swaps to the stored
* file and joins the saved document, exactly like a dropped OS file. `copy`
* defaults to download-and-upload; callers with a server-side copy (the
* library's same-shelf route) pass their own and skip the round trip. */
export function importRemote(
projectId: string,
init: RemoteImportInit,
copy?: (opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }) => Promise<string>
): MediaAsset {
const id = uid();
// The name the bytes will be stored under is the copy's to decide — the
// server dedupes it against what the project already holds — so until they
// land the asset answers to a name of its own. A name borrowed from the
// source would collide with a file already in the project and speak for it:
// the delete guard and the filmstrip cache both key on this field.
const storeAs =
init.fileName || init.url.split("/").pop()?.split("?")[0] || `${init.type}-${id}`;
const asset: MediaAsset = {
id,
fileName: `pending-${id}`,
name: init.name,
type: init.type,
duration: init.duration,
...(init.width !== undefined ? { width: init.width } : {}),
...(init.height !== undefined ? { height: init.height } : {}),
...(init.origin ? { origin: init.origin } : {}),
url: init.url,
upload: { progress: 0, ...(init.server ? { server: true } : {}) },
};
useEditor.getState().addAsset(asset);
void enrichAsset(asset);
// Catalog dimensions are nominal (an aspect, not the file's pixels): read
// the real ones behind the placement so the doc stores the truth.
if (init.type === "video") {
void probeMediaFile(init.url)
.then((m) => {
if (!(init.duration > 0)) {
useEditor.getState().repairAssetDuration(asset.id, {
duration: m.duration,
width: m.width,
height: m.height,
});
}
if (m.hasVideo) void landProbedFacts(asset.id, { width: m.width, height: m.height, color: assetColor(m.color) });
})
.catch(() => {});
}
const download = async (opts?: {
onProgress?: (fraction: number) => void;
signal?: AbortSignal;
}) => {
const dl = await fetch(init.url, { signal: opts?.signal });
if (!dl.ok) throw new UserError("media_fetch_failed", "Could not read the media.");
const blob = await dl.blob();
return uploadProjectMediaTo(getBackend(), projectId, blob, storeAs, opts);
};
startUpload(projectId, {
asset,
localUrl: init.url,
send: (opts) => withRetries(copy ?? download, opts),
});
return asset;
}
/**
* Register media that has already landed in the project's storage as an
* asset, at the address it will always play from.
*
* The other way in, `importRemote`, plays from a source address while the
* bytes copy behind the editor and moves every reader once they land. A copy
* that was over before the asset existed has nothing to move: the first read
* opens the stored file, and no reader ever holds the file at two addresses.
*/
export async function registerLandedAsset(
projectId: string,
init: {
fileName: string;
name: string;
type: AssetType;
duration: number;
width?: number;
height?: number;
}
): Promise<MediaAsset> {
const asset: MediaAsset = {
id: uid(),
fileName: init.fileName,
name: init.name,
type: init.type,
duration: init.duration,
...(init.width !== undefined ? { width: init.width } : {}),
...(init.height !== undefined ? { height: init.height } : {}),
url: await storedMediaUrl(projectId, init.fileName),
};
useEditor.getState().addAsset(asset);
void enrichAsset(asset);
// A shelf's dimensions are what it was told; the file's are what it has.
// Everything sized off the asset — the fit, the reader's budget, an export's
// source size — reads this, so the truth lands behind the placement.
if (init.type === "video") {
void probeMediaFile(asset.url)
.then((m) => {
if (m.hasVideo) void landProbedFacts(asset.id, { width: m.width, height: m.height, color: assetColor(m.color) });
})
.catch(() => {});
}
return asset;
}
/** Store a fetchable image (a stock tile) in the project's media as a
* first-class image asset at its native resolution — no video baking — and
* register it, without placing it on the timeline. Callers choose where it
* lands. Ready as soon as the pixel size is read; the bytes copy in behind
* the editor. */
export async function importImage(
projectId: string,
image: { url: string; name: string }
): Promise<MediaAsset> {
// The size frames the still (and the first-asset aspect guess); the source
// is a same-origin file, so this is one cached header read, not a download.
const dims = await loadImageMeta(image.url);
// 0×0 is how the read reports a source it could not open (a stock id that
// no longer resolves, a dead link). Registering that would place a broken
// still and hand the first-asset aspect guess a meaningless size, so the
// import fails here instead.
if (dims.width === 0 || dims.height === 0) throw new UserError("image_read_failed", "Could not read the image.");
// A stock image lands on the timeline where the caller places it, not in the
// Media panel — tag it so it stays out.
return importRemote(projectId, {
url: image.url,
name: image.name,
type: "image",
duration: 0,
width: dims.width,
height: dims.height,
origin: "stock",
});
}
/** Store a fetchable video (a stock clip) in the project's media as a regular
* video asset and register it, without placing it on the timeline. Callers
* choose where it lands. Instant when the catalog supplies the duration;
* otherwise one metadata read stands between the call and the asset. */
export async function importStockVideo(
projectId: string,
video: { url: string; name: string; duration?: number; width?: number; height?: number }
): Promise<MediaAsset> {
const duration = video.duration ?? (await probeMediaFile(video.url)).duration;
// Like a stock image, it lands where the caller places it, not in Media.
return importRemote(projectId, {
url: video.url,
name: video.name,
type: "video",
duration,
width: video.width,
height: video.height,
origin: "stock",
});
}
/** Store a bundled stock audio file — a music bed or a one-shot sound effect —
* in the project's media as a regular audio asset and register it, without
* placing it on the timeline; callers choose where it lands (the soundtrack).
* Tagged "stock" so it stays out of Media. */
export async function importStockAudio(
projectId: string,
music: { url: string; name: string; duration?: number }
): Promise<MediaAsset> {
const duration = music.duration ?? (await probeMediaFile(music.url)).duration;
return importRemote(projectId, {
url: music.url,
name: music.name,
type: "audio",
duration,
origin: "stock",
});
}
// A URL import that failed, worded the same whichever residency ran it.
const urlImportFailed = () => new UserError("url_import_failed", "Could not import that URL.");
/** Download a URL (TikTok, YouTube, an X post, an article, …) into the project
* through the engine's bundled downloader and register what came back — one
* asset for a video, one per photo for a photo post, none at all for a source
* that is only words — without placing anything on the timeline. Callers
* choose where assets land. `text` is the source's own words: a post's body,
* an article, or a video's title and description. */
export async function importUrlMedia(
projectId: string,
url: string,
opts: { audio?: boolean } = {}
): Promise<{ assets: MediaAsset[]; text?: string }> {
url = normalizeLink(url);
// Pinned at start: the download can outlast navigation into a project of
// the other residency, and every poll must hit the backend the job started on.
const backend = getBackend();
// A browser project has no server of its own to run the fetch, so its import
// rides the cloud worker and the bytes come back down.
if (backend.kind === "browser") {
const staged = await importUrlThroughCloud(projectId, backend, url, opts);
return adoptImportedFiles(projectId, staged, backend);
}
const res = await backend.fetch(`/api/cut/projects/${projectId}/import-url`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url, ...(opts.audio ? { audio: true } : {}) }),
});
let body: { files?: { fileName: string; title: string }[]; text?: string; error?: string };
if (backend.kind === "cloud") {
// The cloud route is async: it answers {jobId} and a worker does the fetch.
const started = await apiJson<{ jobId?: string }>(res);
if (!res.ok || !started.jobId) throw errorFromBody(started, urlImportFailed());
body = await pollCloudJob(started.jobId, backend, urlImportFailed());
} else {
body = await apiJson<{ files?: { fileName: string; title: string }[]; text?: string }>(res);
}
// A source that is only words (a text post, an article, a page) imports as
// that text with no files, so an empty file list with text is a success.
if (!res.ok || (!body.files?.length && !body.text)) {
throw errorFromBody(body, urlImportFailed());
}
return adoptImportedFiles(projectId, body, backend);
}
/** Register the files an import landed in the project, in order, and hand back
* what the caller places. */
export async function adoptImportedFiles(
projectId: string,
imported: { files?: { fileName: string; title: string }[]; text?: string },
backend: CutBackend = getBackend()
): Promise<{ assets: MediaAsset[]; text?: string }> {
const assets: MediaAsset[] = [];
for (const f of imported.files ?? []) {
const asset = await assetFromProjectFile(
projectId,
f.fileName,
f.title || "Imported clip",
backend
);
useEditor.getState().addAsset(asset);
void enrichAsset(asset);
assets.push(asset);
}
return { assets, text: imported.text };
}
/**
* A browser project's URL import. Nothing in the page can fetch a TikTok or a
* YouTube video, so the cloud worker does it — into the account's library,
* where the page can read it — and the bytes come down into this browser's own
* storage. The staged library copy is dropped once the project holds it, so
* the import leaves nothing behind on the cloud shelf.
*/
async function importUrlThroughCloud(
projectId: string,
backend: CutBackend,
url: string,
opts: { audio?: boolean } = {}
): Promise<{ files: { fileName: string; title: string }[]; text?: string }> {
const res = await cloudBackend.fetch("/api/cut/library/import-url", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url, ...(opts.audio ? { audio: true } : {}) }),
});
const started = await apiJson<{ jobId?: string }>(res);
if (!res.ok || !started.jobId) throw errorFromBody(started, urlImportFailed());
const done = await pollCloudJob<{
assets?: { id: string; fileName: string; name: string }[];
text?: string;
}>(started.jobId, cloudBackend, urlImportFailed());
// Words with no media are a successful import; nothing at all is a failure.
if (!done.assets?.length && !done.text) throw urlImportFailed();
const files: { fileName: string; title: string }[] = [];
// Every staged copy is dropped from the cloud shelf, however this ends: a
// download or a local write that throws part way through would otherwise
// leave the user holding — and paying quota for — library media they never
// asked for and cannot see from a browser project.
const dropStaged = async () => {
for (const staged of done.assets ?? []) {
await cloudBackend
.fetch(`/api/cut/library/${staged.id}`, { method: "DELETE" })
.catch(() => {});
}
};
try {
for (const staged of done.assets ?? []) {
const path = `/api/cut/library/media/${encodeURIComponent(staged.fileName)}`;
const bytes = await cloudBackend.fetch(path);
if (!bytes.ok) throw urlImportFailed();
const fileName = await uploadProjectMediaTo(
backend,
projectId,
await bytes.blob(),
staged.fileName
);
files.push({ fileName, title: staged.name });
}
} finally {
void dropStaged();
}
return { files, ...(done.text ? { text: done.text } : {}) };
}
/** Build a runtime asset for a media file the engine already wrote into the
* project folder (freeze frames, AI generations, URL imports) — probe
* metadata, no upload. */
export async function assetFromProjectFile(
projectId: string,
fileName: string,
name: string,
backend: CutBackend = getBackend()
): Promise<MediaAsset> {
const url = await storedMediaUrl(projectId, fileName, backend);
const asset: MediaAsset = {
id: uid(),
fileName,
name,
type: "video",
duration: 0,
url,
};
if (/\.(png|jpe?g|webp|gif|avif|bmp)$/i.test(fileName)) {
asset.type = "image";
const dims = await loadImageMeta(url);
asset.width = dims.width;
asset.height = dims.height;
asset.sizeBytes = await readMediaFileSize(url);
return asset;
}
const meta = await probeMediaFile(url);
asset.duration = meta.duration;
asset.sizeBytes = meta.sizeBytes;
if (!meta.hasVideo) {
asset.type = "audio";
} else {
asset.width = meta.width;
asset.height = meta.height;
const color = assetColor(meta.color);
if (color) asset.color = color;
}
return asset;
}
/** Cloud twin of the engine's freeze route: grab the source frame in the
* browser (seek + canvas at native resolution), store it as a PNG project
* image, and return the same asset shape the engine's freeze response has.
* `duration` (seconds) rides back on the asset so the caller can size the
* placed clip like the engine's baked still video; the stored image itself
* has no intrinsic length. */
export async function captureFreezeFrame(
projectId: string,
sourceUrl: string,
srcTime: number,
duration = 0,
/** Bake the still at the project frame, framed as the clip is — the engine's
* /freeze route does the same with ffmpeg, so a still captured in the browser
* and one captured on the Mac are the same picture. */
framed?: {
frame: { w: number; h: number };
cover: boolean;
zoom?: number;
panX?: number;
panY?: number;
flipH?: boolean;
flipV?: boolean;
}
): Promise<MediaAsset> {
const frame = await baseFrameAt(sourceUrl, srcTime);
if (!frame) throw new Error("Could not render the freeze frame.");
let picture = frame.canvas;
if (framed) {
const { w, h } = framed.frame;
const out = createRasterCanvas(w, h);
const ctx = out.getContext("2d") as CanvasRenderingContext2D;
ctx.fillStyle = "#000000";
ctx.fillRect(0, 0, w, h);
const box = { x: 0, y: 0, w, h };
const r = contentRect(
box,
frame.canvas.width,
frame.canvas.height,
framed.cover,
framed.zoom,
framed.panX,
framed.panY
);
ctx.save();
ctx.beginPath();
ctx.rect(0, 0, w, h);
ctx.clip();
if (framed.flipH || framed.flipV) {
ctx.translate(w / 2, h / 2);
ctx.scale(framed.flipH ? -1 : 1, framed.flipV ? -1 : 1);
ctx.translate(-w / 2, -h / 2);
}
ctx.drawImage(frame.canvas as CanvasImageSource, r.x, r.y, r.w, r.h);
ctx.restore();
picture = out;
}
const blob = await rasterCanvasToBlob(picture, "image/png");
const d = new Date();
const pad = (n: number) => String(n).padStart(2, "0");
const fileName = `freeze-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}-${uid().slice(0, 4)}.png`;
const asset = await uploadProjectImage(projectId, blob, fileName, {
failMessage: "Could not render the freeze frame.",
});
return duration > 0 ? { ...asset, duration } : asset;
}
const round2 = (n: number) => Math.round(n * 100) / 100;
const round3 = (n: number) => Math.round(n * 1000) / 1000;
/** Decoded audio for a span of a source, chunk by chunk, so scanning a long
* file costs one decoded chunk at a time rather than a decoded copy of the
* whole thing. Throws when the file carries no audio at all — whether it does
* is asked of the file, not of the span, so a range past the end of a
* perfectly good recording is an empty answer instead of an error. */
async function* sourcePcm(
sourceUrl: string,
from: number,
to: number | undefined
): AsyncGenerator<PcmChunk> {
const input = openMedia(sourceUrl);
const track = await audioTrackOf(input).finally(() => input.dispose());
if (!track) throw new UserError("media_no_audio_track", "This file has no audio track.");
if (to !== undefined && !(to > from)) return;
for await (const { buffer, timestamp } of audioChunks(sourceUrl, from, to)) {
yield {
channels: Array.from({ length: buffer.numberOfChannels }, (_, c) =>
buffer.getChannelData(c)
),
timestamp,
sampleRate: buffer.sampleRate,
};
}
}
/** Cloud twin of the engine's silence route (ffmpeg silencedetect): read the
* source's audio and scan 20ms RMS windows against the same dB threshold and
* minimum-duration rules. Times are absolute source seconds. */
export async function detectSilenceClientSide(
sourceUrl: string,
opts: { from: number; to?: number; thresholdDb: number; minSilence: number }
): Promise<{ start: number; end: number; duration: number }[]> {
const from = Math.max(0, opts.from);
return scanSilence(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });
}
/** Where the words are in a span of a source. One implementation for every
* residency: the page reads its own media this way in local and cloud mode
* alike, and the headless runner installs the same decoders. */
export async function scanSourceSpeech(
sourceUrl: string,
opts: { from: number; to?: number }
): Promise<SpeechScan> {
const from = Math.max(0, opts.from);
return scanSpeech(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });
}
/** Where a separately recorded sound lines up with a camera's own: read off
* both files where the page or the headless runner decodes them (soundSync.ts
* holds the arithmetic). */
export async function syncSourceSound(
cameraUrl: string,
recordingUrl: string,
opts: { cameraFrom: number; cameraTo: number; recordingTo: number }
): Promise<SyncAnswer | null> {
return syncSound(
{
camera: (from, to) => sourcePcm(cameraUrl, from, to),
recording: (from, to) => sourcePcm(recordingUrl, from, to),
},
opts
);
}
/** How loud a span of a source plays: its RMS level and, from the same
* decode, its BS.1770 loudness and true peak. A mono source is measured as
* the two channels it plays on in the stereo mix. Same residency story as the
* speech scan: the page decodes its own media, the headless runner installs
* the same decoders. */
export async function measureSourceLevel(
sourceUrl: string,
opts: { from: number; to?: number }
): Promise<LevelScan & { loudness: LoudnessMeasure | null }> {
const from = Math.max(0, opts.from);
const metered: { meter?: LoudnessMeter } = {};
async function* tapped(): AsyncGenerator<PcmChunk> {
for await (const chunk of sourcePcm(sourceUrl, from, opts.to)) {
const n = chunk.channels[0]?.length ?? 0;
if (n > 0) {
const rate = chunk.sampleRate;
const meter = (metered.meter ??= new LoudnessMeter(rate, chunk.channels.length));
// A decode opens on the packet before `from` and may run past `to`;
// only the asked span is measured.
const a = Math.max(0, Math.round((from - chunk.timestamp) * rate));
const b = opts.to === undefined ? n : Math.min(n, Math.round((opts.to - chunk.timestamp) * rate));
if (b > a) {
meter.push(
Array.from({ length: meter.channels }, (_, c) => (chunk.channels[c] ?? chunk.channels[0]).subarray(a, b))
);
}
}
yield chunk;
}
}
const level = await scanLevel(tapped(), { ...opts, from });
return { ...level, loudness: metered.meter ? metered.meter.result() : null };
}
/** The source's musical beat grid, end to end. The grid is a property of the
* whole file — tempo tracking needs the run of it — so a scan always reads
* everything; callers window the result. Same residency story as the speech
* scan: the page decodes its own media, the headless runner installs the
* same decoders. */
export async function scanSourceBeats(sourceUrl: string): Promise<BeatScan> {
return scanBeats(sourcePcm(sourceUrl, 0, undefined));
}
/** Cloud twin of the engine's audio-extract route: render a span of the
* source's audio to 16 kHz mono WAV in the browser, for the AI to hear
* inline. An empty `to` runs to the end. */
export async function renderAudioSpanWav(
sourceUrl: string,
from: number,
to: number | undefined
): Promise<Blob> {
// Only the asked-for span is decoded, so pulling thirty seconds out of an
// hour-long file costs thirty seconds of work.
const span = await decodeAudioSpan(sourceUrl, Math.max(0, from), to);
if (!span) throw new UserError("media_no_audio_track", "This file has no audio track.");
const dur = span.duration;
if (!(dur > 0)) throw new Error("from/to describe an empty range.");
const rate = 16000; // encodeWav's fixed sample rate
const ctx = new OfflineAudioContext(1, Math.max(1, Math.ceil(dur * rate)), rate);
const src = ctx.createBufferSource();
src.buffer = span;
src.connect(ctx.destination);
src.start(0);
return encodeWav((await ctx.startRendering()).getChannelData(0));
}
// Watch geometry: cells at a fixed long side, tiled and stamped by
// composeSheets. One sampler serves the local and cloud backends alike — the
// browser decodes the source (local media rides the engine's media URL, cloud
// media the CDN) and the shared selector keeps the frames that differ.
const SHEET_GAP = 4; // tile margin and padding, px
const CANDIDATE_CAP = 150; // decodes per call — the densest sweep
const SELECT_CELL = 480; // what the selector decodes at when the frames are bigger
const REFINE_TARGET_S = 0.35; // localize each hard cut to about this window
const REFINE_MAX_PROBES = 48; // extra seeks the refinement pass may spend per call
const CUT_WALK_EPS = 0.0005; // half a millisecond: lands a walk's ends on the frames they name
const AIMED_SHARE = 0.5; // most of the cap aimed moments may take from the floor
const MERGE_EPS = 0.15; // two candidate times this close are one decode
/** The candidate list: the moments a caller knows matter, plus a steady floor
* across everything else. The floor is built at whatever density the room
* left allows, so aiming widens the sweep instead of cutting it short — every
* second in [from, to] still sits between two candidates. */
function candidateTimes(
from: number,
to: number,
interval: number,
at: number[],
shots: number[]
): number[] {
const cap = Math.floor(CANDIDATE_CAP * AIMED_SHARE);
// Measured shots first, each its own decode: a flash card two frames long
// sits closer to the next than MERGE_EPS, and nothing else would show it.
const measured = [...new Set(thinAcross(shots.filter((t) => t >= from && t < to), cap).map(round2))];
// More lines than the share allows are thinned across the whole span. Taking
// the first of them would aim the head of a densely spoken stretch and leave
// its tail sampled thinner than the floor alone would have.
const lines = thinAcross(at.filter((t) => t >= from && t < to), Math.max(0, cap - measured.length));
const aimed: number[] = [];
for (const t of lines) {
if (aimed.length === 0 || t - aimed[aimed.length - 1] >= MERGE_EPS) {
aimed.push(round2(t));
}
}
const room = Math.max(1, CANDIDATE_CAP - aimed.length - measured.length);
const step = Math.max(interval, (to - from) / room);
const floor: number[] = [];
for (let t = from; t < to && floor.length < room; t += step) floor.push(round2(t));
// A line or a floor sample crowding a measured shot gives way to it.
const crowds = (t: number) => measured.some((m) => Math.abs(m - t) < MERGE_EPS);
const times: number[] = [];
for (const t of [...aimed, ...floor].filter((t) => !crowds(t)).sort((a, b) => a - b))
if (times.length === 0 || t - times[times.length - 1] >= MERGE_EPS) times.push(t);
return [...times, ...measured].sort((a, b) => a - b);
}
/** At most `cap` of the ascending times, spread across all of them. */
function thinAcross(times: number[], cap: number): number[] {
const sorted = [...times].sort((a, b) => a - b);
if (cap <= 0) {
return [];
}
const every = Math.max(1, Math.ceil(sorted.length / cap));
return sorted.filter((_, i) => i % every === 0);
}
/** How much of each kept frame reaches the model, from the tiled thumbnails
* that answer "what happens here" up to the frames themselves.
*
* Every step trades frames for pixels at roughly even cost, because a sheet
* is billed by its area: a 9:16 source gives cells of 405×720, 608×1080 and
* 1080×1920, where a 46px caption lands at about 13px, 20px and full size.
* Small type is the reason the ladder exists — a mosaic cell throws away the
* weight, spacing and edge treatment that reading text or matching a look
* depends on, and nothing downstream can put them back.
*
* `cell` is the cell's long side in px. */
const WATCH_DETAIL: Record<
WatchDetail,
{ cell: number; grid: number; sheets: number; quality: number; rewalk: boolean }
> = {
/** Coverage: what happens, where the cuts fall, how long a stretch runs. */
scan: { cell: 720, grid: 3, sheets: 4, quality: 0.8, rewalk: false },
/** Reading: on-screen text, UI, anything whose words matter. */
read: { cell: 1080, grid: 2, sheets: 4, quality: 0.86, rewalk: false },
/** Matching: type, colour, edges and grain as the source has them. The
* frames are big enough that decoding every candidate at this size would
* cost far more than the handful that are kept, so this step selects small
* and walks the kept times a second time. */
original: { cell: 1920, grid: 1, sheets: 6, quality: 0.92, rewalk: true },
};
/** Frames one call returns at this detail, and the geometry behind them.
* A cell the source is already smaller than hands back the source's own
* pixels, which is what "original" is for a phone or 1080p source; a 4K or
* larger one meets the ceiling, because six 4K frames are several times the
* whole per-call media budget on their own. */
export const watchGeometry = (detail: WatchDetail = "scan") => {
const g = WATCH_DETAIL[detail] ?? WATCH_DETAIL.scan;
return { ...g, maxFrames: g.grid * g.grid * g.sheets };
};
/** The watch sample: kept frames, ascending source time, each carrying why
* the selector kept it (WatchKeepReason in types.ts). */
export interface WatchFrames {
frames: { t: number; image: string; via: WatchKeepReason }[];
/** Per kept frame, the TEXT_GRID² edge energies the selector measured. */
edges: Float32Array[];
candidates: number;
sceneChanges: number[];
coveredTo: number;
truncated: boolean;
}
const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));
// A cell the source is already smaller than passes the frame through at its
// own size — a watch never invents detail by upscaling, and the even-rounding
// below would resample an odd-sized frame to no purpose. A sized cell rounds
// its short side to even, so encoded dimensions stay codec-safe.
const cellDims = (w: number, h: number, cell: number): [number, number] => {
if (cell <= 0 || cell >= Math.max(w, h)) return [Math.max(1, w), Math.max(1, h)];
return w >= h
? [cell, Math.max(2, 2 * Math.round((cell * h) / w / 2))]
: [Math.max(2, 2 * Math.round((cell * w) / h / 2)), cell];
};
/** Watch a still image: one frame at the asked-for detail, no time axis. */
export async function makeStillFrame(
sourceUrl: string,
detail: WatchDetail = "scan"
): Promise<WatchFrames> {
const img = await decodeRasterImageUrl(sourceUrl);
if (!img || img.width === 0 || img.height === 0) throw new UserError("image_read_failed", "Could not read the image.");
const g = watchGeometry(detail);
const [w, h] = cellDims(img.width, img.height, g.cell);
const canvas = createRasterCanvas(w, h);
const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
if (!ctx) throw new UserError("image_read_failed", "Could not read the image.");
ctx.imageSmoothingQuality = "high";
ctx.drawImage(img.source, 0, 0, w, h);
// A picture read through an element (a host that sends no CORS header)
// taints the canvas, and the encode is where that surfaces.
const image = await rasterCanvasToDataUrl(canvas, "image/jpeg", g.quality).catch(() => {
throw new UserError("image_read_failed", "Could not read the image.");
});
return {
frames: [{ t: 0, image, via: "first" }],
edges: [],
candidates: 1,
sceneChanges: [],
coveredTo: 0,
truncated: false,
};
}
/** Watch a video source: decode candidates on a dense steady floor plus any
* moments `at` aims at, then let the shared selector keep the frames that
* actually differ. sceneChanges are
* hard-cut times: each "global" keep says a cut lies inside the grid step
* before it, a short bisection pass narrows that window to about
* REFINE_TARGET_S, and an in-order walk of the window's frames reports the
* first frame of the new shot, to the millisecond.
* A shot shorter than the candidate step can still fall between samples; a
* narrow re-watch with a small interval is the recall pass.
*
* Coverage is honest by construction: the keep-cap, a pause, and the time
* budget all END the pass (coveredTo says where) instead of thinning or
* guessing — everything inside [from, coveredTo] was decoded and judged, and
* every kept frame in it is returned. A source that decodes no frames at all
* throws; it is never reported as watched.
*
* metadataOnly skips the cell copies and JPEG encodes — frames come back with
* an empty image — for the background sweep, which only wants the numbers.
* shouldPause is polled per candidate; budgetMs bounds the whole call the
* same way, for callers racing a tool deadline. */
export async function sampleWatchFrames(
sourceUrl: string,
opts: {
from: number;
to?: number;
interval?: number;
metadataOnly?: boolean;
shouldPause?: () => boolean;
budgetMs?: number;
detail?: WatchDetail;
/** Moments worth a decode on top of the steady floor. What is on screen
* changes when what is heard changes — a cue lands, a line starts, a beat
* hits — so a source whose audio has already been read is sampled on its
* own clock instead of hoping an even sweep lands on each state. */
at?: number[];
/** Moments a frame-by-frame read measured — the middle of each shot
* detect_cuts found and the frame each change inside a shot settles on.
* Each gets a decode of its own, however close to the next. */
shots?: number[];
/** The part of the frame that is the source's picture (a screen
* recording's player, a letterboxed film), centre and size as frame
* fractions: frames are judged and returned inside it, so a shot changing
* in a phone's player is not outvoted by the app around it. */
picture?: { x: number; y: number; w: number; h: number };
}
): Promise<WatchFrames> {
const input = openMedia(sourceUrl);
try {
const track = await videoTrackOf(input);
if (!track) throw new Error("Could not sample the video.");
const reading = colorReadAt(sourceUrl);
const from = Math.max(0, opts.from);
const wanted = opts.to ?? (await input.computeDuration());
if (opts.to === undefined && !(wanted > 0))
throw new Error("Could not read the media duration — pass to (seconds).");
if (!(wanted > from)) throw new Error("from/to describe an empty range.");
const to = Math.min(wanted, from + 600); // bound the work per call; callers resume from coveredTo
const interval = clamp(opts.interval ?? 1, 0.5, 30);
const times = candidateTimes(from, to, interval, opts.at ?? [], opts.shots ?? []);
const geom = watchGeometry(opts.detail);
const fullW = await track.getDisplayWidth();
const fullH = await track.getDisplayHeight();
const area = opts.picture && (opts.picture.w < 0.999 || opts.picture.h < 0.999) ? opts.picture : null;
// Cells are sized to the picture; frames decode at whatever size gives
// the picture those pixels, and `blit` cuts the picture out of each.
const srcW = area ? fullW * area.w : fullW;
const srcH = area ? fullH * area.h : fullH;
const decodeSize = (w: number, h: number) =>
area ? { width: Math.round(w / area.w), height: Math.round(h / area.h), fit: "fill" as const } : { width: w, height: h, fit: "fill" as const };
const blit = (ctx: CanvasRenderingContext2D, src: CanvasImageSource & { width: number; height: number }, w: number, h: number) => {
if (!area) return ctx.drawImage(src, 0, 0, w, h);
ctx.drawImage(src, (area.x - area.w / 2) * src.width, (area.y - area.h / 2) * src.height, area.w * src.width, area.h * src.height, 0, 0, w, h);
};
const twoPass = !opts.metadataOnly && geom.rewalk;
// Selection reads a 192² signature whatever the detail, so a pass that
// returns no pictures — or that walks the kept times again for big ones —
// decodes candidates at the cheap size.
const selectOnly = opts.metadataOnly || twoPass;
const [cw, ch] = cellDims(srcW, srcH, selectOnly ? SELECT_CELL : geom.cell);
const cellSize = decodeSize(cw, ch);
const maxFrames = geom.maxFrames;
// Cell canvases are held only until the selector rules on them (decisions
// run one frame behind), so memory stays near the kept set, never the
// candidate sweep.
const held = new Map<number, RasterSurface>();
const candTimes: number[] = [];
const candG16: Float32Array[] = []; // per-candidate coarse grid, for cut refinement
const candE24: Float32Array[] = []; // per-candidate edge energy, for where a card can sit
let keptCount = 0;
const selector = createDedupSelector({
maxFrames,
onDecision: (d) => {
if (d.verdict === "drop") held.delete(d.index);
else keptCount++;
},
});
const sig = createRasterCanvas(SIGNATURE_SIZE, SIGNATURE_SIZE);
const sigCtx = sig.getContext("2d", {
willReadFrequently: true,
}) as CanvasRenderingContext2D | null;
if (!sigCtx) throw new Error("Could not sample the video.");
const deadline = Date.now() + (opts.budgetMs ?? Infinity);
const outOfTime = () => Date.now() > deadline;
const sigG16Of = (canvas: CanvasImageSource & { width: number; height: number }): Float32Array => {
sigCtx.imageSmoothingQuality = "high";
blit(sigCtx, canvas, SIGNATURE_SIZE, SIGNATURE_SIZE);
const data = sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data;
return frameSig({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 4, data }).g16;
};
// The timestamp of the first frame in (lo, hi] that differs from `pre`,
// decoded in order; null when the walk finds none.
const firstNewFrame = async (lo: number, hi: number, pre: Float32Array): Promise<number | null> => {
const walk = baseFrameSink(track, cellSize, reading).canvases(lo, hi + CUT_WALK_EPS);
try {
for await (const frame of walk) {
if (frame.timestamp <= lo - CUT_WALK_EPS) continue;
const g = sigG16Of(frame.canvas);
if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) return frame.timestamp;
}
} finally {
await walk.return(undefined);
}
return null;
};
const sigG16At = async (t: number): Promise<Float32Array | null> => {
const one = await baseFrameSink(track, cellSize, reading).getCanvas(t);
return one ? sigG16Of(one.canvas) : null;
};
// The times are ascending, so the whole span decodes once, in order.
const cells = baseFrameSink(track, cellSize, reading).canvasesAtTimestamps(times);
let paused = false;
let capped = false;
try {
for (let i = 0; i < times.length; i++) {
if (opts.shouldPause?.() || outOfTime()) {
paused = true;
break;
}
// The keep-cap ends the pass — everything reported stays everything
// seen, and the caller resumes from coveredTo.
if (keptCount >= maxFrames) {
capped = true;
break;
}
const next = await cells.next();
if (next.done || !next.value) continue; // a moment the decoder had no frame for
if (!opts.metadataOnly && !twoPass) {
const copy = createRasterCanvas(cw, ch);
const cctx = copy.getContext("2d") as CanvasRenderingContext2D | null;
if (!cctx) throw new Error("Could not sample the video.");
blit(cctx, next.value.canvas, cw, ch);
held.set(candTimes.length, copy);
}
candTimes.push(times[i]);
sigCtx.imageSmoothingQuality = "high";
blit(sigCtx, next.value.canvas, SIGNATURE_SIZE, SIGNATURE_SIZE);
const rgb = {
width: SIGNATURE_SIZE,
height: SIGNATURE_SIZE,
channels: 4 as const,
data: sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,
};
// One signature serves both the cut-refinement pass and the
// selector; measuring the same frame twice is the whole cost again.
const candSig = frameSig(rgb);
candG16.push(candSig.g16);
candE24.push(candSig.e24);
selector.push(candSig);
}
} finally {
await cells.return();
}
const selection = selector.finish();
// Nothing decoded and nothing interrupted the pass: the source is
// unreadable. Say so — never record an unseen span as watched.
if (candTimes.length === 0 && !paused) throw new Error("Could not sample the video.");
// finish() judges the one pending frame, which can nudge past the cap.
const keptAll = selection.kept.slice(0, maxFrames);
let keptIdx = keptAll;
// The big frames: one more ordered walk over the kept times, each encoded
// as it arrives so one of them is live at a time. A walk that runs out of
// budget delivers what it reached and the kept set shrinks to match, so
// coveredTo below still describes frames that exist.
const encoded = new Map<number, string>();
if (twoPass && keptAll.length > 0) {
const [nw, nh] = cellDims(srcW, srcH, geom.cell);
const shot = createRasterCanvas(nw, nh);
const shotCtx = shot.getContext("2d") as CanvasRenderingContext2D | null;
if (!shotCtx) throw new Error("Could not sample the video.");
const walk = baseFrameSink(track, decodeSize(nw, nh), reading).canvasesAtTimestamps(
keptAll.map((i) => candTimes[i])
);
try {
for (const idx of keptAll) {
if (outOfTime() || opts.shouldPause?.()) break;
const frame = await walk.next();
if (frame.done) break;
if (!frame.value) continue;
blit(shotCtx, frame.value.canvas, nw, nh);
encoded.set(idx, await rasterCanvasToDataUrl(shot, "image/jpeg", geom.quality));
}
} finally {
await walk.return();
}
keptIdx = keptAll.filter((idx) => encoded.has(idx));
// A walk that reached nothing saw nothing. Saying so sends the caller
// somewhere new; an empty pass reported as a success sends it here again.
if (keptIdx.length === 0 && !paused) throw new Error("Could not sample the video.");
}
const frames: WatchFrames["frames"] = [];
for (const idx of keptIdx) {
const canvas = twoPass ? null : held.get(idx);
if (!opts.metadataOnly && !twoPass && !canvas) continue;
const verdict = selection.decisions[idx].verdict;
frames.push({
t: candTimes[idx],
image: twoPass
? encoded.get(idx)!
: canvas
? await rasterCanvasToDataUrl(canvas, "image/jpeg", geom.quality)
: "",
via: verdict === "keep-global" ? "global"
: verdict === "keep-action" ? "action"
: verdict === "keep-text" ? "text"
: verdict === "keep-settled" ? "settled"
: "first",
});
}
held.clear();
encoded.clear();
// A "global" keep only says the cut lies in the grid step before it.
// Bisect that step with a few extra decodes down to REFINE_TARGET_S, then
// walk what is left frame by frame; the kept image stays the grid sample.
let probes = 0;
const sceneChanges: number[] = [];
for (const idx of keptIdx) {
if (selection.decisions[idx].verdict !== "keep-global") continue;
let lo = idx > 0 ? candTimes[idx - 1] : from;
let hi = candTimes[idx];
const pre = idx > 0 ? candG16[idx - 1] : null;
while (
pre !== null &&
hi - lo > REFINE_TARGET_S &&
probes < REFINE_MAX_PROBES &&
!outOfTime() &&
!opts.shouldPause?.()
) {
probes++;
const mid = (lo + hi) / 2;
const g = await sigG16At(mid);
if (!g) break;
if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) hi = mid;
else lo = mid;
}
// The window is a few frames wide now: walk it in order and report the
// first frame of the new shot itself, so a cut lands on its frame.
const first =
pre !== null && probes < REFINE_MAX_PROBES && !outOfTime() && !opts.shouldPause?.()
? await firstNewFrame(lo, hi, pre)
: null;
if (first !== null) probes++;
sceneChanges.push(round3(first ?? hi));
}
// An interrupted or capped scan only covered what it decoded and kept;
// the caller merges that much and comes back for the rest.
const lastKeptT = keptIdx.length > 0 ? candTimes[keptIdx[keptIdx.length - 1]] : from;
// A walk that delivered fewer frames than the selector kept stopped early,
// so the range only reaches the last frame that exists.
const shortWalk = twoPass && keptIdx.length < keptAll.length;
const coveredTo = shortWalk
? lastKeptT
: paused
? (candTimes[candTimes.length - 1] ?? from)
: capped
? lastKeptT
: to;
return {
frames,
candidates: selection.candidateCount,
sceneChanges,
coveredTo,
// What each kept frame holds where, already measured for selection: the
// busy cells are where the picture's detail and its own type live, so a
// card placed over the quiet ones covers nothing.
edges: keptIdx.map((i) => candE24[i]).filter((g): g is Float32Array => !!g),
// The per-call span bound is itself truncation — the caller asked for more.
truncated: paused || capped || shortWalk || to < wanted,
};
} finally {
input.dispose();
}
}
/** Tile kept frames into contact sheets and stamp each cell's source time, at
* the detail's grid. The last sheet may run short; its frames list only the
* real cells. At "original" there is no grid and no stamp — the frames go as
* they are, because a burned-in label is a graphic the source does not have,
* and that detail exists for reading the ones it does. */
export async function composeSheets(
cells: { t: number; image: string }[],
detail: WatchDetail = "scan"
): Promise<{ image: string; frames: { t: number }[] }[]> {
if (cells.length === 0) return [];
const { grid, quality } = watchGeometry(detail);
if (grid <= 1) return cells.map((c) => ({ image: c.image, frames: [{ t: c.t }] }));
const images = await Promise.all(
cells.map(async (c) => {
const img = await decodeRasterImageUrl(c.image);
if (!img) throw new Error("Bad frame image.");
return img;
})
);
const cw = images[0].width;
const ch = images[0].height;
const perSheet = grid * grid;
const sheetW = 2 * SHEET_GAP + grid * cw + (grid - 1) * SHEET_GAP;
const sheetH = 2 * SHEET_GAP + grid * ch + (grid - 1) * SHEET_GAP;
const size = Math.max(13, Math.round(ch / 12));
const sheets: { image: string; frames: { t: number }[] }[] = [];
for (let s = 0; s < cells.length; s += perSheet) {
const batch = cells.slice(s, s + perSheet);
const canvas = createRasterCanvas(sheetW, sheetH);
const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
if (!ctx) throw new Error("No 2d context.");
ctx.fillStyle = "#000";
ctx.fillRect(0, 0, sheetW, sheetH);
ctx.imageSmoothingQuality = "high";
ctx.font = `bold ${size}px ui-monospace, monospace`;
ctx.textBaseline = "bottom";
batch.forEach((c, j) => {
const x = SHEET_GAP + (j % grid) * (cw + SHEET_GAP);
const y = SHEET_GAP + Math.floor(j / grid) * (ch + SHEET_GAP);
ctx.drawImage(images[s + j].source, x, y, cw, ch);
const label = `${round2(c.t)}s`;
const w = ctx.measureText(label).width;
ctx.fillStyle = "rgba(0,0,0,0.6)";
ctx.fillRect(x + 4, y + ch - size - 10, w + 10, size + 8);
ctx.fillStyle = "#fff";
ctx.fillText(label, x + 9, y + ch - 6);
});
sheets.push({
image: await rasterCanvasToDataUrl(canvas, "image/jpeg", quality),
frames: batch.map((c) => ({ t: c.t })),
});
}
return sheets;
}
/**
* One moment of a replica beside the thing it copies: the source's frame on
* the left, the cut's own frame on the right, both stamped with the second
* they are read at.
*
* The pairing is the point. Two images handed over separately are two things
* to hold in mind at once; side by side in one picture, a colour that drifted,
* a card sitting too low or a move that peaks early is simply visible. The
* composite is for the eye; the score beside it comes from frameDiff.ts.
*/
export async function composeComparison(
pairs: { t: number; sourceAt: number; source: string; cut: string }[]
): Promise<string[]> {
const out: string[] = [];
for (const pair of pairs) {
const [a, b] = await Promise.all([
decodeRasterImageUrl(pair.source),
decodeRasterImageUrl(pair.cut),
]);
if (!a || !b) throw new Error("Bad frame image.");
// Both frames stand at one height, so the eye compares like with like
// whatever the source's own shape is.
const h = Math.max(a.height, b.height);
const aw = Math.round((a.width / a.height) * h);
const bw = Math.round((b.width / b.height) * h);
const size = Math.max(13, Math.round(h / 22));
const bar = size + 10;
const w = SHEET_GAP * 3 + aw + bw;
const canvas = createRasterCanvas(w, h + bar + SHEET_GAP * 2);
const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
if (!ctx) throw new Error("No 2d context.");
ctx.fillStyle = "#000";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.imageSmoothingQuality = "high";
ctx.drawImage(a.source, SHEET_GAP, SHEET_GAP + bar, aw, h);
ctx.drawImage(b.source, SHEET_GAP * 2 + aw, SHEET_GAP + bar, bw, h);
ctx.font = `bold ${size}px ui-monospace, monospace`;
ctx.textBaseline = "top";
ctx.fillStyle = "#fff";
ctx.fillText(`SOURCE ${round2(pair.sourceAt)}s`, SHEET_GAP, SHEET_GAP);
ctx.fillText(`YOUR CUT ${round2(pair.t)}s`, SHEET_GAP * 2 + aw, SHEET_GAP);
out.push(await rasterCanvasToDataUrl(canvas, "image/jpeg", 0.82));
}
return out;
}
// Filmstrip generation that failed, by asset id — session state, never doc
// state. The clip box reads it to swap the loading pulse for a still
// placeholder; each new enrich attempt (imports fire one, and every project
// open sweeps all assets) clears the mark before trying again.
const failedStrips = new Set<string>();
const stripListeners = new Set<() => void>();
export function subscribeStripStatus(fn: () => void) {
stripListeners.add(fn);
return () => {
stripListeners.delete(fn);
};
}
/** Whether the last filmstrip attempt for this asset failed. */
export const stripFailedFor = (assetId: string) => failedStrips.has(assetId);
function setStripFailed(assetId: string, failed: boolean) {
if (failed === failedStrips.has(assetId)) return;
if (failed) failedStrips.add(assetId);
else failedStrips.delete(assetId);
for (const fn of stripListeners) fn();
}
// Assets whose cloud file is not there, by id — session state, like a failed
// strip. One read finds the gap and the clip says so; nothing reads the file
// again until the page reloads or an upload of it starts. The bytes of an
// upload that never finished live only in the browser that started it, so
// every other device, and a share of the project, finds nothing.
const missingMedia = new Set<string>();
/** Whether this asset's file was found missing from the cloud this session. */
export const missingMediaFor = (assetId: string) => missingMedia.has(assetId);
function setMediaMissing(assetId: string, missing: boolean) {
if (missing === missingMedia.has(assetId)) return;
if (missing) missingMedia.add(assetId);
else missingMedia.delete(assetId);
for (const fn of stripListeners) fn();
}
/** Whether to skip reading this asset's file: it was found missing, and no
* upload of it has started since. An upload clears the mark. */
function knownMissing(asset: MediaAsset): boolean {
if (asset.upload) setMediaMissing(asset.id, false);
return missingMedia.has(asset.id);
}
// A cloud route answers this when the request carries no session.
const UNAUTHORIZED = 401;
// The firewall answers this to a burst of requests from one address; the
// enrich retry reads again after the window passes.
const TOO_MANY_REQUESTS = 429;
/**
* Settle a failed read of an asset's file. A file that is not there marks the
* asset missing, unless an upload of it is running and the bytes are on their
* way. A read refused for want of a session belongs to the session gate, which
* hears the same refusal (chunkCache.ts). Neither is a fault to report; every
* other failure is reported against `what`, the step that failed.
*/
function readFailed(asset: MediaAsset, what: string, err: unknown, subject = asset.fileName) {
const live = useEditor.getState().assets.find((a) => a.id === asset.id) ?? asset;
if (cloudMediaGone(err)) {
if (!live.upload) setMediaMissing(asset.id, true);
console.warn(`${what} for ${subject}`, err);
return;
}
const status = cloudMediaStatus(err);
if (status === UNAUTHORIZED || status === TOO_MANY_REQUESTS) {
console.warn(`${what} for ${subject}`, err);
return;
}
reportSwallowed(what, err, subject);
}
// The DOMException names WebCodecs gives a decoder that refused its input: a
// config it cannot open, or packets it failed on.
const DECODER_REFUSALS = new Set(["EncodingError", "NotSupportedError"]);
/** Whether `err` is this page's decoders refusing a stream: a track no decoder
* here opens (HEVC in Firefox), or a decoder failing on the packets it was
* handed (a malformed MP3 track in a screen recording). */
function decodeRefused(err: unknown): boolean {
if (err instanceof DOMException) return DECODER_REFUSALS.has(err.name);
if (err instanceof UnreadableMediaError) return err.undecodable;
return err instanceof UserError && err.code === "media_no_readable_video";
}
/**
* Hand a stream this page failed to decode to a compat proxy, which a
* converter writes in codecs every browser decodes and whose landing enriches
* the asset again. True when the failure went that way; false when it is the
* file's own: the stream already reads from a compat proxy, or failed here
* before.
*/
function coverRefused(asset: MediaAsset, cover: CompatCover, err: unknown): boolean {
if (frameSinkIsCustom() || !decodeRefused(err)) return false;
const live = liveAsset(asset.id) ?? asset;
if (live.proxy && compatCovers(live.proxy.fileName, cover)) return false;
if (!noteUndecoded(asset.id, cover)) return false;
// An upload still running asks for its proxy when it lands (importQueue.ts).
const projectId = useEditor.getState().projectId;
if (projectId && !live.upload) void ensureProxy(projectId, live);
return true;
}
// A failed enrich books itself more tries. Filmstrip and waveform reads drop
// out together with the preview's when media reads fail, and the sweep that
// would try again runs only on project open — so a blip at open used to leave
// every clip blank for the whole session. The tries are few and well spaced;
// an asset that gains its thumbs or peaks stops the clock, and one that keeps
// failing goes quiet after the last try.
const ENRICH_RETRY_BASE_MS = 30_000;
const ENRICH_RETRIES = 3;
const enrichRetries = new Map<string, number>();
const enrichRetryTimers = new Map<string, ReturnType<typeof setTimeout>>();
function scheduleEnrichRetry(assetId: string) {
if (typeof window === "undefined") return;
if (enrichRetryTimers.has(assetId)) return;
const n = enrichRetries.get(assetId) ?? 0;
if (n >= ENRICH_RETRIES) return;
enrichRetries.set(assetId, n + 1);
const timer = setTimeout(() => {
enrichRetryTimers.delete(assetId);
const asset = useEditor.getState().assets.find((a) => a.id === assetId);
if (!asset) return;
const done =
asset.type === "audio"
? !!asset.peaks?.length
: asset.type === "video"
? stripComplete(asset) && asset.peaks !== undefined
: !!asset.thumbs?.length;
if (done) return;
void enrichAsset(asset);
}, ENRICH_RETRY_BASE_MS * 3 ** n);
enrichRetryTimers.set(assetId, timer);
}
/** A strip holds one tile per slot the grid asks for. A sweep that died
* midway published fewer, so the tile count is what tells a finished strip
* from a partial one — a non-empty strip is not necessarily a whole strip. */
function stripComplete(asset: MediaAsset) {
return (asset.thumbs?.length ?? 0) >= thumbGrid(asset.duration).count;
}
/** One enrich per asset at a time. A reload re-sweeps every asset, and a
* second pass landing mid-sweep would read the partial strip as the finished
* one and refine against it. */
const enrichInFlight = new Set<string>();
/**
* Bytes an asset was just made from, held until its waveform is read.
*
* Peaks are a decode of the whole file and they run after the asset lands, so
* the file that was in hand a moment ago would otherwise be read back out of
* storage — a second pass over local disk, and a download on the cloud. The
* entry is a reference to the file already in memory, taken by the first
* enrich and dropped there; a handful is kept in case none ever comes.
*/
const importedBytes = new Map<string, Blob>();
const IMPORTED_BYTES_CAP = 8;
export function keepImportedBytes(assetId: string, bytes: Blob): void {
importedBytes.set(assetId, bytes);
for (const id of importedBytes.keys()) {
if (importedBytes.size <= IMPORTED_BYTES_CAP) break;
importedBytes.delete(id);
}
}
/**
* Resolves once nothing is playing.
*
* The reads behind the editor — a waveform, a strip, a scene probe, a tile
* capture — open the same file the preview is playing from, and on a big
* file they take what the picture needs: decoder slots, the link's width, and
* the thread the frames are drawn on. A play is the urgent thing; this work
* waits for it to stop and picks up where it was. Cheap when nothing is
* playing, which is when it is awaited most.
*/
/** What a long read consults to know the preview wants the file back. */
export const IDLE_GATE: IdleGate = {
busy: () => useEditor.getState().playing,
idle: () => idlePlayback(),
};
const idleWaiters = new Set<() => void>();
let idleWatch: (() => void) | null = null;
/** Wake `resolve` when the cut stops playing; the handle stops the wait. One
* store subscription serves every waiter, since it runs on every write the
* editor makes and a play is when the writes are busiest. */
function waitForIdle(resolve: () => void): () => void {
idleWaiters.add(resolve);
idleWatch ??= useEditor.subscribe((s) => {
if (s.playing) return;
const woken = [...idleWaiters];
idleWaiters.clear();
idleWatch?.();
idleWatch = null;
for (const fn of woken) fn();
});
return () => {
idleWaiters.delete(resolve);
if (idleWaiters.size) return;
idleWatch?.();
idleWatch = null;
};
}
export function idlePlayback(): Promise<void> {
if (!useEditor.getState().playing) return Promise.resolve();
return new Promise((resolve) => void waitForIdle(resolve));
}
/** The same wait, with a handle for a caller that may stop wanting it — a
* wait left running holds its place in the set until the play ends. */
export function idleWait(): { done: Promise<void>; cancel: () => void } {
if (!useEditor.getState().playing) return { done: Promise.resolve(), cancel: () => {} };
let cancel = () => {};
const done = new Promise<void>((resolve) => {
cancel = waitForIdle(resolve);
});
return { done, cancel };
}
/**
* `framesAt`, given back while the cut is playing.
*
* A sweep that merely stops pulling still holds everything it opened: the
* file, the decoder, and the frames the decode pump counts in flight — the
* slot the preview was short of. So a play ends the walk outright, which
* disposes the reader with it, and the times still owed are read by a fresh
* walk once the play stops.
*/
async function* idleFramesAt(
url: string,
times: number[],
size?: { height: number }
): AsyncGenerator<WrappedCanvas | null> {
let from = 0;
while (from < times.length) {
await idlePlayback();
const before = from;
for await (const frame of baseFramesAt(url, times.slice(from), size)) {
yield frame;
from++;
if (useEditor.getState().playing) break;
}
// A pass that answered nothing has nothing more to give — an unreadable
// file, a walk that died — and another would open it forever.
if (from === before) return;
}
}
/** Assets whose file was rewritten this session after reading no frames:
* one try each, so a file no converter can fix never loops. */
const rewritten = new Set<string>();
/**
* A strip that read no frames from a stream this page decodes — one that
* opens on no keyframe, a broken reference chain — hands the file to the
* project's converter for a forced re-encode to clean H.264, once. The asset
* keeps its id and moves to the new file, which enriches again. False when
* the stream does not decode here at all, or was rewritten already.
*/
async function rewriteFrameless(asset: MediaAsset, src: string): Promise<boolean> {
const s = useEditor.getState();
const projectId = s.projectId;
if (!projectId || s.readOnly || asset.upload || rewritten.has(asset.id)) return false;
const decodes = await withMedia(src, async (input) => !!(await videoTrackOf(input))).catch(() => false);
if (!decodes) return false;
rewritten.add(asset.id);
void import("./mediaConvert")
.then(async (m) => {
const made = await m.convertAssetToMp4(projectId, liveAsset(asset.id) ?? asset, { mode: "reencode" });
// A converter that kept the file has nothing new to read.
if (made.unchanged) setStripFailed(asset.id, true);
})
.catch((err: unknown) => {
setStripFailed(asset.id, true);
readFailed(asset, "[cut] frameless rewrite failed", err);
});
return true;
}
/** Generate filmstrip thumbnails / waveform peaks and merge them into the
* store. Safe to call repeatedly; skips assets that are already enriched.
* `src` overrides where the frames are read from — an import still uploading
* has the bytes in the browser already, so it need not wait or re-download. */
export async function enrichAsset(asset: MediaAsset, src = asset.url) {
// A block paints itself at whatever size asks, so there is no file to probe
// and no still to stand in as its strip.
if (asset.block) return;
if (knownMissing(asset)) return;
if (enrichInFlight.has(asset.id)) return;
enrichInFlight.add(asset.id);
const held = importedBytes.get(asset.id);
importedBytes.delete(asset.id);
try {
if (asset.type === "image") {
// A still is its own filmstrip: one frame, tiled across the clip.
if (!asset.thumbs?.length) {
useEditor.getState().updateAsset(asset.id, { thumbs: [src], thumbStep: IMAGE_CLIP_SECONDS });
}
} else if (asset.type === "video") {
// A clip dropped and played at once is played first; the strip and the
// sound band fill in when the play stops.
const openProject = useEditor.getState().projectId;
await idlePlayback();
// A play is long enough to leave the project in it. Nothing this sweep
// writes belongs to whatever is open now.
if (useEditor.getState().projectId !== openProject) return;
// Footage imported before its color was read at import settles it now,
// from the header alone: the grade pipeline and the preview's decoder
// both read `color`, and a file without it would draw as Rec.709.
// The asset's live record decides: the import's own probe may have
// written it while this sweep waited, and that walk is shared.
asset = liveAsset(asset.id) ?? asset;
if (!asset.color && !asset.upload) {
const color = await probeMediaColor(held ?? src).catch(() => null);
if (useEditor.getState().projectId !== openProject) return;
if (color) {
await landProbedFacts(asset.id, { color: assetColor(color) });
asset = liveAsset(asset.id) ?? asset;
}
}
// A ProRes master previews through the WASM decoder until its proxy
// lands, a heavy video streamed from the cloud stalls at its cuts, and
// a stream this browser cannot decode reads from a compat proxy; the
// proxy is built behind the editor, once the bytes are at rest in the
// project's own storage.
if (openProject && !asset.upload) void ensureProxy(openProject, asset);
// A picture this browser cannot decode has nothing to sweep until its
// compat proxy lands; the landing enriches the asset again.
if (await awaitsCompat(asset, "video")) return;
// Footage stored without a length — a shelf row that was never measured
// — reads its own before anything is swept from it: a zero-length sweep
// has no tiles to place and no buckets to fill, and the clips cut from
// it open to the footage.
if (!(asset.duration > 0)) {
const probe = await probeMediaFile(held ?? src);
useEditor.getState().repairAssetDuration(asset.id, {
duration: probe.duration,
width: probe.width,
height: probe.height,
});
asset = liveAsset(asset.id) ?? asset;
if (!(asset.duration > 0)) throw new UnreadableMediaError("The video has no length.");
}
// The clip box draws the sound under the picture, so a video enriches
// waveform peaks alongside its filmstrip. The two are separate reads of
// the file, so the peaks decode starts here and runs beside the strip's
// sweep: the sound band lands the moment its own read is done, however
// long a source's frame grabs take. `[]` is an answer too — a silent
// file — and stops the strip from re-decoding every open.
const peaksRun =
asset.peaks === undefined && !(await awaitsCompat(asset, "audio"))
? loadPeaks(asset, held ?? src, (partial) =>
landDerived(asset.id, { peaks: partial })
)
.then((peaks) => {
if (peaks) landDerived(asset.id, { peaks });
})
.catch((err: unknown) => {
// The strip paints on its own; a peaks failure (an expired
// signed URL, an undecodable track) never marks it failed.
// What the walk had published comes off the asset, so the
// next open reads the file again in place of keeping a
// strip that stops partway.
landDerived(asset.id, { peaks: undefined });
if (coverRefused(asset, "audio", err)) return;
readFailed(asset, "[cut] peaks failed", err);
})
: undefined;
if (!stripComplete(asset)) {
setStripFailed(asset.id, false);
const key = stripCacheKey(useEditor.getState().projectId, asset.id);
const cached = await readCachedStrip(key, asset.duration);
if (cached) {
landDerived(asset.id, {
thumbs: cached.thumbs,
thumbStep: cached.thumbStep,
sceneCuts: cached.cuts,
});
} else {
// Midpoint grabs, published tile by tile: the strip fills in as the
// sweep runs, and the scene pass refines the grab points behind it.
const { thumbs, thumbStep } = await makeThumbs(src, asset.duration, [], (partial, step) =>
landDerived(asset.id, { thumbs: partial, thumbStep: step })
);
// A stream this page decodes that gave no frames has its file
// rewritten by a converter, which enriches the asset again.
if (!thumbs.length && (await rewriteFrameless(asset, src))) return;
// An empty strip is a failure, not a result: persisting it would
// leave the asset looking permanently mid-load.
if (!thumbs.length) throw new UnreadableMediaError("No frames could be read for the filmstrip.");
landDerived(asset.id, { thumbs, thumbStep });
writeCache(key, { thumbs, thumbStep, duration: asset.duration, at: Date.now() });
}
}
// The cuts a strip was built without — a fresh sweep, or a cache written
// before the scene pass finished — are probed once the tiles are up.
const live = liveAsset(asset.id);
if (live && stripComplete(live) && live.sceneCuts === undefined) {
void refineSceneCuts(asset.id, src, asset.duration).catch((err) => {
// Scene cuts only sharpen the strip; the tiles already painted.
readFailed(asset, "[cut] scene probe failed", err);
});
}
await peaksRun;
enrichRetries.delete(asset.id);
} else if (asset.type === "audio" && !asset.peaks?.length) {
const openProject = useEditor.getState().projectId;
// Sound this browser cannot decode reads from a compat proxy, built
// once the bytes are at rest; its landing enriches the asset again.
if (openProject && !asset.upload) void ensureProxy(openProject, asset);
if (await awaitsCompat(asset, "audio")) return;
await idlePlayback();
if (useEditor.getState().projectId !== openProject) return;
const publish = (peaks: number[]) => landDerived(asset.id, { peaks });
try {
const peaks = await loadPeaks(asset, held ?? src, publish);
if (peaks) publish(peaks);
} catch (err) {
// A walk that published partway comes off the asset before the retry
// below, so the retry reads the file again.
landDerived(asset.id, { peaks: undefined });
throw err;
}
enrichRetries.delete(asset.id);
}
} catch (err) {
// A stream this page cannot decode reads from a compat proxy, whose
// landing enriches the asset again.
if ((asset.type === "video" || asset.type === "audio") && coverRefused(asset, asset.type, err)) return;
// Thumbnails and waveforms are decorative; editing works without them.
// The failure still gets recorded — clips stop their loading pulse — and
// reported, which carries it into PostHog error tracking with the file's
// shape beside the cause (lib/report.ts).
if (asset.type === "video") setStripFailed(asset.id, true);
readFailed(asset, "[cut] enrich failed", err, `${asset.fileName} (${asset.type}, ${Math.round(asset.duration)}s)`);
// A missing file stays missing; trying again would ask for the same nothing.
if (asset.type !== "image" && !missingMedia.has(asset.id)) scheduleEnrichRetry(asset.id);
} finally {
enrichInFlight.delete(asset.id);
}
}
// Beat scans in flight, by asset id — session state. The clip menu reads it
// to show the scan running, and a second ask joins the first's promise.
const beatScans = new Map<string, Promise<AssetBeats>>();
// Sources a scan has already read and found no steady pulse in. Nothing lands
// on the asset in that case — an empty grid is no grid — so without this a
// caller that only wants the stored grid would pay for the whole scan again
// every time it asked about a speech recording.
const beatlessScans = new Set<string>();
const beatListeners = new Set<() => void>();
export function subscribeBeatsBusy(fn: () => void) {
beatListeners.add(fn);
return () => {
beatListeners.delete(fn);
};
}
/** Whether a beat scan is running for this asset. */
export const beatsBusyFor = (assetId: string) => beatScans.has(assetId);
/** Whether a scan has already read this source and heard no pulse. A caller
* reading the stored grid uses it to tell "never scanned" from "scanned, no
* beats" — the second needs no second scan. A re-scan clears it. */
export const beatlessFor = (assetId: string) => beatlessScans.has(assetId);
/**
* Detect the source's beat grid and write it onto the asset (types.ts
* AssetBeats): its clips draw the beats as dots and every drag or trim snaps
* to them. Replaces whatever grid was there — detection is the starting
* point, the dots are the user's to edit from there. A scan that hears no
* steady pulse clears the grid and returns the empty scan.
*/
export async function detectAssetBeats(asset: MediaAsset): Promise<AssetBeats> {
const running = beatScans.get(asset.id);
if (running) return running;
const scan = (async () => {
const { beats, bpm } = await scanSourceBeats(asset.url);
const grid = { beats, bpm };
useEditor.getState().updateAsset(asset.id, { beats: beats.length ? grid : undefined });
if (beats.length) beatlessScans.delete(asset.id);
else beatlessScans.add(asset.id);
return grid;
})();
beatScans.set(asset.id, scan);
for (const fn of beatListeners) fn();
try {
return await scan;
} finally {
beatScans.delete(asset.id);
for (const fn of beatListeners) fn();
}
}
/** Waveform peaks on demand — e.g. when a video clip's audio is detached
* onto the soundtrack track (video assets don't get peaks at import). */
export async function ensurePeaks(asset: MediaAsset) {
if (asset.peaks?.length) return;
if (knownMissing(asset)) return;
const publish = (peaks: number[]) => landDerived(asset.id, { peaks });
try {
const peaks = await loadPeaks(asset, asset.url, publish);
if (peaks?.length) publish(peaks);
} catch (err) {
// Waveforms are decorative; editing works without them. A walk that
// published partway comes off the asset, so the next read starts clean.
landDerived(asset.id, { peaks: undefined });
if (coverRefused(asset, "audio", err)) return;
readFailed(asset, "[cut] peaks failed", err);
}
}
// Filmstrip frames render at 60 CSS px tall — capture at 3× so they stay sharp
// on Retina (and when a tall timeline scales the row up).
const THUMB_H = 180;
// Filmstrips persist in IndexedDB keyed by project + file name + thumbnail
// geometry — not the URL, which churns on cloud signed-URL refreshes — so
// reopening a project paints clips from cache instead of re-seeking every
// video. Cache failures fall through to regeneration.
const STRIP_DB = "cut-filmstrips";
const STRIP_STORE = "strips";
const STRIP_CAP = 500; // prune oldest beyond this many cached strips
type CachedStrip = {
thumbs: string[];
thumbStep: number;
cuts?: number[];
duration: number;
at: number;
};
// Cache entries are keyed by asset id: each import is its own asset, so a
// file re-imported under a name the project has seen before starts a fresh
// strip, and reopening the project still finds its assets' entries.
function stripCacheKey(projectId: string | null, assetId: string) {
// "s3" marks strips that carry scene cuts; strips cached before them regenerate.
return `${projectId ?? ""}/${assetId}@${THUMB_H}s3`;
}
type CachedPeaks = { peaks: number[]; duration: number; at: number };
function peaksCacheKey(projectId: string | null, assetId: string) {
// The density marker versions the cache; peaks sampled differently regenerate.
return `${projectId ?? ""}/${assetId}@peaks${PEAKS_PER_SECOND}`;
}
function openStripDb(): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
const req = indexedDB.open(STRIP_DB, 1);
req.onupgradeneeded = () => {
const store = req.result.createObjectStore(STRIP_STORE);
store.createIndex("at", "at");
};
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
}
async function readCachedStrip(key: string, duration: number): Promise<CachedStrip | null> {
try {
const db = await openStripDb();
const strip = await new Promise<CachedStrip | undefined>((resolve, reject) => {
const req = db.transaction(STRIP_STORE).objectStore(STRIP_STORE).get(key);
req.onsuccess = () => resolve(req.result as CachedStrip | undefined);
req.onerror = () => reject(req.error);
});
db.close();
// A same-path file with a different duration was rewritten; regenerate.
if (!strip?.thumbs?.length || Math.abs(strip.duration - duration) > 0.25) return null;
return strip;
} catch {
return null;
}
}
async function readCachedPeaks(key: string, duration: number): Promise<number[] | null> {
try {
const db = await openStripDb();
const entry = await new Promise<CachedPeaks | undefined>((resolve, reject) => {
const req = db.transaction(STRIP_STORE).objectStore(STRIP_STORE).get(key);
req.onsuccess = () => resolve(req.result as CachedPeaks | undefined);
req.onerror = () => reject(req.error);
});
db.close();
// A same-path file with a different duration was rewritten; regenerate.
if (!Array.isArray(entry?.peaks) || Math.abs(entry.duration - duration) > 0.25) return null;
return entry.peaks;
} catch {
return null;
}
}
/** Waveform peaks for an asset, from the cache when this file was already
* decoded — a real audio decode otherwise. Empty peaks are an answer (a file
* with no audible track) and cache like any other. A headless runtime returns
* undefined so the browser decodes on its next open. */
async function loadPeaks(
asset: MediaAsset,
src: string | Blob = asset.url,
onPartial?: (peaks: number[]) => void
): Promise<number[] | undefined> {
if (typeof AudioDecoder === "undefined") return undefined;
const key = peaksCacheKey(useEditor.getState().projectId, asset.id);
const cached = await readCachedPeaks(key, asset.duration);
if (cached) return cached;
const peaks = await makePeaks(src, asset.duration, onPartial, IDLE_GATE);
writeCache(key, { peaks, duration: asset.duration, at: Date.now() });
return peaks;
}
function writeCache(key: string, entry: CachedStrip | CachedPeaks) {
void (async () => {
try {
const db = await openStripDb();
const tx = db.transaction(STRIP_STORE, "readwrite");
const store = tx.objectStore(STRIP_STORE);
store.put(entry, key);
const count = await new Promise<number>((resolve, reject) => {
const req = store.count();
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
if (count > STRIP_CAP) {
const cursorReq = store.index("at").openCursor();
let toDrop = count - STRIP_CAP;
cursorReq.onsuccess = () => {
const cursor = cursorReq.result;
if (!cursor || toDrop <= 0) return;
cursor.delete();
toDrop--;
cursor.continue();
};
}
tx.oncomplete = () => db.close();
tx.onerror = () => db.close();
} catch {
// Cache writes are best-effort; the strip is already on screen.
}
})();
}
/** Probes per strip bucket for scene-change detection, and the cap on the
* whole probe sweep so long files stay a bounded pass. */
const THUMB_PROBE_CAP = 120;
/** % of 16-grid cells that must change between adjacent probes to call a
* scene cut. Far above the watch selector's "meaningfully new" bar (8%), so
* motion and pans pass under it and only a real cut splits a bucket. */
const THUMB_PROBE_CUT_PCT = 45;
/** g16 grid signatures for ascending `times`, decoded small in one sweep;
* null where the decoder had no frame. */
async function g16Sweep(url: string, times: number[]): Promise<(Float32Array | null)[]> {
const sig = createRasterCanvas(SIGNATURE_SIZE, SIGNATURE_SIZE);
const ctx = sig.getContext("2d", {
willReadFrequently: true,
}) as CanvasRenderingContext2D | null;
if (!ctx) return times.map(() => null);
const out: (Float32Array | null)[] = [];
for await (const frame of idleFramesAt(url, times, { height: SIGNATURE_SIZE })) {
if (!frame) {
out.push(null);
continue;
}
ctx.drawImage(frame.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
out.push(
frameSig({
width: SIGNATURE_SIZE,
height: SIGNATURE_SIZE,
channels: 4,
data: ctx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,
}).g16
);
}
return out;
}
/** Scene-change signals across the clip: one small decoded frame every
* fraction of a bucket, compared through the watch pipeline's grid signature.
* A probe is flagged `cut` when the picture changed scene since the probe
* before it. */
async function thumbProbes(url: string, duration: number, count: number): Promise<ThumbProbe[]> {
const perBucket = Math.max(3, Math.floor(THUMB_PROBE_CAP / count));
const n = count * perBucket;
const times = Array.from({ length: n }, (_, i) =>
Math.max(0, Math.min(duration - 0.05, ((i + 0.5) * duration) / n))
);
const sigs = await g16Sweep(url, times);
return times.map((t, i) => {
const p = i > 0 ? sigs[i - 1] : null;
const q = sigs[i];
return {
t,
cut: !!p && !!q && diffCellPct(q, p, DEDUP_TUNING.GLOBAL_TOL) > THUMB_PROBE_CUT_PCT,
};
});
}
/** How many cut brackets get the dense second sweep, samples per bracket, and
* the most cuts an asset stores at all. */
const CUT_REFINE_MAX = 32;
const CUT_REFINE_STEPS = 8;
const SCENE_CUTS_MAX = 256;
/** The clip's scene-change moments. The probe pass brackets each cut between
* two probes; a denser sweep inside each bracket places the cut within an
* eighth of the probe spacing (tens of milliseconds on a typical clip).
* Brackets past the refinement cap keep their bracket midpoint. */
async function sceneCutTimes(url: string, probes: ThumbProbe[]): Promise<number[]> {
const brackets: [number, number][] = [];
for (let i = 1; i < probes.length; i++) {
if (probes[i].cut) brackets.push([probes[i - 1].t, probes[i].t]);
}
const refine = brackets.slice(0, CUT_REFINE_MAX);
const cuts = brackets.slice(CUT_REFINE_MAX).map(([a, b]) => (a + b) / 2);
if (refine.length) {
const S = CUT_REFINE_STEPS;
const times = refine.flatMap(([a, b]) =>
Array.from({ length: S + 1 }, (_, j) => a + (j * (b - a)) / S)
);
const sigs = await g16Sweep(url, times);
refine.forEach(([a, b], r) => {
let at = (a + b) / 2;
let strongest = -1;
for (let j = 1; j <= S; j++) {
const p = sigs[r * (S + 1) + j - 1];
const q = sigs[r * (S + 1) + j];
if (!p || !q) continue;
const d = diffCellPct(q, p, DEDUP_TUNING.GLOBAL_TOL);
if (d > strongest) {
strongest = d;
at = a + ((j - 0.5) * (b - a)) / S;
}
}
cuts.push(at);
});
}
return cuts
.map((c) => Math.round(c * 100) / 100)
.sort((x, y) => x - y)
.slice(0, SCENE_CUTS_MAX);
}
/** The strip's bucket count and spacing: one frame every ~2s (min 10, max 24)
* so long clips don't repeat frames. */
function thumbGrid(duration: number) {
const count = Math.min(24, Math.max(10, Math.round(duration / 2)));
return { count, thumbStep: duration / count };
}
/** How often a sweep in progress hands the tiles it has over to the strip. */
const THUMB_PUBLISH_MS = 250;
/**
* Sweep a clip's filmstrip frames, handing over the tiles that have landed as
* the pass runs.
*
* Ascending times over one decode pass — the strip is a single sweep of the
* file rather than `count` seeks into it — and each tile is on screen as soon
* as it is read. A long clip streaming in from storage takes real time to
* sweep, and a strip that showed nothing until the last tile landed read as
* broken next to a preview that plays immediately.
*
* The strip is read back by position (`thumbs[floor(t / thumbStep)]`), so a
* time the decoder has no frame for cannot simply be dropped: that would
* slide every later tile onto the wrong moment for the rest of the clip, and
* the wrong strip would be cached. A gap repeats the frame before it, which
* keeps every index meaning what it says — and the same fill carries the tail
* of a strip that is still being swept.
*/
async function makeThumbs(
url: string,
duration: number,
probes: ThumbProbe[] = [],
onTiles?: (thumbs: string[], thumbStep: number) => void
) {
const { count, thumbStep } = thumbGrid(duration);
const times = pickThumbTimes(count, thumbStep, duration, probes);
const captured: (string | null)[] = [];
// Fill gaps from the frame before, so the strip is either populated to the
// tile the sweep has reached or empty.
const settled = () => {
const thumbs: string[] = [];
let fill: string | null = captured.find((c) => c !== null) ?? null;
if (fill) {
for (const shot of captured) {
if (shot) fill = shot;
thumbs.push(fill);
}
}
return thumbs;
};
let published = 0;
for await (const frame of idleFramesAt(url, times, { height: THUMB_H })) {
captured.push(frame ? await rasterCanvasToDataUrl(frame.canvas, "image/jpeg", 0.92) : null);
if (!onTiles || captured.length >= times.length) continue;
const now = Date.now();
if (now - published < THUMB_PUBLISH_MS) continue;
published = now;
const partial = settled();
if (partial.length) onTiles(partial, thumbStep);
}
return { thumbs: settled(), thumbStep };
}
/**
* The clip's scene-change moments, and the probe signals behind them.
*
* This is the expensive half of building a strip: the probe sweep and its
* refinement decode many times the frames the strip itself needs, so it runs
* behind the strip rather than in front of it.
*/
async function sceneProbe(url: string, duration: number) {
const { count } = thumbGrid(duration);
const probes = await thumbProbes(url, duration, count);
const cuts = await sceneCutTimes(url, probes);
return { probes, cuts };
}
/** A grab point has to move by this share of its bucket before the tile is
* worth reading again. */
const THUMB_REGRAB_SHARE = 0.1;
/** One scene probe per asset at a time. The probe outlives the enrich that
* launched it, so the enrich guard cannot cover it: the next pass would see a
* complete strip with no cuts yet and start the same sweep over again. */
const sceneProbeInFlight = new Set<string>();
/**
* Scene-aware refinement, once the strip is already on screen: probe the clip
* for its cuts, store them, and re-read only the tiles whose grab point moved
* far enough that the midpoint was showing a neighbouring scene.
*/
async function refineSceneCuts(assetId: string, url: string, duration: number) {
if (sceneProbeInFlight.has(assetId)) return;
sceneProbeInFlight.add(assetId);
try {
const live = () => liveAsset(assetId);
const { probes, cuts } = await sceneProbe(url, duration);
if (!live()) return;
landDerived(assetId, { sceneCuts: cuts });
const { count, thumbStep } = thumbGrid(duration);
// The cuts land in the cache however the tiles turn out. A sweep that
// moves no grab point still answered the expensive question, and a cache
// written without the answer sends every later open through the whole
// probe again.
const persist = (thumbs: string[]) =>
writeCache(stripCacheKey(useEditor.getState().projectId, assetId), {
thumbs,
thumbStep,
cuts,
duration,
at: Date.now(),
});
const midpoints = pickThumbTimes(count, thumbStep, duration, []);
const scened = pickThumbTimes(count, thumbStep, duration, probes);
const moved = scened
.map((t, i) => ({ t, i }))
.filter(({ t, i }) => Math.abs(t - midpoints[i]) > thumbStep * THUMB_REGRAB_SHARE);
const asset = live();
if (!asset?.thumbs?.length) return;
if (!moved.length) {
persist(asset.thumbs);
return;
}
const regrabbed = new Map<number, string>();
let k = 0;
for await (const frame of idleFramesAt(
url,
moved.map((m) => m.t),
{ height: THUMB_H }
)) {
const at = moved[k++];
if (!frame || !at) continue;
regrabbed.set(at.i, await rasterCanvasToDataUrl(frame.canvas, "image/jpeg", 0.92));
}
// The sweep above took a while; the strip on screen is the one to patch.
const current = live();
if (!current?.thumbs?.length) return;
const thumbs = [...current.thumbs];
for (const [i, tile] of regrabbed) if (i < thumbs.length) thumbs[i] = tile;
landDerived(assetId, { thumbs });
persist(thumbs);
} finally {
sceneProbeInFlight.delete(assetId);
}
}
// Exact edge frames: a clip's first and last filmstrip tiles show the true
// frames at its in/out points. Asset thumbs are fixed-interval midpoint
// samples, so edges are captured on demand at the precise source time. Each
// clip edge is a "slot" whose newest request supersedes queued ones (trim
// drags stay cheap); one serial loop reads frames from a small pool of
// per-URL readers.
//
// The pool is what makes a trim drag cheap. A reader holds a warm decoder and
// its cache of recently decoded frames, so the frames either side of where the
// handle already is come back without re-reading the file — which is the whole
// difference between this and re-opening the source per request.
const EDGE_CACHE_CAP = 300;
/** Canvases one strip sink cycles, and strip heights a reader keeps sinks
* for — a zoom in and the height it came from, so a wobble between two levels
* rebuilds neither. */
const EDGE_SINK_POOL = 4;
const EDGE_SINK_HEIGHTS = 2;
/**
* What the timeline's own pictures cost.
*
* The captured frames are JPEG data URLs — text, two bytes a character in the
* heap — and each open reader cycles a pool of canvases per strip height it
* has been asked for. Both are held for as long as the timeline is on screen,
* so a report of what the tab holds counts them. The readers are counted from
* the sinks they actually opened: a reader asked for two heights holds two
* pools, and at 16:9 a pool's canvas is about twice its height across.
*/
const stripBytes = (): number => {
let n = 0;
for (const url of edgeCache.values()) n += url.length * 2;
for (const open of openEdgeReaders)
for (const h of open.sinks.keys()) n += canvasBytes(EDGE_SINK_POOL * h * h * 2);
return n;
};
/** Readers whose sinks have opened, for the count above. A reader still
* opening holds nothing yet. */
const openEdgeReaders = new Set<{ sinks: Map<number, unknown> }>();
/**
* What the derived pictures on the assets themselves cost.
*
* A strip's thumbnails and a waveform's peaks are read once and then live on
* the asset for the session: a couple of dozen JPEG data URLs and up to sixty
* thousand samples apiece, which on a project of a hundred files is more than
* the decoders are allowed between them. Both have a disk cache behind them,
* so this is memory standing in front of storage.
*/
const derivedBytes = (): number => {
let n = 0;
for (const a of useEditor.getState().assets) {
for (const t of a.thumbs ?? []) n += t.length * 2;
n += (a.peaks?.length ?? 0) * 8;
}
return n;
};
/**
* Give back the strips and waveforms of the files nothing in the cut is using.
*
* A library entry that was imported and never placed still carries everything
* the timeline drew for it once. Every one of those reads back from its own
* IndexedDB cache the moment a tile asks again, so holding them is a choice
* about latency; under memory pressure it is the wrong one. Assets a clip
* points at keep what they have, so nothing on the timeline ever redraws.
*/
/** What the strips and waveforms of unplaced library entries cost — the part
* of the total that a release could actually give back. */
function unusedDerivedBytes(): number {
const s = useEditor.getState();
const used = assetIdsInUse(s);
let n = 0;
for (const a of s.assets) {
if (used.has(a.id)) continue;
for (const t of a.thumbs ?? []) n += t.length * 2;
n += (a.peaks?.length ?? 0) * 8;
}
return n;
}
function releaseUnusedDerived(): void {
const s = useEditor.getState();
const used = assetIdsInUse(s);
for (const a of s.assets) {
if (used.has(a.id) || (!a.thumbs?.length && !a.peaks?.length)) continue;
s.updateAsset(a.id, { thumbs: undefined, peaks: undefined });
}
}
/** Derived pictures the assets may carry between them. */
const DERIVED_MAX = 128 * 2 ** 20;
/** Least time between two sweeps. A capture lands many times a second while a
* strip fills, and the answer cannot have changed between two of them. */
const DERIVED_SWEEP_MS = 5_000;
let derivedSweptAt = 0;
/**
* Give the unplaced library entries' pictures back, at most now and then.
*
* The test is what a release would recover, so a project whose placed clips
* are themselves over the budget asks once and finds nothing, in place of
* sweeping every asset on every captured frame and stripping entries a tile
* has just read back.
*/
function sweepDerived(): void {
const now = Date.now();
if (now - derivedSweptAt < DERIVED_SWEEP_MS) return;
derivedSweptAt = now;
const cap = allowance("libraryPictures", DERIVED_MAX);
if (derivedBytes() > cap && unusedDerivedBytes() > 0) releaseUnusedDerived();
}
const EDGE_POOL_CAP = 4;
type EdgeRequest = {
url: string;
time: number;
height: number;
key: string;
resolvers: ((src: string | null) => void)[];
};
/** A file held open for repeated frame reads, with a decoding sink per capture
* height — trim edges and tile previews read different sizes from the same
* warm reader. */
type EdgeReader = {
reader: MediaHandle;
sinkFor: (height: number) => ReturnType<typeof frameSink>;
sinks: Map<number, ReturnType<typeof frameSink>>;
};
const edgeCache = new Map<string, string>();
const edgeQueue = new Map<string, EdgeRequest>();
/** Filmstrip tile captures wait here, behind every queued clip-edge capture,
* so a trim drag's frames land first while a zoomed strip fills in behind. */
const edgeBackQueue = new Map<string, EdgeRequest>();
const edgePool = new Map<string, Promise<EdgeReader>>();
holdMemory("edgeFrames", stripBytes);
holdMemory("libraryPictures", derivedBytes);
let edgePumping = false;
/** Wakes a pump holding the strip's captures for a play, when a clip-edge
* capture arrives that should not wait for it. */
let edgeWake: (() => void) | null = null;
function edgeKey(url: string, time: number, height: number) {
return `${url}#${time.toFixed(2)}@${height}`;
}
/** Synchronous cache read — the frame if a matching capture already landed. */
export function peekEdgeFrame(url: string, time: number, height = THUMB_H): string | null {
return edgeCache.get(edgeKey(url, time, height)) ?? null;
}
/** Capture the frame at `time`, latest-wins per `slot` (a clip edge or a
* strip tile). Resolves with the frame, or null when superseded by a newer
* request or on a failed read. `background` requests wait behind every queued
* foreground one. */
export function requestEdgeFrame(
slot: string,
url: string,
time: number,
height = THUMB_H,
opts?: { background?: boolean }
): Promise<string | null> {
const key = edgeKey(url, time, height);
const hit = edgeCache.get(key);
if (hit) return Promise.resolve(hit);
const queue = opts?.background ? edgeBackQueue : edgeQueue;
return new Promise((resolve) => {
const prev = queue.get(slot);
if (prev?.key === key) {
prev.resolvers.push(resolve);
} else {
prev?.resolvers.forEach((r) => r(null));
queue.set(slot, { url, time, height, key, resolvers: [resolve] });
}
if (!opts?.background) edgeWake?.();
void pumpEdgeFrames();
});
}
/** Let go of every queued capture whose slot starts with `prefix`: the
* surface that asked is gone, and a 4K keyframe decoded for a ghost that has
* already landed is a decoder taken from the clip it became. A capture
* mid-read finishes; only the queue empties. */
export function dropEdgeFrames(prefix: string): void {
for (const queue of [edgeQueue, edgeBackQueue]) {
for (const [slot, req] of queue) {
if (!slot.startsWith(prefix)) continue;
queue.delete(slot);
req.resolvers.forEach((r) => r(null));
}
}
}
/** Captures still queued or mid-read. The filmstrip eval waits on zero to
* know a zoomed strip has finished filling in. */
export function edgeFramesPending(): number {
return edgeQueue.size + edgeBackQueue.size + (edgePumping ? 1 : 0);
}
function edgeReader(url: string): Promise<EdgeReader> {
const hit = edgePool.get(url);
if (hit) {
// Re-insert to refresh recency; the pool evicts oldest-first.
edgePool.delete(url);
edgePool.set(url, hit);
return hit;
}
const opening = (async () => {
const handle = openMediaShared(url);
const track = await videoTrackOf(handle.input);
if (!track) {
handle.release();
throw new UnreadableMediaError("This file has no readable video.");
}
// A pool of canvases each sink cycles through, so a drag that reads
// hundreds of frames keeps its allocation flat.
// One sink per strip height, because a zoomed timeline asks for a
// different one. Only the heights still on screen are worth keeping: each
// holds its own pool of canvases, and a drag through a dozen zoom levels
// would otherwise leave a dozen pools behind it.
const sinks = new Map<number, ReturnType<typeof frameSink>>();
const reading = colorReadAt(url);
const sinkFor = (height: number) => {
let sink = sinks.get(height);
if (sink) {
sinks.delete(height);
sinks.set(height, sink);
return sink;
}
sink = baseFrameSink(track, { height }, reading, { poolSize: EDGE_SINK_POOL, lowLatency: true });
sinks.set(height, sink);
while (sinks.size > EDGE_SINK_HEIGHTS) sinks.delete(sinks.keys().next().value!);
return sink;
};
const reader = { reader: handle, sinkFor, sinks };
openEdgeReaders.add(reader);
return reader;
})();
edgePool.set(url, opening);
while (edgePool.size > EDGE_POOL_CAP) {
const [oldUrl, old] = edgePool.entries().next().value!;
edgePool.delete(oldUrl);
old
.then((r) => {
openEdgeReaders.delete(r);
r.reader.release();
})
.catch(() => {});
}
return opening;
}
async function pumpEdgeFrames() {
if (edgePumping) return;
edgePumping = true;
try {
for (;;) {
// Strip tiles read behind everything: behind every clip-edge capture,
// and behind a play. The timeline follows the playhead while it plays,
// and every scroll asks for the tiles that came into view — captures
// that would open a second decoder on the file being played and read
// its keyframes beside the walk. They land once the play stops. An
// edge capture arriving meanwhile wakes the pump for itself.
if (edgeQueue.size === 0 && edgeBackQueue.size > 0) {
const idle = idleWait();
await Promise.race([
idle.done,
new Promise<void>((resolve) => {
edgeWake = resolve;
}),
]);
// Whichever won, the other is no longer wanted: a wait left in the
// set outlives every wake of a long play.
idle.cancel();
edgeWake = null;
}
// A clip-edge capture waits out a play too. The edges of the clips the
// scroll brings into view open each clip's file while the play is
// opening the same clips a moment later, and on a fast cut those reads
// took the link from the picture and stalled it. A paused timeline
// still gets its edges ahead of the strip tiles.
if (edgeQueue.size > 0) await idleWait().done;
const queue = edgeQueue.size ? edgeQueue : edgeBackQueue;
const next = queue.entries().next();
if (next.done) break;
const [slot, req] = next.value;
queue.delete(slot);
let src = edgeCache.get(req.key) ?? null;
if (!src) {
try {
const { sinkFor } = await edgeReader(req.url);
const frame = await sinkFor(req.height).getCanvas(Math.max(0, req.time));
if (!frame) throw new Error("No frame at that time.");
src = await rasterCanvasToDataUrl(frame.canvas, "image/jpeg", 0.92);
edgeCache.set(req.key, src);
// Oldest first, by count and by what they cost. A count on its own
// says nothing about the size of what is counted: three hundred
// frames of a phone video are a different number of megabytes from
// three hundred of a screen recording.
const cap = allowance(
"edgeFrames",
canvasBytes(EDGE_CACHE_CAP * THUMB_H * THUMB_H * 2)
);
let bytes = stripBytes();
while (edgeCache.size > EDGE_CACHE_CAP || (edgeCache.size > 1 && bytes > cap)) {
const oldest = edgeCache.keys().next().value!;
bytes -= (edgeCache.get(oldest)?.length ?? 0) * 2;
edgeCache.delete(oldest);
}
// The strips and waveforms sitting on unused library entries are the
// other half of this bucket, and they come back off disk.
sweepDerived();
} catch {
src = null;
}
}
req.resolvers.forEach((r) => r(src));
}
} finally {
edgePumping = false;
}
}
/** Waveform buckets per second of media. At the deepest zoom (800 px/s) a
* bucket spans 4px — about one drawn bar — so the strip stays honest at every
* zoom. */
const PEAKS_PER_SECOND = 200;
/** Most buckets one source may hold. The array stays resident for the whole
* session — on the asset and in the cache entry beside it — so length is what
* bounds the memory a project of long sources costs. Anything up to five
* minutes keeps the full density; past that the strip folds more of the
* source into each bar the further in it is zoomed. */
const PEAKS_MAX = 60_000;
function makePeaks(
src: string | Blob,
duration: number,
onPartial?: (peaks: number[]) => void,
gate?: IdleGate
): Promise<number[]> {
const buckets =
duration > 0
? Math.min(PEAKS_MAX, Math.max(400, Math.round(duration * PEAKS_PER_SECOND)))
: 1600;
return audioPeaks(src, buckets, onPartial, gate);
}