Open source CapCut alternative.
TypeScript
"use client";
import { isCatalogRef, refFromAsset, refFromTextFile, type AssetRef } from "./assetRef";
import { tagChatAsset } from "./chatAssets";
import { getBackend } from "./backend";
import { checkImportRoom } from "./importRoom";
import { startUpload } from "./importQueue";
import {
enrichAsset,
importFileToProject,
isMediaFile,
isTextFile,
prepareImport,
renderAudioSpanWav,
} from "./media";
import { frameSink, openMedia, videoTrackOf } from "./mediaRead";
import { reportSwallowed } from "./report";
import { createRasterCanvas, rasterCanvasToDataUrl } from "./raster";
import { useEditor } from "./store";
import { formatTime } from "./time";
import { UserError } from "./userError";
// Turn refs and dropped files into what the hosted models take. Generation
// routes read `inputs.images = [{ data, mimeType }]`: stock images upload
// as-is; video refs (project clips, library media, generated stills) upload a
// single captured frame — the browser reads the same media URLs the previews
// use. Responses calls (prompt composition, chat) read content parts, where
// text refs contribute their contents.
export interface InlineImage {
/** Base64 payload, no data: prefix. */
data: string;
mimeType: string;
}
const MAX_W = 1280;
const JPEG_Q = 0.85;
// Every picture riding to a hosted model is fitted to this budget first — a
// generated reference sheet comes back at the image model's full output size, and
// a handful of those is more than one model call should carry. Getting the bytes
// to the model is storage's job (hostedBlobs.ts), so this budget is about what
// the model reads: it resamples inputs to roughly this size anyway, so nothing
// it sees is lost.
const MAX_INLINE_EDGE = 2048;
const MAX_INLINE_BYTES = 4 * 1024 * 1024;
// Vertex image models accept only real image mime types. Read the format from the
// bytes rather than the transport Content-Type: a media file served as
// application/octet-stream (or with none) otherwise reaches the model labelled with a
// mimeType it rejects with INVALID_ARGUMENT.
function sniffImageMime(base64: string): string | null {
let head: string;
try {
head = atob(base64.slice(0, 24));
} catch {
return null;
}
const at = (i: number) => head.charCodeAt(i);
if (at(0) === 0x89 && at(1) === 0x50 && at(2) === 0x4e && at(3) === 0x47) return "image/png";
if (at(0) === 0xff && at(1) === 0xd8 && at(2) === 0xff) return "image/jpeg";
if (at(0) === 0x47 && at(1) === 0x49 && at(2) === 0x46) return "image/gif";
if (
at(0) === 0x52 && at(1) === 0x49 && at(2) === 0x46 && at(3) === 0x46 &&
at(8) === 0x57 && at(9) === 0x45 && at(10) === 0x42 && at(11) === 0x50
)
return "image/webp";
return null;
}
function splitDataUrl(dataUrl: string): InlineImage {
const comma = dataUrl.indexOf(",");
// Between "data:" and the first ";" (or "," when there are no params).
const labelled = dataUrl.slice(5, comma).split(";")[0];
const data = dataUrl.slice(comma + 1);
// Trust the bytes over the label: a data URL may be typeless ("data:;base64,…") or
// carry a non-image type (application/octet-stream from a media URL). Sniff the real
// format, keeping a valid image/* label, and fall back to png only when unknown.
const mimeType =
(labelled.startsWith("image/") ? labelled : null) ?? sniffImageMime(data) ?? "image/png";
return { data, mimeType };
}
async function blobToDataUrl(blob: Blob): Promise<string> {
const bytes = new Uint8Array(await blob.arrayBuffer());
// Chunked so a large picture doesn't blow the argument limit.
let binary = "";
for (let i = 0; i < bytes.length; i += 0x8000)
binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
return `data:${blob.type || "application/octet-stream"};base64,${btoa(binary)}`;
}
/** Re-encode a picture down into the inline budget: scaled to the edge cap,
* then stepped down until it fits. Null when the browser can't decode it — the
* caller keeps the original bytes rather than dropping the reference. */
async function encodeWithin(blob: Blob): Promise<InlineImage | null> {
const bitmap = await createImageBitmap(blob).catch(() => null);
if (!bitmap) return null;
try {
let edge = MAX_INLINE_EDGE;
let out: InlineImage | null = null;
for (const quality of [JPEG_Q, 0.7, 0.55]) {
const scale = Math.min(1, edge / Math.max(bitmap.width, bitmap.height));
const canvas = createRasterCanvas(
Math.max(1, Math.round(bitmap.width * scale)),
Math.max(1, Math.round(bitmap.height * scale))
);
const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
if (!ctx) return null;
ctx.imageSmoothingQuality = "high";
// JPEG carries no alpha, so a cut-out sheet composites onto white here
// instead of the canvas's transparent black.
ctx.fillStyle = "#fff";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
out = splitDataUrl(await rasterCanvasToDataUrl(canvas, "image/jpeg", quality));
if (out.data.length * 0.75 <= MAX_INLINE_BYTES) return out;
edge = Math.round(edge * 0.75);
}
return out;
} finally {
bitmap.close();
}
}
/** A blob's bytes as an inline image part, fitted to the inline budget, mime
* read from the bytes. */
export async function blobToInline(blob: Blob): Promise<InlineImage> {
if (blob.size > MAX_INLINE_BYTES) {
const fitted = await encodeWithin(blob);
if (fitted) return fitted;
}
return splitDataUrl(await blobToDataUrl(blob));
}
/** A media blob's bytes as an inline audio part, mime from the blob's own type
* (the engine's extractor hands back audio/aac). Null when it clears the inline
* cap — a longer stretch than the model can take at once. */
export async function blobToInlineAudio(blob: Blob): Promise<InlineImage | null> {
if (blob.size > MAX_AUDIO_BYTES) return null;
const dataUrl = await blobToDataUrl(blob);
const comma = dataUrl.indexOf(",");
const labelled = dataUrl.slice(5, comma).split(";")[0];
return {
data: dataUrl.slice(comma + 1),
mimeType: labelled.startsWith("audio/") ? labelled : "audio/aac",
};
}
/** Video models take input images only as JPEG or PNG — the render rejects anything
* else by format, killing the render after it was billed. Re-encode any other
* picture (webp saved off the web is the common case) to JPEG; jpeg and png
* pass through untouched. */
export async function videoSafeInline(img: InlineImage): Promise<InlineImage> {
if (img.mimeType === "image/jpeg" || img.mimeType === "image/png") return img;
const blob = await (await fetch(`data:${img.mimeType};base64,${img.data}`)).blob();
const fitted = await encodeWithin(blob);
if (!fitted) throw new UserError("ref_format_unreadable", "A reference image is in a format the video model can't read.");
return fitted;
}
// Keeps the inline payload well under the Gemini per-request inline-data cap
// (base64 inflates by 4/3); larger audio degrades to a name-only marker.
export const MAX_AUDIO_BYTES = 12 * 1024 * 1024;
/** A ref's bytes, naming what failed and where when they don't arrive. A bare
* fetch rejection is the string "Failed to fetch" and nothing else — no asset,
* no address — and the failures that reach here are exactly the ones that need
* both: a URL built against the wrong backend, a cross-origin hop that answers
* without CORS, an object that was never written. Which host and path was asked
* for tells them apart. */
async function readRefBytes(ref: AssetRef): Promise<Blob> {
const where = mediaAddress(ref.url);
let res: Response;
try {
res = await fetch(ref.url);
} catch (err) {
// A rejected fetch never reached a server: DNS, CORS, mixed content, or a
// refused connection.
const why = err instanceof Error ? err.message : String(err);
throw new UserError("ref_unreachable", `Could not reach “${ref.name}” at ${where} — ${why}.`, { name: ref.name, where, why });
}
if (!res.ok) throw new UserError("ref_load_failed", `Could not load “${ref.name}” — ${res.status} from ${where}.`, { name: ref.name, status: res.status, where });
return res.blob();
}
/** The audio bytes for `ref`, base64 with a real audio mime, or null when the
* file is too large to ride inline. */
export async function refToInlineAudio(ref: AssetRef): Promise<InlineImage | null> {
const blob = await readRefBytes(ref);
if (blob.size > MAX_AUDIO_BYTES) return null;
const dataUrl = await blobToDataUrl(blob);
const comma = dataUrl.indexOf(",");
const labelled = dataUrl.slice(5, comma).split(";")[0];
return {
data: dataUrl.slice(comma + 1),
mimeType: labelled.startsWith("audio/") ? labelled : "audio/mpeg",
};
}
/** A media URL as much of itself as may be repeated. Origin and path are what
* identify a failure; the query string is not, and on cloud media it holds the
* signed token that grants read access to the object — these messages reach the
* chat transcript, which is persisted and readable by anyone a project's chat is
* shared with. */
export function mediaAddress(url: string): string {
try {
const u = new URL(url, window.location.origin);
return u.protocol === "data:" ? "an inline attachment" : `${u.origin}${u.pathname}`;
} catch {
return "an unreadable address";
}
}
async function captureFrame(
url: string,
duration?: number,
opts?: { at?: number; width?: number }
): Promise<InlineImage> {
const input = openMedia(url);
try {
const track = await videoTrackOf(input);
if (!track) {
const where = mediaAddress(url);
throw new UserError("ref_frame_unreadable", `Could not read ${where} for a reference frame.`, { where });
}
const dur = duration || (await input.computeDuration()) || 2;
// A pinned moment reads exactly there (kept inside the source); otherwise
// the same poster spot the cards show, so the reference matches the preview.
const at =
opts?.at !== undefined
? Math.min(Math.max(0, opts.at), Math.max(0, dur - 0.05))
: Math.min(1, Math.max(0.1, dur / 10));
const width = Math.min(opts?.width ?? MAX_W, await track.getDisplayWidth());
const frame = await frameSink(track, { width }).getCanvas(at);
if (!frame) throw new UserError("video_frame_unreadable", "The video has no readable frame.");
const canvas = frame.canvas;
return splitDataUrl(
canvas instanceof OffscreenCanvas
? await blobToDataUrl(await canvas.convertToBlob({ type: "image/jpeg", quality: JPEG_Q }))
: canvas.toDataURL("image/jpeg", JPEG_Q)
);
} finally {
input.dispose();
}
}
/** The length of the SOURCE behind a ref, as far as the ref knows it. A clip
* ref's `duration` is the clip's own length, not its source file's — times on
* a clip ref (its `t` pin) are source seconds, so the capture must probe the
* real source length instead. */
const sourceDuration = (ref: AssetRef): number | undefined =>
ref.scope === "clip" ? undefined : ref.duration;
/** A single reference image for `ref`: the file itself for images, a captured
* frame for videos — the pinned moment when the user chose one, the poster
* spot otherwise. Audio and text have no picture and reject. */
export async function refToInlineImage(ref: AssetRef): Promise<InlineImage> {
if (ref.kind === "audio" || ref.kind === "text") {
throw new UserError("ref_no_picture", `“${ref.name}” has no picture — it can't be a visual reference.`, { name: ref.name });
}
if (ref.kind === "image") return blobToInline(await readRefBytes(ref));
return captureFrame(ref.url, sourceDuration(ref), ref.t !== undefined ? { at: ref.t } : undefined);
}
/**
* Put the picture a ref points at on the system clipboard, as a PNG so it
* pastes into anything that takes an image.
*
* The blob goes in as a promise rather than an awaited value: Safari ties a
* clipboard write to the gesture that asked for it, and decoding a frame first
* spends that gesture. Throws when the frame cannot be read or the browser
* refuses the write, which is the caller's cue to say so.
*/
export async function copyRefImage(ref: AssetRef): Promise<void> {
const png = (async () => {
const inline = await refToInlineImage(ref);
const bytes = Uint8Array.from(atob(inline.data), (c) => c.charCodeAt(0));
const blob = new Blob([bytes], { type: inline.mimeType });
if (inline.mimeType === "image/png") return blob;
// Clipboard image writes are PNG everywhere; a JPEG frame re-encodes.
const bitmap = await createImageBitmap(blob);
const canvas = document.createElement("canvas");
canvas.width = bitmap.width;
canvas.height = bitmap.height;
canvas.getContext("2d")!.drawImage(bitmap, 0, 0);
bitmap.close();
return new Promise<Blob>((resolve, reject) =>
canvas.toBlob((b) => (b ? resolve(b) : reject(new UserError("frame_encode_failed", "Could not encode the frame."))), "image/png")
);
})();
await navigator.clipboard.write([new ClipboardItem({ "image/png": png })]);
}
/** Reference images for a whole ref list, in order; skips nothing — a broken
* ref fails the call so the user knows the reference didn't ride along. */
export function refsToInlineImages(refs: AssetRef[]): Promise<InlineImage[]> {
return Promise.all(refs.map(refToInlineImage));
}
/** The refs that can ride to a visual model as input images. */
export const visualRefs = (refs: AssetRef[]): AssetRef[] =>
refs.filter((r) => r.kind === "image" || r.kind === "video");
/** Attach OS files dropped on a composer, returned as refs in drop order.
* Media files import into the project first; text files become transient file
* refs read at send time. Files that are neither, and failed imports, are
* skipped. `chatId` marks the imports chat-owned from creation: they live on
* the chat (its card, thread deletion), never in the Media panel — a drop on
* the chat composer is an attachment, not a filing. Panel drops omit it and
* file into Media as user imports.
*
* A drop shows its chip the moment the local bytes are probed: the ref plays
* from the file's own object URL while the upload queue sends the bytes
* behind the composer, and the asset swaps to its stored URL when they land
* (chips and the send paths read the live URL from the asset). */
export async function refsFromDroppedFiles(
projectId: string,
files: File[],
opts?: { chatId?: string }
): Promise<AssetRef[]> {
const refs: AssetRef[] = [];
checkImportRoom(files.filter(isMediaFile));
for (const file of files) {
try {
if (isTextFile(file) && !isMediaFile(file)) {
refs.push(await refFromTextFile(file));
continue;
}
const pending = await prepareImport(projectId, file);
const asset = pending?.asset ?? (await importFileToProject(projectId, file, getBackend(), "caller"));
if (!asset) continue;
useEditor.getState().addAsset(asset);
if (opts?.chatId) tagChatAsset(asset.id, opts.chatId);
// A pending import still holds its bytes — enrich from those.
void enrichAsset(asset, pending?.localUrl);
if (pending) startUpload(projectId, pending);
refs.push(refFromAsset(asset));
} catch (err) {
reportSwallowed("[cut] attach failed", err, file.name);
}
}
return refs;
}
/** A text ref's contents (a dropped script, notes, subtitles…). */
export async function readRefText(ref: AssetRef): Promise<string> {
const res = await fetch(ref.url);
if (!res.ok) throw new UserError("media_read_failed", `Could not read “${ref.name}”.`, { name: ref.name });
return res.text();
}
// A pinned video moment also samples a frame this far to each side, so the
// pick lands even when the user's scrub was a beat off the exact frame.
const PIN_SAMPLE_SPREAD = 1;
const PIN_SAMPLE_W = 640;
// A pinned audio moment reads as a segment reaching this far back and forward
// — enough lead-in to place the moment and enough tail to hear the passage.
const PIN_AUDIO_BACK = 5;
const PIN_AUDIO_FORWARD = 25;
/** The audio around a pinned moment as an inline part, rendered in the
* browser from the source URL. Null when the span can't be read (no audio
* track, an unreadable source) — the caller falls back to the whole file. */
async function pinnedAudioSegment(
ref: AssetRef,
t: number
): Promise<{ audio: InlineImage; from: number; to: number } | null> {
const dur = sourceDuration(ref);
const from = Math.max(0, t - PIN_AUDIO_BACK);
const to = dur !== undefined ? Math.min(dur, t + PIN_AUDIO_FORWARD) : t + PIN_AUDIO_FORWARD;
if (to <= from) return null;
try {
const audio = await blobToInlineAudio(await renderAudioSpanWav(ref.url, from, to));
return audio ? { audio, from, to } : null;
} catch {
return null;
}
}
/** A ref list as hosted-Responses content parts, for any Gemini call that
* should actually see the attachments: each visual ref contributes a numbered
* label plus its picture (image as-is, video by poster frame), each text ref
* its contents, and each audio ref its actual sound (so the model can answer
* about what's said without touching the project; oversized files fall back
* to a name-only marker). A ref with a pinned moment (`t`) reads from there
* instead: a video contributes the pinned frame plus a sampled frame each
* side, an audio ref the segment around the moment. `visuals` returns the
* pictures in label order (`Image 1` = visuals[0]) so a model can pick among
* them. */
export async function refsToParts(
refs: AssetRef[]
): Promise<{ parts: Record<string, unknown>[]; visuals: InlineImage[] }> {
const parts: Record<string, unknown>[] = [];
const visuals: InlineImage[] = [];
const pushImage = (image: InlineImage, label: string) => {
visuals.push(image);
parts.push({ text: `Image ${visuals.length} — ${label}` });
parts.push({ type: "input_image", dataBase64: image.data, mimeType: image.mimeType });
};
for (const ref of refs) {
if (ref.kind === "audio") {
const segment = ref.t !== undefined ? await pinnedAudioSegment(ref, ref.t) : null;
const audio = segment?.audio ?? (await refToInlineAudio(ref));
if (!audio) {
parts.push({ text: `Attached audio "${ref.name}" (too large to include inline).` });
continue;
}
parts.push({
text: segment
? `Attached audio "${ref.name}" — the segment around the user's pinned moment ${formatTime(ref.t!)} (${formatTime(segment.from)}–${formatTime(segment.to)}):`
: `Attached audio "${ref.name}":`,
});
parts.push({ type: "input_audio", dataBase64: audio.data, mimeType: audio.mimeType });
continue;
}
if (ref.kind === "text") {
parts.push({
text:
ref.scope === "note"
? `Referenced note "${ref.name}". Read its current text with read_note: ${JSON.stringify({ id: ref.id })}`
: ref.scope === "projectDoc"
? `Referenced project "${ref.name}". Read its current document with read_project: ${JSON.stringify({ link: ref.id })}`
: ref.use
? `Referenced ${ref.use.noun} "${ref.name}" in the Library. Use it with ${ref.use.tool}: ${JSON.stringify(ref.use.args)}`
: ref.scope === "folder"
? `Referenced folder "${ref.name}". Read its current contents with read_folder: ${JSON.stringify(ref.folder)}`
: isCatalogRef(ref)
? `Referenced "${ref.name}" — ${await readRefText(ref)}`
: ref.scope === "entity"
? `Referenced timeline entity "${ref.name}":\n${await readRefText(ref)}`
: `Attached file "${ref.name}":\n${await readRefText(ref)}`,
});
continue;
}
if (ref.kind === "video" && ref.t !== undefined) {
const t = ref.t;
const dur = sourceDuration(ref);
pushImage(
await refToInlineImage(ref),
`the frame the user pinned, ${formatTime(t)} into video "${ref.name}":`
);
for (const dt of [-PIN_SAMPLE_SPREAD, PIN_SAMPLE_SPREAD]) {
const at = Math.max(0, dur !== undefined ? Math.min(t + dt, dur) : t + dt);
// A neighbor clamped onto the pin itself adds nothing.
if (Math.abs(at - t) < 0.25) continue;
const image = await captureFrame(ref.url, dur, { at, width: PIN_SAMPLE_W }).catch(
() => null
);
if (!image) continue;
pushImage(image, `the same video ${formatTime(at)}, sampled near the pinned frame:`);
}
continue;
}
pushImage(
await refToInlineImage(ref),
ref.kind === "video" ? `a frame of video "${ref.name}":` : `image "${ref.name}":`
);
}
return { parts, visuals };
}