Open source CapCut alternative.
TypeScript
"use client";
import { useEffect } from "react";
import { create } from "zustand";
import { getBackend, type CutBackend, type CutMode } from "./backend";
import { browserBackend } from "./backend/browser";
import { cloudBackend } from "./backend/cloud";
import { localBackend } from "./backend/local";
import { normalizeRef, type AssetRef } from "./assetRef";
import { bytesFromBase64 } from "./bytes";
import { loadProjectThreads, readThreadIds, threadsLoaded } from "./chatThreads";
import { composeGenPrompt, foldTextRefs } from "./composeGen";
import {
projectWriteMode,
removeRendersInDoc,
stockAssetInDoc,
upsertRenderInDoc,
} from "./genvideo/docWriter";
import { NoCreditsError, readHostedFailure } from "./hostedError";
import { CUT_APP_BASE } from "./nav";
import { hostedFetch, hostedPost } from "./hosted";
import { enrichAsset, importFileToProject, uploadProjectImage } from "./media";
import { refsToInlineImages, videoSafeInline, visualRefs, type InlineImage } from "./refMedia";
import { useGenNotify } from "./genNotify";
import { IMAGE_ASPECTS, useImageGen } from "./imageGen";
import { useEditor } from "./store";
import { mediaSlug, nearestAspect, type MediaAsset, type RenderRecord } from "./types";
import { UserError, errorFields, type UserErrorParams } from "./userError";
import {
DEFAULT_VIDEO_RESOLUTION,
aspectFramingNote,
defaultVideoAspects,
videoModel,
type VideoResolution,
} from "./videoModels";
import { walkLadder, type VideoAttempt } from "./videoLadder";
// AI generation jobs, held outside the panels so a tab switch (which unmounts
// them) doesn't orphan a running generation — video renders take minutes.
//
// Generation runs on Donkey's hosted inference routes with the user's Donkey
// sign-in and credits. The cut hosts serve the same Next app and the auth
// cookie rides the registrable domain, so these are plain same-origin calls.
// Finished media lands back in the project through the local engine: videos
// upload as regular files, images bake into still clips.
export interface GenerateJob {
id: string;
projectId: string;
kind: "image" | "video";
prompt: string;
/** Epoch ms when the render started — live cards show a ticking elapsed. */
startedAt: number;
status: "running" | "done" | "error";
error?: string;
errorCode?: string;
errorParams?: UserErrorParams;
/** The project asset the finished generation landed as. */
assetId?: string;
/** The chat thread that launched this render, when it wasn't the panel.
* An owned job shows on its chat card only — the Video/Image panels list
* panel renders, so they skip it — and its asset lands chat-tagged. */
chatId?: string;
/** Stable identity of the work this render is FOR (a scene run's shot), so a
* resumed caller re-adopts exactly its own in-flight job — never another
* shot's that happens to share a prompt. */
genKey?: string;
/** The backend the job started on. Landing work can outlive navigation into
* a project of the other residency, so every project write for this job pins
* here instead of reading the (rebound) global mode at land time. */
residency?: CutMode;
/** Which ladder rung landed the render (0-based index into the attempts) —
* how a caller knows whether its take rode an image anchor. */
rung?: number;
/** The ladder this video job submitted — prompt, refs, aspect, fallbacks —
* kept on the job so a failed render can resubmit itself (`retry`) instead
* of dead-ending with only its prompt on screen. Rung gates are functions,
* so they survive in memory but not the localStorage round trip; a retried
* job from a past session walks every rung. */
attempts?: VideoAttempt[];
/** The provider poll payload for an in-flight render — persisting it is what
* lets a reload re-attach to the running job instead of orphaning it. */
poll?: {
id: string;
provider: string;
model: string;
providerJobId?: string | null;
providerGenerationId?: string | null;
providerPollingUrl?: string | null;
metadata?: Record<string, unknown>;
};
}
/** A render's initial submission outcome, distinct from its settlement: `ok`
* once the backend accepts the render (it is genuinely in flight, still minutes
* from landing), or the reason when the submit is rejected outright — auth,
* credits, a bad request — before any render started. Lets a caller claim the
* render is underway only when it actually is, never on an instant reject. */
export type VideoSubmitOutcome = { ok: true } | { ok: false; error: string };
interface GenerateState {
/** Whether a Donkey session exists; null = not probed yet. */
signedIn: boolean | null;
jobs: GenerateJob[];
probe: () => void;
/** Awaitable probe, for callers that must know before acting (the AI
* assistant's generate tools); resolves the fresh signed-in answer. */
probeNow: () => Promise<boolean>;
/** Kick off a generation. Returns when it settles, resolving to the final
* job (status "done" with assetId, or "error" with a message) so the AI
* assistant can act on the result; the panel just fires and forgets.
* `refs` are references of any kind — images upload as-is, videos by a
* captured poster frame, text files by their contents folded into the
* prompt (see composeGen.ts). */
generateImage: (
projectId: string,
prompt: string,
opts?: {
refs?: AssetRef[];
aspect?: "16:9" | "9:16" | "1:1";
resolution?: "1K" | "2K" | "4K";
/** Rewrite the prompt around the references before rendering (the
* default when refs are present) — see composeGen.ts. */
composeRefs?: boolean;
/** The owning chat thread when chat (or a scene run) asked — stamps the
* job and the landed asset, keeping both off the generate panels. */
chatId?: string;
}
) => Promise<GenerateJob>;
/** Kick off a video render. Returns the job id right away (the render
* outlives most callers) plus the settled job for anyone who waits. */
generateVideo: (
projectId: string,
prompt: string,
opts?: VideoGenOptions
) => { jobId: string; settled: Promise<GenerateJob>; submitted: Promise<VideoSubmitOutcome> };
/** Kick off a video render that walks an identity ladder: ONE job spans the
* attempts in order, and a failed rung submits the next instead of failing
* the job — so a safety-blocked or rejected anchor degrades the render
* instead of killing it. An empty balance stops the walk (every rung would
* fail the same way). Both the scene pipeline's shots and chat's one-off
* renders run on this, so the two have identical fallback behavior. */
generateVideoLadder: (
projectId: string,
attempts: VideoAttempt[],
jobOpts?: {
chatId?: string;
genKey?: string;
onDone?: (asset: MediaAsset) => void;
/** Called as each rung starts (0-based) — progress narration. */
onAttempt?: (rung: number) => void;
}
) => { jobId: string; settled: Promise<GenerateJob>; submitted: Promise<VideoSubmitOutcome> };
/** Resubmit a failed video job as itself: same job id (so its chat card and
* doc mirror follow along), same ladder. A render that failed on an empty
* balance retries once credits land without retyping the prompt or losing its
* references. */
retry: (id: string) => void;
dismiss: (id: string) => void;
/** Cancel every render owned by a deleted thread or project: matching jobs are
* dropped from the list and their in-flight polls refuse to land, so a render
* still running when the user deletes its owner can't re-add media afterward.
* Called by thread deletion (chatId) and project deletion (projectId). */
cancelForOwner: (owner: { chatId?: string; projectId?: string }) => void;
/** Re-attach the poll loop of every persisted running render (reload
* recovery) — called once when the app boots. Idempotent. */
resumeRunningJobs: () => void;
}
export interface VideoGenOptions {
/** Composition shape; defaults to the project's orientation. */
aspect?: "16:9" | "9:16";
/** Output size; defaults to the registry's default (720p). */
resolution?: VideoResolution;
/** Clip length in whole seconds, within the registry's range; unset lets
* the model pick. */
durationSeconds?: number;
/** What the render must avoid — the wrong medium's tells, letterbox bars,
* on-screen text. The model has no negative-prompt parameter, so the
* adapter folds this into the prompt as an avoid clause. */
negativePrompt?: string;
/** References of any kind — video, image, text file, audio. The model takes
* one seed image, so at most one picture seeds the render as its literal
* opening frame; audio and text contribute through the compose rewrite. */
refs?: AssetRef[];
/** Identity anchors (up to the registry's maxReferenceImages): the render
* keeps these characters/objects/scenes consistent instead of playing one
* as the first frame. Mutually exclusive with a `refs` image seed; the
* prompt rides as written (no compose rewrite). */
referenceImages?: AssetRef[];
/** Rewrite the prompt around the references before rendering (the
* default when refs are present) — see composeGen.ts. The model plays the
* input image as the literal first frame, so a prompt that transforms
* the reference must become a standalone description with the image
* dropped; the rewrite decides which. Character mode passes false — the
* poster seed is the point. */
composeRefs?: boolean;
/** Called once with the landed asset when the render completes and the
* project is still open — lets the AI place the clip after a background
* render it couldn't wait out (the assistant tool bridge caps at 2min). */
onDone?: (asset: MediaAsset) => void;
/** The owning chat thread when chat (or a scene run) asked — stamps the
* job and the landed asset, keeping both off the generate panels. */
chatId?: string;
/** Stable identity of the work this render is for — see GenerateJob.genKey. */
genKey?: string;
}
export type { VideoAttempt } from "./videoLadder";
const REFRESH_MS = 8000;
const VIDEO_DEADLINE_MS = 12 * 60_000;
const uid = () => crypto.randomUUID().slice(0, 8);
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
/** The backend a job's results land on: its pinned residency; jobs persisted
* before residency existed fall back to the active backend. */
const jobBackend = (job: { residency?: CutMode }): CutBackend =>
job.residency === "cloud"
? cloudBackend
: job.residency === "local"
? localBackend
: job.residency === "browser"
? browserBackend
: getBackend();
/** Display name for the asset: the prompt, tidied and capped. */
const promptName = (prompt: string) => {
const line = prompt.trim().replace(/\s+/g, " ");
return line.length > 60 ? `${line.slice(0, 57)}…` : line;
};
/** The Donkey sign-in URL, same-host, returning to the Cut page the user
* started from. Image, video, and voiceover all run on the user's Donkey
* account, so every generation surface links here when signed out. */
export function signInUrl(): string {
if (typeof window === "undefined") return "/sign-in";
return `/sign-in?callbackURL=${encodeURIComponent(window.location.href)}`;
}
/** Cut's settings page, where the plans are sold. Linked from any
* generation error caused by an empty balance. */
export function creditsUrl(): string {
return `${CUT_APP_BASE}/settings`;
}
/** Creation-time ownership for generated media, in one place so the rule can't
* drift per call site: chat-owned media lives on its chat card — never a
* generate-panel row or arrival badge — and everything else is a panel render
* (origin "generated", so it stays out of the Media panel's user imports). */
export function applyOwnership(asset: MediaAsset, chatId: string | undefined | null): void {
if (chatId) {
asset.origin = "chat";
asset.chatId = chatId;
} else {
asset.origin = "generated";
}
}
interface GenerationOutput {
dataBase64?: string;
contentType?: string;
filename?: string;
url?: string;
}
interface GenerationResponse {
id: string;
status: string;
provider: string;
model: string;
providerJobId: string | null;
providerGenerationId: string | null;
providerPollingUrl: string | null;
outputs: GenerationOutput[];
error?: unknown;
metadata?: Record<string, unknown>;
}
const readFailure = (res: Response, fallback: UserError) =>
readHostedFailure(res, new UserError("sign_in_to_generate_media", "Sign in to Donkey to generate media."), fallback);
const videoFailed = () => new UserError("video_generation_failed", "Video generation failed.");
const providerError = (error: unknown): string | null => {
if (typeof error === "string" && error) return error;
if (error && typeof error === "object" && "message" in error) {
const m = (error as { message?: unknown }).message;
if (typeof m === "string" && m) return m;
}
return null;
};
/** Resolve a generation's prompt + input images from its refs. Runs the shared
* compose step (see composeGen.ts) unless the caller opted out; on a compose
* failure, falls back to the visual refs as-is with text-ref contents folded
* into the prompt. `maxImages` caps what the generator accepts (video: 1 seed). */
export async function promptAndImages(
target: "video" | "image",
prompt: string,
refs: AssetRef[],
compose: boolean,
maxImages: number
): Promise<{ prompt: string; images: InlineImage[] }> {
if (refs.length === 0) return { prompt, images: [] };
if (compose) {
const composed = await composeGenPrompt(target, prompt, refs);
if (composed) return { prompt: composed.prompt, images: composed.images.slice(0, maxImages) };
}
return {
prompt: await foldTextRefs(prompt, refs),
images: await refsToInlineImages(visualRefs(refs).slice(0, maxImages)),
};
}
/** Video jobs persist per browser: settled ones so an unplaced render (a chat
* card, the Video panel's renders list) survives a reload, and running ones
* with their provider poll payload so a reload RE-ATTACHES to the in-flight
* render (resumeRunningJobs) instead of orphaning credits already spent. A
* running job persisted without a payload (it died before the provider
* answered) restores as an error. */
const JOBS_KEY = "cut-gen-video-jobs";
const JOBS_CAP = 50;
/** Awaitable settlements for in-flight renders, fresh and reload-resumed —
* how a resumed caller (the scene pipeline) adopts a running job's result
* instead of billing a second take. */
const settlements = new Map<string, Promise<GenerateJob>>();
// Projects whose chat history the resume sweep has asked to load.
const threadLoads = new Set<string>();
/** The settled-job promise for a running render, when this session owns it. */
export function videoJobSettlement(jobId: string): Promise<GenerateJob> | undefined {
return settlements.get(jobId);
}
function readPersistedJobs(): GenerateJob[] {
try {
const v = JSON.parse(localStorage.getItem(JOBS_KEY) ?? "[]") as unknown;
if (!Array.isArray(v)) return [];
return (v as GenerateJob[])
.filter((j) => j?.id)
.map((j) =>
j.status === "running" && !j.poll
? {
...j,
status: "error" as const,
...errorFields(
null,
new UserError("video_render_interrupted", "Interrupted by a reload before the render started."),
),
}
: j
);
} catch {
return [];
}
}
function persistJobs(jobs: GenerateJob[]) {
try {
localStorage.setItem(
JOBS_KEY,
JSON.stringify(
jobs
.filter((j) => j.kind === "video" && (j.status !== "running" || j.poll))
.slice(0, JOBS_CAP)
)
);
} catch {
// Storage full/blocked — render history just won't persist.
}
}
// Exactly one browser tab polls (and lands) a given render. Persisted running
// jobs are visible to every tab of the origin, so each job carries a lease in
// localStorage: the owning tab renews it on every poll, and another tab takes
// over only once the lease has gone stale (the owner closed or crashed).
const TAB_ID = crypto.randomUUID().slice(0, 8);
const LEASE_TTL_MS = 30_000;
const leaseKey = (jobId: string) => `cut-gen-lease-${jobId}`;
/** Take (or renew) the poll lease for a job. False when a live sibling tab
* holds it. Storage failures grant the lease — a tab that can't read storage
* can't see other tabs' leases either, and single-tab correctness wins. */
function claimJobLease(jobId: string): boolean {
try {
const raw = localStorage.getItem(leaseKey(jobId));
if (raw) {
const lease = JSON.parse(raw) as { tab?: string; at?: number };
if (lease.tab !== TAB_ID && Date.now() - (lease.at ?? 0) < LEASE_TTL_MS) return false;
}
localStorage.setItem(leaseKey(jobId), JSON.stringify({ tab: TAB_ID, at: Date.now() }));
return true;
} catch {
return true;
}
}
function releaseJobLease(jobId: string): void {
try {
localStorage.removeItem(leaseKey(jobId));
} catch {
// Stale lease expires on its own TTL.
}
}
export const useGenerate = create<GenerateState>((set, get) => {
// A single session probe is in flight at a time, but the answer stays
// re-checkable for the page's whole life: a sign-in can complete in another
// tab, or the session cookie can land a beat after this page loads.
let probing: Promise<boolean> | null = null;
// Jobs whose owner (thread or project) was deleted mid-render. Their poll loop
// checks this before landing anything, so a render that finishes after its
// owner is gone drops its result instead of re-adding orphaned media.
const cancelledJobs = new Set<string>();
// Chat-launched video renders mirror into the project doc (doc.renders): the
// job store is per browser, the doc travels with the project, so the mirror
// is what makes the chat's render card show the same outcome everywhere.
// Open project → the store (autosave persists it); closed → its doc.
const mirrorRender = (id: string) => {
const j = get().jobs.find((x) => x.id === id);
if (!j || j.kind !== "video" || !j.chatId) return;
const record: RenderRecord = {
id: j.id,
chatId: j.chatId,
prompt: j.prompt,
startedAt: j.startedAt,
status: j.status,
...(j.error !== undefined ? { error: j.error } : {}),
...(j.errorCode !== undefined ? { errorCode: j.errorCode, errorParams: j.errorParams ?? {} } : {}),
...(j.assetId !== undefined ? { assetId: j.assetId } : {}),
};
const backend = jobBackend(j);
void (async () => {
if ((await projectWriteMode(j.projectId)) === "store" &&
useEditor.getState().projectId === j.projectId && useEditor.getState().loaded)
useEditor.getState().upsertRender(record);
else await upsertRenderInDoc(j.projectId, record, backend);
})().catch(() => {});
};
const unmirrorRenders = (projectId: string, victims: GenerateJob[]) => {
const mirrored = victims.filter((j) => j.kind === "video" && j.chatId);
if (mirrored.length === 0) return;
const ids = mirrored.map((j) => j.id);
const backend = jobBackend(mirrored[0]);
void (async () => {
if ((await projectWriteMode(projectId)) === "store" &&
useEditor.getState().projectId === projectId && useEditor.getState().loaded)
useEditor.getState().removeRenders(ids);
else await removeRendersInDoc(projectId, ids, backend);
})().catch(() => {});
};
const update = (id: string, patch: Partial<GenerateJob>) => {
set((s) => ({ jobs: s.jobs.map((j) => (j.id === id ? { ...j, ...patch } : j)) }));
persistJobs(get().jobs);
// Poll-payload and rung refreshes stay browser-local; only record-visible
// fields re-mirror.
if ("status" in patch || "assetId" in patch || "error" in patch) mirrorRender(id);
};
const fail = (id: string, err: unknown) =>
update(id, {
status: "error",
errorCode: undefined,
errorParams: undefined,
...errorFields(err, String(err)),
});
// The job after it has settled, for callers that awaited the run.
const settled = (id: string, fallback: GenerateJob): GenerateJob =>
get().jobs.find((j) => j.id === id) ?? fallback;
// The media file exists either way. The open project records it in the live
// store; a closed one records it in its persisted doc so nothing the render
// made is orphaned. Returns whether the OPEN project adopted it (badges and
// placement callbacks only make sense on screen).
const adopt = (projectId: string, asset: MediaAsset, backend?: CutBackend): boolean => {
if (useEditor.getState().projectId !== projectId) {
void stockAssetInDoc(projectId, asset, backend).catch(() => {});
return false;
}
useEditor.getState().addAsset(asset);
void enrichAsset(asset);
return true;
};
/** Poll an in-flight video render to completion and land its file as a project
* asset — the shared tail of a fresh render and a reload-resumed one. Keeps
* the job's poll payload fresh so the NEXT reload re-attaches too. */
const finishVideo = async (
jobId: string,
first: GenerationResponse,
onDone?: (asset: MediaAsset) => void,
since?: number
): Promise<void> => {
const job = get().jobs.find((j) => j.id === jobId);
if (!job) return;
let gen = first;
// The render is abandoned once its owner is deleted: cancelForOwner drops the
// job from the list, so a vanished job (or one flagged mid-render) means stop
// polling and land nothing — the media it would have added is unwanted.
const aborted = () => cancelledJobs.has(jobId) || !get().jobs.some((j) => j.id === jobId);
// A resumed render keeps its original deadline, floored so a reload near
// the limit still gets a couple of polls before giving up. A ladder rung
// passes its own start so a slow first rung doesn't starve the fallback.
const deadline = Math.max((since ?? job.startedAt) + VIDEO_DEADLINE_MS, Date.now() + 2 * 60_000);
while (gen.status === "in_progress") {
if (aborted()) return;
claimJobLease(jobId); // renewed every cycle; siblings back off until it staled
update(jobId, {
poll: {
id: gen.id,
provider: gen.provider,
model: gen.model,
providerJobId: gen.providerJobId,
providerGenerationId: gen.providerGenerationId,
providerPollingUrl: gen.providerPollingUrl,
metadata: gen.metadata ?? {},
},
});
if (Date.now() > deadline) {
throw new UserError("video_render_timeout", "The video render is taking too long — try again.");
}
await sleep(REFRESH_MS);
const poll = await hostedPost("/api/inference/assets/refresh", {
id: gen.id,
kind: "video",
provider: gen.provider,
model: gen.model,
providerJobId: gen.providerJobId,
providerGenerationId: gen.providerGenerationId,
providerPollingUrl: gen.providerPollingUrl,
metadata: gen.metadata ?? {},
});
if (!poll.ok) {
throw await readFailure(poll, videoFailed());
}
gen = (await poll.json()) as GenerationResponse;
}
if (gen.status !== "completed") {
const reason = providerError(gen.error);
throw reason ? new Error(reason) : videoFailed();
}
// Owner deleted while the render finished: skip the import entirely so no
// file is even written for media nobody will ever see.
if (aborted()) return;
const out = gen.outputs.find((o) => o.dataBase64) ?? gen.outputs.find((o) => o.url);
const fileName = `ai-${mediaSlug(job.prompt, "generated")}.mp4`;
let file: File;
if (out?.dataBase64) {
file = new File([bytesFromBase64(out.dataBase64)], fileName, {
type: out.contentType ?? "video/mp4",
});
} else if (out?.url) {
const dl = await fetch(out.url);
if (!dl.ok) {
throw new UserError("video_download_failed", "Could not download the generated video.");
}
file = new File([await dl.arrayBuffer()], fileName, {
type: out.contentType ?? "video/mp4",
});
} else {
throw new UserError("video_output_missing", "The provider returned no video.");
}
const backend = jobBackend(job);
const asset = await importFileToProject(job.projectId, file, backend);
if (!asset) {
throw new UserError("video_import_failed", "Could not import the generated video.");
}
// The owner may have been deleted during the import (the file write is a
// round trip): drop the file we just wrote rather than leave it orphaned.
if (aborted()) {
void backend
.fetch(`/api/cut/projects/${job.projectId}/media/${encodeURIComponent(asset.fileName)}`, {
method: "DELETE",
})
.catch(() => {});
return;
}
asset.name = promptName(job.prompt);
applyOwnership(asset, job.chatId);
if (adopt(job.projectId, asset, backend)) {
if (!job.chatId) useGenNotify.getState().landed("video", asset.id);
onDone?.(asset);
}
update(jobId, { status: "done", assetId: asset.id, poll: undefined });
};
/** An attempt with its refs re-resolved against the live catalogs — a
* project ref's persisted URL can be a short-lived signed one, and the
* asset it names may have moved since the job first ran. */
const freshRefs = (refs: AssetRef[] | undefined): AssetRef[] | undefined =>
refs?.map((r) => normalizeRef(r) ?? r);
const freshAttempt = (a: VideoAttempt): VideoAttempt =>
a.opts
? {
...a,
opts: {
...a.opts,
...(a.opts.refs ? { refs: freshRefs(a.opts.refs) } : {}),
...(a.opts.referenceImages
? { referenceImages: freshRefs(a.opts.referenceImages) }
: {}),
},
}
: a;
/** Walk a video job's ladder to settlement: submit rungs in order, poll the
* accepted one to landing, settle the job. The job is already in the list
* with its lease claimed — this is the shared engine behind a fresh
* generateVideoLadder call and retry's resubmission of a failed job. */
const runLadder = (
job: GenerateJob,
attempts: VideoAttempt[],
jobOpts?: {
onDone?: (asset: MediaAsset) => void;
onAttempt?: (rung: number) => void;
}
) => {
// Resolves the moment the backend accepts a rung's submit (a real render
// is in flight) or, if every rung is rejected before acceptance, with the
// reason. A caller awaits this to distinguish "rendering" from a render
// that never started (auth, credits, a bad request reject in well under a
// second) instead of claiming success on a fire-and-forget kickoff.
let resolveSubmitted!: (outcome: VideoSubmitOutcome) => void;
const submitted = new Promise<VideoSubmitOutcome>((resolve) => {
resolveSubmitted = resolve;
});
let submitReported = false;
const reportSubmit = (outcome: VideoSubmitOutcome) => {
if (submitReported) return;
submitReported = true;
resolveSubmitted(outcome);
};
/** Compose and submit one rung's render request; resolves the provider's
* first response (usually in_progress) or throws the readable error. */
const submitRung = async ({ prompt, opts }: VideoAttempt): Promise<GenerationResponse> => {
// The model seeds from a single first-frame image, so at most one kept
// picture rides along. Identity anchors travel separately: with
// referenceImages set, the prompt stands as written and no seed image
// rides (the render takes one or the other).
// Every picture riding to the video model goes through videoSafeInline:
// it takes only JPEG/PNG, and a webp reference would fail the render
// after it was already billed.
const anchors = opts?.referenceImages?.length
? await Promise.all(
(
await refsToInlineImages(
visualRefs(opts.referenceImages).slice(0, videoModel("omni").maxReferenceImages)
)
).map(videoSafeInline)
)
: [];
const { prompt: sent, images: rawImages } = anchors.length
? { prompt, images: [] as InlineImage[] }
: await promptAndImages("video", prompt, opts?.refs ?? [], opts?.composeRefs !== false, 1);
const images = await Promise.all(rawImages.map(videoSafeInline));
const projectAspect = useEditor.getState().aspect;
const aspectRatio = opts?.aspect ?? nearestAspect(projectAspect, defaultVideoAspects());
// A project on a shape the model can't render gets its clip cropped, so
// the prompt carries the frame the take is really for.
const framing = aspectFramingNote(projectAspect, aspectRatio);
const res = await hostedPost("/api/inference/assets", {
kind: "video",
prompt: framing ? `${sent}\n\n${framing}` : sent,
provider: "gemini-omni",
...(anchors.length > 0
? { inputs: { referenceImages: anchors } }
: images.length > 0
? { inputs: { images } }
: {}),
parameters: {
aspectRatio,
resolution: opts?.resolution ?? DEFAULT_VIDEO_RESOLUTION,
...(opts?.durationSeconds !== undefined ? { durationSeconds: opts.durationSeconds } : {}),
...(opts?.negativePrompt ? { negativePrompt: opts.negativePrompt } : {}),
},
});
if (!res.ok) {
throw await readFailure(res, videoFailed());
}
return (await res.json()) as GenerationResponse;
};
// The last rung's thrown failure, kept whole so the job records its code;
// the walk itself reports only the English.
let lastFailure: unknown = null;
const settledRun = (async () => {
try {
const outcome = await walkLadder(
attempts,
async (attempt, rung) => {
jobOpts?.onAttempt?.(rung);
// The rung persists as the attempt starts, not when the walk
// settles — a reload that adopts this in-flight job must still
// know which rung (anchored or not) produced the take.
update(job.id, { rung });
try {
const gen = await submitRung(attempt);
// The backend took the submit — a render is genuinely in flight
// now (instant rejects throw above and never reach here). A later
// polling failure still fails the job, but it did start.
reportSubmit({ ok: true });
await finishVideo(job.id, gen, jobOpts?.onDone, Date.now());
} catch (err) {
lastFailure = err;
throw err;
}
},
{
// The next rung is a fresh submission: drop the dead rung's poll
// payload so a reload can't resume a render that already failed.
onRungFailed: () => update(job.id, { poll: undefined }),
// The balance fails every rung identically — stop there so a
// render nobody can pay for fails fast, not once per rung.
fatal: () => lastFailure instanceof NoCreditsError,
}
);
if (!outcome.ok) {
fail(job.id, lastFailure ?? videoFailed());
}
} finally {
releaseJobLease(job.id);
// No rung was ever accepted: the render never started, so report the
// submit as failed with the job's recorded reason. A no-op once an
// accepted rung already reported success.
const errored = get().jobs.find((x) => x.id === job.id);
reportSubmit({ ok: false, error: errored?.error ?? "Video generation failed." });
}
return settled(job.id, job);
})();
settlements.set(job.id, settledRun);
return { jobId: job.id, settled: settledRun, submitted };
};
return {
signedIn: null,
jobs: typeof window === "undefined" ? [] : readPersistedJobs(),
probe: () => {
void get().probeNow();
},
probeNow: () => {
probing ??= hostedFetch("/api/auth/get-session", { cache: "no-store" })
.then((r) => (r.ok ? r.json() : null))
.then((s) => {
const signedIn = Boolean((s as { user?: unknown } | null)?.user);
set({ signedIn });
return signedIn;
})
// A transient failure shouldn't knock a known session back to signed-out;
// only fall to false when we never learned otherwise.
.catch(() => {
const signedIn = get().signedIn ?? false;
set({ signedIn });
return signedIn;
})
.finally(() => {
probing = null;
});
return probing;
},
generateImage: (projectId, prompt, opts) => {
const backend = getBackend(); // pinned: the render outlives navigation
const job: GenerateJob = {
id: uid(),
projectId,
kind: "image",
prompt,
startedAt: Date.now(),
status: "running",
residency: backend.kind,
...(opts?.chatId ? { chatId: opts.chatId } : {}),
};
set((s) => ({ jobs: [job, ...s.jobs] }));
return (async () => {
try {
const aspect =
opts?.aspect ?? nearestAspect(useEditor.getState().aspect, IMAGE_ASPECTS);
const resolution = opts?.resolution ?? useImageGen.getState().resolution;
// The job (and the landed asset's name) keeps the user's own words;
// only the render sees the composed prompt. The image model takes
// several input images, so every kept reference rides along.
const { prompt: sent, images } = await promptAndImages(
"image",
prompt,
opts?.refs ?? [],
opts?.composeRefs !== false,
Infinity
);
const res = await hostedPost("/api/inference/assets", {
kind: "image",
prompt: sent,
...(images.length > 0 ? { inputs: { images } } : {}),
// The image model takes a real frame + detail via imageConfig; no prompt steering.
parameters: { aspectRatio: aspect, imageSize: resolution },
});
if (!res.ok) {
throw await readFailure(res, new UserError("image_generation_failed", "Image generation failed."));
}
const gen = (await res.json()) as GenerationResponse;
const out = gen.outputs.find((o) => o.dataBase64);
if (!out?.dataBase64) {
throw new UserError("image_output_missing", "The provider returned no image.");
}
// Mark it generated so it surfaces in the generate panel; a stock
// image imported from a tile sends no origin (plain import).
const asset = await uploadProjectImage(
projectId,
new Blob([bytesFromBase64(out.dataBase64)], {
type: out.contentType ?? "image/png",
}),
out.filename ?? "image.png",
{
name: promptName(prompt),
origin: "generated",
failMessage: new UserError("image_add_to_project_failed", "Could not add the image to the project."),
backend,
}
);
// Owner deleted while the image rendered (cancelForOwner dropped the
// job): drop the baked file instead of adopting orphaned media.
if (cancelledJobs.has(job.id) || !get().jobs.some((j) => j.id === job.id)) {
void backend
.fetch(`/api/cut/projects/${projectId}/media/${encodeURIComponent(asset.fileName)}`, {
method: "DELETE",
})
.catch(() => {});
return settled(job.id, job);
}
applyOwnership(asset, opts?.chatId);
if (adopt(projectId, asset, backend) && !opts?.chatId) {
useGenNotify.getState().landed("image", asset.id);
}
update(job.id, { status: "done", assetId: asset.id });
} catch (err) {
fail(job.id, err);
}
return settled(job.id, job);
})();
},
generateVideo: (projectId, prompt, opts) =>
get().generateVideoLadder(projectId, [{ prompt, opts }], {
...(opts?.chatId ? { chatId: opts.chatId } : {}),
...(opts?.genKey ? { genKey: opts.genKey } : {}),
...(opts?.onDone ? { onDone: opts.onDone } : {}),
}),
generateVideoLadder: (projectId, attempts, jobOpts) => {
// The job (and the landed asset's name) keeps the caller's first-rung
// words; only the render sees each rung's composed prompt.
const job: GenerateJob = {
id: uid(),
projectId,
kind: "video",
prompt: attempts[0]?.prompt ?? "",
startedAt: Date.now(),
status: "running",
residency: getBackend().kind, // pinned: the render outlives navigation
attempts,
...(jobOpts?.chatId ? { chatId: jobOpts.chatId } : {}),
...(jobOpts?.genKey ? { genKey: jobOpts.genKey } : {}),
};
set((s) => ({ jobs: [job, ...s.jobs] }));
claimJobLease(job.id); // this tab started it, this tab polls it
mirrorRender(job.id);
return runLadder(job, attempts, jobOpts);
},
retry: (id) => {
const job = get().jobs.find((j) => j.id === id);
if (!job || job.kind !== "video" || job.status !== "error") return;
// The job's own ladder resubmits; one persisted by a build that predates
// `attempts` re-runs as its bare prompt. Placement callbacks (onDone) are
// closures and are gone by retry time — the take still lands on this
// job's card or panel row, same as a reload-resumed render.
const attempts = (job.attempts?.length ? job.attempts : [{ prompt: job.prompt }]).map(
freshAttempt
);
update(id, {
status: "running",
startedAt: Date.now(),
error: undefined,
errorCode: undefined,
errorParams: undefined,
assetId: undefined,
rung: undefined,
poll: undefined,
attempts,
});
claimJobLease(id); // this tab retried it, this tab polls it
runLadder({ ...job, status: "running" }, attempts);
},
cancelForOwner: (owner) => {
const match = (j: GenerateJob) =>
(owner.chatId !== undefined && j.chatId === owner.chatId) ||
(owner.projectId !== undefined && j.projectId === owner.projectId);
const victims = get().jobs.filter(match);
if (victims.length === 0) return;
// Flag the running ones so their poll loop lands nothing (finishVideo reads
// cancelledJobs), then drop every matching job — running or settled, its
// card belonged to the owner the user just deleted.
for (const j of victims) {
cancelledJobs.add(j.id);
releaseJobLease(j.id);
}
set((s) => ({ jobs: s.jobs.filter((j) => !match(j)) }));
persistJobs(get().jobs);
// The mirrored records go with the owner — except on project deletion,
// where the whole doc is going away anyway.
if (owner.projectId === undefined) {
const byProject = new Map<string, GenerateJob[]>();
for (const j of victims) {
byProject.set(j.projectId, [...(byProject.get(j.projectId) ?? []), j]);
}
for (const [projectId, group] of byProject) unmirrorRenders(projectId, group);
}
},
resumeRunningJobs: () => {
const stored = new Map(readPersistedJobs().map((j) => [j.id, j]));
// Read each project's surviving thread ids once per sweep.
const threadCache = new Map<string, Set<string>>();
const threadIds = (projectId: string) => {
let ids = threadCache.get(projectId);
if (!ids) {
ids = readThreadIds(projectId);
threadCache.set(projectId, ids);
}
return ids;
};
for (const j of get().jobs) {
// A project's history loads from IndexedDB; until it has, a missing
// thread means unread, not deleted. Sweep again once it lands.
if (j.chatId && !threadsLoaded(j.projectId)) {
if (!threadLoads.has(j.projectId)) {
threadLoads.add(j.projectId);
void loadProjectThreads(j.projectId).then(() => get().resumeRunningJobs());
}
continue;
}
// A render whose owning thread no longer exists — the thread was deleted,
// or its whole project was (which clears the project's threads) — must
// never land: drop it so a reload can't resurrect media the user removed.
if (j.chatId && !threadIds(j.projectId).has(j.chatId)) {
get().dismiss(j.id);
continue;
}
if (j.kind !== "video" || j.status !== "running" || !j.poll || settlements.has(j.id)) continue;
// Storage is the cross-tab truth: a sibling tab may have settled this
// render already — adopt its outcome instead of re-polling (and
// re-importing) a finished job.
const theirs = stored.get(j.id);
if (theirs && theirs.status !== "running") {
set((s) => ({ jobs: s.jobs.map((x) => (x.id === j.id ? theirs : x)) }));
continue;
}
// Another live tab is already polling this render — landing it twice
// would import the file twice. The boot sweep re-runs on an interval,
// so if that tab closes, its lease stales and this one takes over.
if (!claimJobLease(j.id)) continue;
const poll = j.poll;
const run = (async () => {
try {
// Re-enter the poll loop where the last session left it. The
// timeline placement callback (onDone) is a closure and cannot
// survive a reload — the asset still lands on its card/panel.
await finishVideo(j.id, {
status: "in_progress",
outputs: [],
...poll,
} as GenerationResponse);
} catch (err) {
fail(j.id, err);
} finally {
releaseJobLease(j.id);
}
return settled(j.id, j);
})();
settlements.set(j.id, run);
}
},
dismiss: (id) => {
const victim = get().jobs.find((j) => j.id === id);
set((s) => ({ jobs: s.jobs.filter((j) => j.id !== id) }));
persistJobs(get().jobs);
if (victim) unmirrorRenders(victim.projectId, [victim]);
},
};
});
// Reload recovery: re-attach every persisted in-flight render as soon as the
// app boots, so a refresh never orphans credits already spent. The sweep
// repeats so a render whose owning tab closed (its lease staled) is picked up
// by a surviving tab instead of orphaned; resumeRunningJobs itself is
// idempotent and lease-guarded, so the repeats are cheap no-ops otherwise.
if (typeof window !== "undefined") {
setTimeout(() => useGenerate.getState().resumeRunningJobs(), 0);
setInterval(() => useGenerate.getState().resumeRunningJobs(), LEASE_TTL_MS);
}
/** Donkey sign-in state for the generation surfaces. Probes on mount and again
* whenever the tab regains focus, so signing in — in this tab's round trip or a
* separate tab — flips the UI without a manual reload. Null until first known. */
export function useSignedIn(): boolean | null {
const signedIn = useGenerate((s) => s.signedIn);
useEffect(() => {
const probe = () => useGenerate.getState().probe();
probe();
const onVisible = () => {
if (document.visibilityState === "visible") probe();
};
window.addEventListener("focus", probe);
document.addEventListener("visibilitychange", onVisible);
return () => {
window.removeEventListener("focus", probe);
document.removeEventListener("visibilitychange", onVisible);
};
}, []);
return signedIn;
}