Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { normalizeVideoFailure } from '../../lib/videoFailure.js';
/** Multi-chunk local-video orchestration. */
import { join } from 'path';
import { randomUUID } from 'crypto';
import { tmpdir } from 'os';
import { PATHS, unlinkGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { probeFrameCount, trimVideoFromFrame } from '../../lib/ffmpeg.js';
import {
resolveContextFrames, resolveContinuityStrategy, extendLatentFrames,
contextPrefixFrames, tailWindowStartFrame,
} from '../../lib/videoContinuity.js';
import { broadcastSse, closeJobAfterDelay } from '../../lib/sseUtils.js';
import { videoGenEvents } from './events.js';
import { loadHistory, mutateVideoHistory } from './history.js';
import { videoChainUnsupportedError } from './modeContract.js';
import { estimateRenderMs } from './eta.js';
import {
resolveVideoSpeedProfileForModes, speedProfileDeclineReasonForModes,
resolveVideoSampler, inferEffectiveVideoMode,
} from '../../lib/videoSpeedProfiles.js';
import { videoJobState } from './jobState.js';
import { DEFAULT_NUM_FRAMES, resolveVideoDimensions } from './renderArgs.js';
import { extractLastFrame } from './frameExtraction.js';
import { stitchVideos } from './stitchVideos.js';
import { generateVideo, resolveVideoModel, defaultVideoModelId } from './generateVideo.js';
// Generate a chain of N video chunks, each conditioned on the one before it,
// then stitch them into a single longer clip. Reports progress + terminal
// events against the OUTER jobId (so the mediaJobQueue's dispatcher sees one
// logical job through the chain) while each inner chunk runs as a normal
// generateVideo() with its own inner jobId, file, and history entry.
//
// Continuation strategy (`lib/videoContinuity.js`, tunable via
// `contextFrames`):
//
// 'window' — chunk N+1 is an `extend` render conditioned on a clip cut from
// the last `contextFrames` frames of chunk N, so the model inherits the
// scene's MOTION, not just its final pose. Requires a runtime with an
// extend pipeline (ltx2 today) and applies whatever mode the chain started
// in. Because extend returns `source + extension`, each windowed chunk
// opens with an echo of the window; that echo is measured and dropped
// inside the stitch's concat filter graph, so the timeline holds each
// frame exactly once while the chunk files stay as the model rendered
// them.
// 'frame' — the historical hop: extract chunk N's last frame and run chunk
// N+1 as image-to-video off that still. What every other runtime gets, and
// what `contextFrames: 0` opts back into.
//
// On completion the inner chunk entries are hidden so only the stitched clip
// is visible by default; the user can toggle hidden in the gallery to
// inspect individual chunks.
//
// On cancel the chain stops before the next chunk; the in-flight chunk's
// child is SIGTERM'd by cancel() and surfaces a 'failed' event we translate
// into a chain-level failure. Already-completed inner chunks are hidden but
// not deleted (the partial output is still on disk if the user wants it).
//
// `chunkPrompts` (#3695) optionally steers each chunk individually: entry i is
// chunk i's prompt, and a null/blank/missing entry falls back to the main
// `prompt`. It is destructured out of `rest` on purpose so the per-chunk
// generateVideo() calls below never receive the whole list.
export async function generateChainedVideo({ chunks, chunkPrompts, contextFrames, jobId: outerJobId, ...rest }) {
if (Number(chunks) > 1 && Number(rest.batchSize) > 1) {
throw new ServerError('A render batch cannot be combined with chained clips.', { status: 400, code: 'VIDEO_BATCH_CONFLICT' });
}
const totalChunks = Number(chunks) || 1;
if (totalChunks === 1) {
return generateVideo({ jobId: outerJobId, ...rest });
}
if (!outerJobId) throw new ServerError('generateChainedVideo requires jobId', { status: 500, code: 'INTERNAL' });
const chainModel = resolveVideoModel(rest.modelId || defaultVideoModelId());
const chainError = videoChainUnsupportedError(chainModel);
if (chainError) throw chainError;
// How chunk N+1 sees chunk N. 'window' hands LTX-2's extend pipeline the
// prior chunk's last `windowFrames` frames (motion + appearance); 'frame'
// is the historical single-still i2v hop, and is what every runtime without
// an extend pipeline resolves to. See lib/videoContinuity.js.
const windowFrames = resolveContextFrames(contextFrames);
const continuity = resolveContinuityStrategy({ model: chainModel, contextFrames: windowFrames });
// Mirrors generateVideo's own default — the trim math converts frame indices
// to audio timestamps, so it can't run on an undefined rate.
const chainFps = Number(rest.fps) > 0 ? Number(rest.fps) : 24;
// Latents each chained chunk asks ExtendPipeline to append. buildLtx2Args
// derives the same number from the same numFrames; the trim below subtracts
// the pixel frames they decode to, so the two have to stay in step —
// INCLUDING when the caller omitted numFrames. `numFrames` is optional on
// the route, and generateVideo resolves the same default before deriving
// `--extend-frames`; resolving it there but not here made the orchestrator
// assume an 8-frame extension against a ~120-frame one, so the prefix trim
// would have kept 8 frames of each hop and thrown the rest away.
// Optional-chained on purpose: `videoChainUnsupportedError` waves an
// unresolvable model through (an unknown runtime resolves to the base mode
// set, which includes 'image'), so a model removed between enqueue and
// dispatch reaches here as null. Falling back keeps the chain's first chunk
// the thing that reports it — generateVideo throws a 400 "Unknown video
// model: X" — instead of a null-deref out of the dispatcher.
const chunkExtendLatents = extendLatentFrames(
rest.numFrames ?? chainModel?.defaultFrames ?? DEFAULT_NUM_FRAMES,
);
const chainState = { stopped: false };
videoJobState.activeChain = chainState;
// Hold an outer job entry so attachSseClient(outerJobId) wires up against
// the same SSE stream the queue sees. Without this, /api/video-gen/:id/events
// attached at the outer id would 404 because no shared job-map entry exists.
const outerJob = { id: outerJobId, clients: [], status: 'running' };
videoJobState.jobs.set(outerJobId, outerJob);
// Chain-level wall-clock estimate (#3801). Every chunk is a full render that
// pays the fixed per-render cost again, so the chain estimate is the
// per-chunk estimate times the chunk count — `chunks` is handed to the
// estimator rather than folding the chain into one oversized render.
// The dimension/step defaults mirror generateVideo's model-aware resolution
// and samplerLocked step rule; the estimator returns null on anything it
// can't resolve, so malformed custom registry defaults degrade to the legacy
// canvas rather than producing a wrong or non-finite estimate.
// Deliberately placed AFTER the active chain and shared job are registered: it is the
// first await in this function, and a cancel arriving during the history
// read must still find the chain to stop.
// A chain's chunks do NOT all run in the request's mode: chunk 0 keeps it,
// and chunks 1+ re-enter as `extend` on a window-continuity chain or `image`
// on a frame hop (see the dispatch below — these two must stay in step).
// `extend` routes through ExtendPipeline, which no two-stage speed profile is
// validated for, so a Fast chain would render chunk 0 fast and the rest at
// the model default: a visible seam mid-clip, plus a chain ETA claiming a
// speed-up most of the render never takes. resolveVideoSpeedProfileForModes
// therefore applies the profile to the whole chain or to none of it, and
// `speedProfileId` is stripped from `rest` below so the per-chunk renders
// agree with this decision instead of each re-deciding.
const chainChunkModes = [
inferEffectiveVideoMode(rest),
...(totalChunks > 1 ? [continuity === 'window' ? 'extend' : 'image'] : []),
];
const chainSpeedProfile = resolveVideoSpeedProfileForModes({
model: chainModel, profileId: rest.speedProfileId, modes: chainChunkModes,
});
const chainSpeedDeclined = chainSpeedProfile ? null : speedProfileDeclineReasonForModes({
model: chainModel, profileId: rest.speedProfileId, modes: chainChunkModes,
});
if (chainSpeedDeclined) {
console.log(`⚠️ Speed profile declined for chain [${outerJobId.slice(0, 8)}] ${chainSpeedDeclined.code}: ${chainSpeedDeclined.message}`);
}
// What every chunk will actually be handed. Absent (not 'quality') when the
// chain declined, so a chunk stamps no profile it didn't render with.
const chainSpeedProfileId = chainSpeedProfile ? rest.speedProfileId : undefined;
// `resolveVideoSampler` is the SAME precedence rule generateVideo applies, so
// the chain estimate can't drift from what the chunks actually render at.
const { steps: chainSteps } = resolveVideoSampler({
model: chainModel, steps: rest.steps, guidanceScale: rest.guidanceScale, speedProfile: chainSpeedProfile,
});
const chainDimensions = resolveVideoDimensions(chainModel, rest.width, rest.height);
const chainEta = estimateRenderMs({
history: await loadHistory(),
modelId: rest.modelId || defaultVideoModelId(),
width: chainDimensions.width,
height: chainDimensions.height,
numFrames: rest.numFrames ?? chainModel?.defaultFrames ?? DEFAULT_NUM_FRAMES,
steps: chainSteps,
speedProfileId: chainSpeedProfile?.id ?? null,
chunks: totalChunks,
});
const chainEtaField = chainEta ? { etaMs: chainEta.etaMs } : {};
console.log(`🎬 Chained video [${outerJobId.slice(0, 8)}]: ${totalChunks} chunks, eta=${chainEta ? `${Math.round(chainEta.etaMs / 1000)}s (${chainEta.basis}, n=${chainEta.sampleCount})` : 'unknown'}`);
const chunkIds = [];
let currentSource = rest.sourceImagePath;
// 'window' continuation: the tail slice of the prior chunk that the next one
// conditions on. A fresh short clip per hop, NOT the prior chunk's whole
// output — extend_from_video returns `source + extension`, so conditioning
// on the full clip would make every chunk re-contain the one before it (the
// stitch then repeats that content once per hop) while the conditioning cost
// grew with the chain. Written under tmpdir and deleted when the chain ends.
let currentContextClip = null;
const contextClipPaths = [];
// Echoed-context cut for each chunk that has one: where the chunk's own new
// footage starts, and how many frames it therefore contributes. Handed to
// stitchVideos, which applies the cuts in its concat filter graph — the
// chunk files themselves are never rewritten, so the chain pays exactly one
// encode (the stitch) instead of one per chunk plus the stitch.
const chunkTrims = new Map();
// First chunk always preserves the user's mode (text, image, fflf or extend)
// and is never trimmed: in an extend chain its output is `source clip +
// extension`, and the source clip belongs in the result exactly once — here.
// Chunks 1+ take the resolved continuity path instead.
const firstMode = rest.mode || (currentSource ? 'image' : 'text');
const runChunk = (i) => new Promise((resolve, reject) => {
const innerJobId = randomUUID();
chunkIds.push(innerJobId);
const onProgress = (e) => {
if (e.generationId !== innerJobId) return;
const innerProg = typeof e.progress === 'number' ? e.progress : 0;
const aggregate = (i + Math.max(0, Math.min(1, innerProg))) / totalChunks;
videoGenEvents.emit('progress', {
generationId: outerJobId,
progress: aggregate,
step: typeof e.step === 'number' ? e.step : undefined,
totalSteps: typeof e.totalSteps === 'number' ? e.totalSteps : undefined,
message: `Chunk ${i + 1}/${totalChunks}${e.message ? ` — ${e.message}` : ''}`,
// Chain-level estimate, NOT the inner chunk's — the outer id's
// consumers are watching the whole chain's clock.
...chainEtaField,
// The chunk's own render phase, so a chained render names its step the
// same way a single-shot one does instead of falling back to the
// page's load/render heuristic for the whole chain (#5872).
phase: typeof e.phase === 'string' && e.phase ? e.phase : undefined,
});
broadcastSse(outerJob, {
type: 'progress',
progress: aggregate,
message: `Chunk ${i + 1}/${totalChunks}`,
});
};
const detach = () => {
videoGenEvents.off('progress', onProgress);
videoGenEvents.off('completed', onCompleted);
videoGenEvents.off('failed', onFailed);
};
const onCompleted = (e) => {
if (e.generationId !== innerJobId) return;
detach();
resolve(e);
};
const onFailed = (e) => {
if (e.generationId !== innerJobId) return;
detach();
resolve({ error: e.error || 'chunk failed', failure: e.failure });
};
videoGenEvents.on('progress', onProgress);
videoGenEvents.on('completed', onCompleted);
videoGenEvents.on('failed', onFailed);
// Bump the seed by chunk index when the user supplied one — keeps each
// chunk visually varied while remaining reproducible from the user's
// chosen seed. When seed is unset, generateVideo picks one randomly
// per chunk (existing behavior).
const chunkSeed = rest.seed != null && rest.seed !== ''
? Number(rest.seed) + i
: undefined;
// Chunks 1+ on a window-continuity chain re-enter as extend renders
// conditioned on the tail clip built after the previous chunk, whatever
// mode the chain STARTED in — a text or i2v chain gets the same motion
// carry-over an extend chain does, instead of restarting from a still.
const isWindowHop = continuity === 'window' && i > 0;
// Per-chunk beat (#3695). `chunkPrompts` is already normalized (blank →
// null) by prepareVideoGenParams, but re-guard on emptiness here too so a
// direct service caller can't hand a chunk an empty prompt — an absent beat
// must always resolve to the main prompt, never to ''.
const beat = chunkPrompts?.[i];
const chunkPrompt = (typeof beat === 'string' && beat.trim() !== '') ? beat : rest.prompt;
generateVideo({
...rest,
// Decided ONCE for the chain above, not re-decided per chunk: `undefined`
// when any chunk's mode declines the profile, so every chunk renders on
// the same sampler and the stitched clip has no mid-clip seam.
speedProfileId: chainSpeedProfileId,
prompt: chunkPrompt,
seed: chunkSeed,
jobId: innerJobId,
// window hop: condition on the tail clip cut from the prior chunk, so
// the model reads motion out of it rather than a single static pose.
// frame hop: condition on the prior chunk's extracted last frame.
sourceImagePath: isWindowHop ? null : currentSource,
extendFromVideoPath: isWindowHop ? currentContextClip : (i === 0 ? rest.extendFromVideoPath : null),
// Only the first chunk consumes the user's uploadedTempPath (durable
// copy under data/uploads). Later chunks condition on the prior chunk
// instead — a tail window (window hop) or an extracted frame (frame hop).
uploadedTempPath: i === 0 ? rest.uploadedTempPath : null,
uploadedTempPaths: i === 0 ? (rest.uploadedTempPaths || []) : [],
hidden: true,
// Chunk 0 of an extend chain consumes the user's own clip, so its render
// keeps that clip instead of the model's decode of it (#11179).
keepExtendSource: i === 0 && firstMode === 'extend',
mode: isWindowHop ? 'extend' : (i === 0 ? firstMode : 'image'),
// After the first chunk, drop FFLF-style last image — chained
// continuation conditions on one thing, the tail of the chunk before it,
// and has no second anchor to pin an end frame to.
lastImagePath: i === 0 ? rest.lastImagePath : null,
// Multi-keyframe interpolation only makes sense for the first chunk
// (the user pinned specific frame indices in a single clip). Subsequent
// chunks fall through to the image-chain path, conditioning on the
// prior chunk's tail frame.
keyframes: i === 0 ? rest.keyframes : null,
// The reference-mode promise (#4874) belongs to the first chunk alone. Every
// continuation conditions on the PRIOR chunk's tail — a still or a window
// clip the chain produced, not something the user offered as inspiration —
// and loosening that hop would break the seam it exists to hold. A window hop
// isn't image mode at all, so carrying the mode forward would also fail the
// gate outright.
i2vReferenceMode: i === 0 ? rest.i2vReferenceMode : null,
}).catch((err) => {
detach();
reject(err);
});
});
// The conditioning windows are scratch input to the next chunk, never
// output — drop them on every terminal path (both of which always run) so a
// long chain doesn't leave a trail of clips in tmpdir. Best-effort and
// fire-and-forget: a leftover temp file must never fail a finished render.
const cleanupContextClips = () => {
for (const p of contextClipPaths) unlinkGuarded(p).catch(() => {});
contextClipPaths.length = 0;
};
const finishOk = (payload) => {
if (videoJobState.activeChain === chainState) videoJobState.activeChain = null;
cleanupContextClips();
videoGenEvents.emit('completed', { generationId: outerJobId, ...payload });
broadcastSse(outerJob, { type: 'complete', result: payload });
closeJobAfterDelay(videoJobState.jobs, outerJobId);
};
const failureOptions = { prompts: [rest.prompt, rest.negativePrompt, ...(chunkPrompts || [])] };
const finishFail = (error, failure = normalizeVideoFailure(error, failureOptions)) => {
if (videoJobState.activeChain === chainState) videoJobState.activeChain = null;
cleanupContextClips();
videoGenEvents.emit('failed', { generationId: outerJobId, error, failure });
broadcastSse(outerJob, { type: 'error', error });
closeJobAfterDelay(videoJobState.jobs, outerJobId);
};
// Schedule the chain on the next tick and return the descriptor
// synchronously — matches generateVideo's spawn-then-emit contract.
(async () => {
for (let i = 0; i < totalChunks; i++) {
if (chainState.stopped) {
await setHistoryItemsHidden(chunkIds, true);
finishFail('Canceled mid-chain');
return;
}
// eslint-disable-next-line no-await-in-loop
const completed = await runChunk(i).catch((err) => ({ error: err.message, failure: normalizeVideoFailure(err, failureOptions) }));
if (completed?.error) {
await setHistoryItemsHidden(chunkIds, true);
finishFail(completed.error, completed.failure);
return;
}
// The chunk's output file is always <innerJobId>.mp4 under PATHS.videos
// (see generateVideo: filename = `${jobId}.mp4`).
const chunkId = chunkIds[chunkIds.length - 1];
const chunkPath = join(PATHS.videos, `${chunkId}.mp4`);
if (continuity === 'window') {
// One probe serves both cuts below. Neither rewrites chunkPath, so the
// count stays valid for both — worth keeping to one call because
// probeFrameCount falls back to a full decode pass when the container
// header carries no nb_frames.
// eslint-disable-next-line no-await-in-loop
const frames = await probeFrameCount(chunkPath);
// A window hop's render is `context window + new frames`. The window is
// the tail of the chunk before it, which the stitched timeline already
// holds, so record where the new footage starts and let the stitch drop
// the echo in its filter graph. Measured from the RENDERED length rather
// than from the window we supplied: the VAE snaps the encoded context up
// to a latent boundary, so the echo is usually a few frames longer than
// what we handed in.
let contextPrefix = 0;
if (i > 0) {
contextPrefix = contextPrefixFrames({ totalFrames: frames, extendLatents: chunkExtendLatents });
if (contextPrefix <= 0) {
console.log(`⚠️ Chunk ${i + 1}/${totalChunks} context prefix unmeasurable (frames=${frames ?? 'unknown'}), leaving it untrimmed`);
}
}
// Record the measurement even when there's nothing to cut (chunk 0, or
// an unmeasurable prefix). An extend render's file is `source +
// extension`, so a chunk's own history `numFrames` — the count it was
// REQUESTED at — understates what it contributes to the timeline, and
// that's the only other number the stitch could fall back on.
if (frames != null) chunkTrims.set(chunkId, { startFrame: contextPrefix, frames: frames - contextPrefix });
if (i < totalChunks - 1) {
// Cut the next hop's conditioning window off this chunk's tail. Both
// the count and the cut clamp: a window longer than the chunk simply
// conditions on all of it.
//
// An unprobeable length clamps the same way, which means the whole
// chunk becomes the window — the unbounded conditioning this exists
// to avoid, for one hop. It self-corrects (the next chunk's prefix
// trim measures the echo off the render, not off the window), but say
// so rather than letting the fallback be silent.
if (frames == null) {
console.log(`⚠️ Chunk ${i + 1}/${totalChunks} length unprobeable — conditioning the next chunk on the whole clip instead of a ${windowFrames}-frame window`);
}
const contextPath = join(tmpdir(), `chaincontext-${chunkId}.mp4`);
// eslint-disable-next-line no-await-in-loop
const cut = await trimVideoFromFrame(chunkPath, contextPath, {
// Floored at the echo the stitch is going to drop: the window must
// come from this chunk's OWN footage, never from the replay of the
// one before it. (The chunk file still holds that replay now that
// the cut happens at stitch time.)
startFrame: Math.max(contextPrefix, tailWindowStartFrame({ totalFrames: frames, frames: windowFrames })),
fps: chainFps,
});
if (!cut.ok) {
await setHistoryItemsHidden(chunkIds, true);
finishFail(`Failed to build the continuation context window between chunks: ${cut.reason}`);
return;
}
contextClipPaths.push(contextPath);
currentContextClip = contextPath;
}
} else if (i < totalChunks - 1) {
// extractLastFrame caches by id, so re-clicks (e.g. from gallery
// "Continue") don't re-spawn ffmpeg.
// eslint-disable-next-line no-await-in-loop
const frame = await extractLastFrame(chunkId).catch((err) => ({ error: err.message }));
if (frame?.error) {
await setHistoryItemsHidden(chunkIds, true);
finishFail(`Failed to extract frame between chunks: ${frame.error}`);
return;
}
currentSource = join(PATHS.images, frame.filename);
}
}
const stitched = await stitchVideos(chunkIds, {
id: outerJobId,
filenamePrefix: 'chained',
historyKey: 'chainedFrom',
promptOverride: rest.prompt || null,
// Persist the beats alongside the stitched clip so the gallery entry (and
// a Remix off it) carries the same source of truth the chain rendered
// from — the individual chunk entries only ever hold their own resolved
// prompt, which loses which of them were explicit beats vs. fallbacks.
chunkPrompts: chunkPrompts?.some(Boolean) ? chunkPrompts : null,
// Echoed-context prefixes to cut out of each windowed chunk as the
// timeline is assembled, rather than by pre-encoding the chunk files.
trims: chunkTrims.size ? chunkTrims : null,
}).catch((err) => ({ error: err.message }));
if (stitched?.error) {
await setHistoryItemsHidden(chunkIds, true);
finishFail(`Stitch failed: ${stitched.error}`);
return;
}
await setHistoryItemsHidden(chunkIds, true);
finishOk({
filename: stitched.filename,
thumbnail: stitched.thumbnail,
path: `/data/videos/${stitched.filename}`,
chainedFrom: chunkIds,
});
})().catch((err) => {
console.error(`❌ chain orchestration crashed [${outerJobId.slice(0, 8)}]: ${err.message}`);
finishFail(err.message, normalizeVideoFailure(err, failureOptions));
});
// Match the synchronous shape of generateVideo so the route's response
// assembly doesn't need a chain-specific branch. The actual filename is
// delivered via SSE 'complete' once the chain settles.
return {
jobId: outerJobId,
generationId: outerJobId,
filename: `chained-${outerJobId}.mp4`,
mode: 'local',
model: rest.modelId,
};
}
// Hide many history entries in one load+save. The per-id setHistoryItemHidden
// would re-read + atomic-write the entire history file once per id; for an
// 8-chunk chain that's 16 file ops on every terminal path. Best-effort —
// errors are swallowed because the stitched clip is more important than
// the visibility flag.
async function setHistoryItemsHidden(ids, hidden) {
if (!ids?.length) return;
const wanted = new Set(ids);
// Serialized through the shared history tail so a concurrent write path
// (a download completing, a render finalizing) can't clobber this update.
await mutateVideoHistory((history) => {
for (const item of history) {
if (wanted.has(item.id)) item.hidden = !!hidden;
}
return history;
}).catch(() => {});
}