Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

End local LTX music video clips on the shot's end frame

Claude08f53a0d81b9
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 92bdfa60d7..ecb97280ac 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -93,7 +93,7 @@ export default function SceneCard({
onMove, onDelete, onEditLocal, onSave,
onGenerateFrame, onGenerateVideo, onContinueVideo, onGenerateEndFrame, generatingEndFrame = false, onContinueFromPrevious, previousHasClip = false,
onOpenPreview, onSelectTake, onReviewTake, onImportTake, onImportClipTake, takeBusy = false, layered = false, documentComposition = false,
- lipSyncBackend = '', songDurationSec = null, onSplit, onMergeNext, nextEndSec = null, falVideoSettings = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,
+ lipSyncBackend = '', songDurationSec = null, onSplit, onMergeNext, nextEndSec = null, falVideoSettings = null, localVideoRuntime = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,
expanded, onToggleExpand, footageOptional = false, failedScenes = null, songReview = null,
}) {
const detailsRef = useRef(null);
@@ -473,11 +473,11 @@ export default function SceneCard({
onSelect={(take) => onSelectTake?.(scene, take)}
onReview={(take, review) => onReviewTake?.(scene, take, review)}
onOpenPreview={onOpenPreview} />
- {/* Style-break end frame (#11173) — what a fal clip ends on */}
+ {/* Style-break end frame (#11173) — what a fal or local LTX clip ends on */}
{onGenerateEndFrame && layer === 'footage' && (
<SceneEndFrame scene={scene} index={index} generating={!!generatingEndFrame} onGenerate={onGenerateEndFrame}
onEditLocal={onEditLocal} onSave={onSave} onOpenPreview={onOpenPreview} takeBusy={takeBusy}
- onSelectTake={onSelectTake} onReviewTake={onReviewTake} videoSettings={falVideoSettings} />
+ onSelectTake={onSelectTake} onReviewTake={onReviewTake} videoSettings={falVideoSettings} localVideoRuntime={localVideoRuntime} />
)}
{/* Render: the motion prompt, where this shot renders and what it costs, then its clip (Phase 1). */}
<section aria-label="Render" className="space-y-2 rounded border border-port-border p-2">
diff --git a/client/src/components/musicVideo/SceneEndFrame.jsx b/client/src/components/musicVideo/SceneEndFrame.jsx
index 3709326e2c..a45d5d11ca 100644
--- a/client/src/components/musicVideo/SceneEndFrame.jsx
+++ b/client/src/components/musicVideo/SceneEndFrame.jsx
@@ -7,17 +7,18 @@ import { sceneEndFramePlan } from '../../../../server/lib/musicVideoEndFrame.js'
* into from its photographic start frame, such as a pencil-sketch rotoscope or
* a magazine-cutout collage. The director describes it, renders it from the
* selected start frame, and picks among its takes like any frame. A fal clip on
- * a model that takes an end frame ends on it; the note says when the current
- * renderer will ignore it.
+ * a model that takes an end frame ends on it, and so does a local LTX clip
+ * (`localVideoRuntime`, the local model's runtime); the note says when the
+ * current renderer will ignore it.
*/
export default function SceneEndFrame({
scene, index, generating = false, onGenerate, onEditLocal, onSave, onOpenPreview,
- takeBusy = false, onSelectTake, onReviewTake, videoSettings = null,
+ takeBusy = false, onSelectTake, onReviewTake, videoSettings = null, localVideoRuntime = null,
}) {
const settings = videoSettings || {};
// Same mode the clip request is built with (useMusicVideoSceneMedia generateSceneVideo).
const mode = settings.generationMode === 'suppliedAudio' || (settings.backend === 'local' && settings.generationMode === 'audioReactive') ? 'a2v' : 'image';
- const plan = sceneEndFramePlan(scene, { backend: settings.backend || null, falModelId: settings.falModelId || null, mode });
+ const plan = sceneEndFramePlan(scene, { backend: settings.backend || null, falModelId: settings.falModelId || null, mode, localRuntime: localVideoRuntime });
const promptId = `mv-scene-${scene.sceneId}-end-frame-prompt`;
return (
<details className="rounded border border-port-border bg-port-bg/40 px-2 py-1.5" open={Boolean(scene.endImageId || scene.endFramePrompt)}>
diff --git a/client/src/components/musicVideo/SceneEndFrame.test.jsx b/client/src/components/musicVideo/SceneEndFrame.test.jsx
new file mode 100644
index 0000000000..68fd5c2049
--- /dev/null
+++ b/client/src/components/musicVideo/SceneEndFrame.test.jsx
@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { render, screen } from '@testing-library/react';
+import SceneEndFrame from './SceneEndFrame.jsx';
+
+const scene = { sceneId: 's1', referenceImageId: 'start.png', endImageId: 'end.png', endFramePrompt: 'head thrown back' };
+const renderFor = (localVideoRuntime) => render(
+ <SceneEndFrame scene={scene} index={0} onEditLocal={() => {}} onSave={() => {}}
+ videoSettings={{ backend: 'local' }} localVideoRuntime={localVideoRuntime} />,
+);
+
+// Uniquely catches the card telling the director a local LTX clip ignores the
+// end frame it now renders to (#11258), or promising it on a one-frame model.
+describe('SceneEndFrame on the local renderer', () => {
+ it('says a local LTX clip ends on the end frame', () => {
+ renderFor('ltx25');
+ expect(screen.getByText('The clip ends on this frame.')).toBeInTheDocument();
+ });
+
+ it('warns that a single-frame local model ignores it', () => {
+ renderFor('fastmetal');
+ expect(screen.getByText(/animates from the start frame only/)).toBeInTheDocument();
+ });
+});
diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsx
index cebdf5905c..539591adfe 100644
--- a/client/src/components/musicVideo/stages/BoardStage.jsx
+++ b/client/src/components/musicVideo/stages/BoardStage.jsx
@@ -200,6 +200,7 @@ export default function BoardStage({ board }) {
failedScenes={sceneMedia.failedScenes}
songReview={attentionCtx.songReview}
falVideoSettings={videoSettings.settings}
+ localVideoRuntime={videoSettings.activeModel?.runtime ?? null}
canContinueShot={board.canContinueShot}
onMove={board.moveScene}
onDelete={board.onDeleteScene}
diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.js
index 200c8c1475..b263e6ab17 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.js
+++ b/client/src/hooks/useMusicVideoSceneMedia.js
@@ -401,8 +401,11 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
: { grokDuration: settings.grokDuration, disableAudio: true }),
mode,
sourceImageFile: scene.referenceImageId,
- // A style-break end frame (#11173), when this fal model takes one.
- ...sceneEndFrameRequestFields(scene, { backend: settings.backend, falModelId: falFields?.falModelId, mode }),
+ // A style-break end frame (#11173), when this fal or local model takes one
+ // (a local LTX clip then renders first+last frame, #11258).
+ ...sceneEndFrameRequestFields(scene, {
+ backend: settings.backend, falModelId: falFields?.falModelId, mode, localRuntime: videoSettings.activeModel?.runtime ?? null,
+ }),
...(audioReactiveSelected ? {
audioStartSec: scene.startSec || 0,
loraFilenames: [detectedAudioReactiveLora.filename],
diff --git a/server/lib/musicVideoEndFrame.js b/server/lib/musicVideoEndFrame.js
index 9aeee2a317..b9978e94cd 100644
--- a/server/lib/musicVideoEndFrame.js
+++ b/server/lib/musicVideoEndFrame.js
@@ -3,36 +3,55 @@
*
* A scene may carry an optional end frame (`scene.endImageId`, the selected
* `endImage` take): a stylized still the clip moves into from its photographic
- * start frame, such as a pencil-sketch or magazine-cutout version. Only fal
- * models whose catalog entry takes an end image can honour it. Every other lane
- * ignores it, and this module says why, so the board and the production
- * dispatcher make the same call.
+ * start frame, such as a pencil-sketch or magazine-cutout version. fal models
+ * whose catalog entry takes an end image honour it, and so does a local model
+ * whose runtime anchors the last frame of a first+last-frame render (LTX-2.x,
+ * #11258). Every other lane ignores it, and this module says why, so the board
+ * and the production dispatcher make the same call.
*/
import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, getFalVideoModel } from './falVideoModels.js';
import { isPerformanceScene } from './musicVideoShotTiming.js';
+import { isLastFrameAnchoredRuntime } from './runners.js';
+import { VIDEO_RUNTIME_MODES } from './videoModeProfiles.js';
+
+// A local clip ends on the end frame through a first+last-frame render, so the
+// runtime must both run that mode and anchor its last frame (MiniMax H3 ref2va
+// anchors it but renders song-driven clips only).
+const localRuntimeEndsOnFrame = (runtime) => isLastFrameAnchoredRuntime(runtime)
+ && Boolean(VIDEO_RUNTIME_MODES[runtime]?.includes('fflf'));
const endImageOf = (scene) => (typeof scene?.endImageId === 'string' && scene.endImageId.trim() ? scene.endImageId : null);
/**
- * Whether a clip for `scene` on `backend` (with `falModelId` for fal, and the
- * request's video `mode`, image-to-video by default) sends the scene's end
- * frame. Returns `{ endImageId, used, reason }`: `reason` names why a selected
- * end frame is ignored, and is null when it is used or absent.
+ * Whether a clip for `scene` on `backend` (with `falModelId` for fal, the local
+ * model's `localRuntime` for local, and the request's video `mode`,
+ * image-to-video by default) sends the scene's end frame. Returns
+ * `{ endImageId, used, reason }`: `reason` names why a selected end frame is
+ * ignored, and is null when it is used or absent.
*/
-export function sceneEndFramePlan(scene, { backend = null, falModelId = null, mode = 'image' } = {}) {
+export function sceneEndFramePlan(scene, { backend = null, falModelId = null, mode = 'image', localRuntime = null } = {}) {
const endImageId = endImageOf(scene);
if (!endImageId) return { endImageId: null, used: false, reason: null };
const ignored = (reason) => ({ endImageId, used: false, reason });
if (isPerformanceScene(scene)) return ignored('A performance shot follows the song and takes no end frame');
if (mode !== 'image') return ignored('A song-driven clip takes no end frame');
- if (backend !== 'fal') return ignored('Only fal.ai models that take an end frame use it; this renderer ignores it');
+ if (backend === 'local') {
+ return localRuntimeEndsOnFrame(localRuntime)
+ ? { endImageId, used: true, reason: null }
+ : ignored('This local model animates from the start frame only; local LTX-2.5 ends on the end frame');
+ }
+ if (backend !== 'fal') return ignored('Only fal.ai models that take an end frame and local LTX use it; this renderer ignores it');
const model = getFalVideoModel(falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL);
if (!model?.endImage) return ignored(`${model?.label || 'This fal.ai model'} takes no end frame; choose one that does`);
return { endImageId, used: true, reason: null };
}
-/** The `/api/video-gen` field that carries the end frame, or nothing. */
+/**
+ * The `/api/video-gen` fields that carry the end frame, or nothing. A local
+ * render anchors it only in first+last-frame mode, so that lane also names it.
+ */
export const sceneEndFrameRequestFields = (scene, options) => {
const plan = sceneEndFramePlan(scene, options);
- return plan.used ? { lastImageFile: plan.endImageId } : {};
+ if (!plan.used) return {};
+ return options?.backend === 'local' ? { lastImageFile: plan.endImageId, mode: 'fflf' } : { lastImageFile: plan.endImageId };
};
diff --git a/server/lib/musicVideoEndFrame.test.js b/server/lib/musicVideoEndFrame.test.js
index 1a98e287ba..8fc7d892f1 100644
--- a/server/lib/musicVideoEndFrame.test.js
+++ b/server/lib/musicVideoEndFrame.test.js
@@ -10,7 +10,10 @@ describe('sceneEndFramePlan (#11173)', () => {
it.each([
['a performance shot', { ...scene, shotMode: 'performance' }, kling],
['a song-driven clip', scene, { ...kling, mode: 'a2v' }],
- ['a non-fal renderer', scene, { backend: 'local' }],
+ ['a renderer with no end-frame input', scene, { backend: 'grok' }],
+ ['a local model that animates from one frame', scene, { backend: 'local', localRuntime: 'fastmetal' }],
+ ['a local model it cannot identify', scene, { backend: 'local' }],
+ ['a local model that anchors a last frame only in song-driven clips', scene, { backend: 'local', localRuntime: 'minimax_h3_ref2va' }],
['a fal model with no end image', scene, { backend: 'fal', falModelId: 'fal-ai/veo3.1/fast/image-to-video' }],
])('ignores the end frame for %s and says why', (_label, s, options) => {
const plan = sceneEndFramePlan(s, options);
@@ -24,6 +27,14 @@ describe('sceneEndFramePlan (#11173)', () => {
expect(sceneEndFramePlan(scene, { backend: 'fal' })).toEqual({ endImageId: 'end.png', used: true, reason: null });
});
+ // #11258: a local render anchors the last frame only in first+last-frame mode.
+ it('sends it to a local model whose runtime anchors the last frame, in first+last-frame mode', () => {
+ for (const localRuntime of ['ltx25', 'ltx2']) {
+ expect(sceneEndFramePlan(scene, { backend: 'local', localRuntime })).toEqual({ endImageId: 'end.png', used: true, reason: null });
+ expect(sceneEndFrameRequestFields(scene, { backend: 'local', localRuntime })).toEqual({ lastImageFile: 'end.png', mode: 'fflf' });
+ }
+ });
+
it('reports nothing for a scene without an end frame', () => {
expect(sceneEndFramePlan({ sceneId: 's1', endImageId: ' ' }, kling)).toEqual({ endImageId: null, used: false, reason: null });
});
diff --git a/server/lib/runners.js b/server/lib/runners.js
index 3ea376c310..354d5a5bcc 100644
--- a/server/lib/runners.js
+++ b/server/lib/runners.js
@@ -68,6 +68,12 @@ export const isMiniMaxH3Runtime = (runtime) => MINIMAX_H3_RUNTIME_SET.has(runtim
export const LTX2_FAMILY_RUNTIMES = Object.freeze(['ltx2', 'ltx25']);
const LTX2_FAMILY_RUNTIME_SET = new Set(LTX2_FAMILY_RUNTIMES);
export const isLtx2FamilyRuntime = (runtime) => LTX2_FAMILY_RUNTIME_SET.has(runtime);
+// Runtimes whose first+last-frame (FFLF) render really anchors the LAST frame:
+// the LTX-2 family runs a keyframe-interpolation pipeline and both MiniMax H3
+// runtimes pack both frames as conditioning rows. Every other runtime keeps one
+// frame and drops the other. Re-exported by services/videoGen/runtimes.js.
+export const LAST_FRAME_ANCHORED_RUNTIMES = Object.freeze(new Set([...LTX2_FAMILY_RUNTIMES, ...MINIMAX_H3_RUNTIMES]));
+export const isLastFrameAnchoredRuntime = (runtime) => LAST_FRAME_ANCHORED_RUNTIMES.has(runtime);
export const AUDIO_TO_VIDEO_RUNTIMES = Object.freeze([
...LTX2_FAMILY_RUNTIMES,
MINIMAX_H3_REF2VA_RUNTIME,
diff --git a/server/services/musicVideo/productionDispatch.js b/server/services/musicVideo/productionDispatch.js
index 6f96f92c00..fe1f61d435 100644
--- a/server/services/musicVideo/productionDispatch.js
+++ b/server/services/musicVideo/productionDispatch.js
@@ -125,6 +125,11 @@ async function dispatchClip({ project, scene, route, tag }) {
fal: falClipParams(project, scene, route),
reactor: { disableAudio: true },
};
+ // A local model ends on the end frame only when its runtime anchors the last
+ // frame (#11258), so the plan needs the model this route actually renders on.
+ const localRuntime = route.mode === 'local' && scene.endImageId
+ ? (await (await import('../videoGen/modelSelection.js')).resolveVideoModelSelection(route.model || undefined)).model?.runtime ?? null
+ : null;
const { submitVideoGenJob } = await import('../videoGen/submitJob.js');
// The submit service runs the revision guard itself as its last step before
// the queue write (submitJob.js), like every board-submitted clip.
@@ -134,8 +139,10 @@ async function dispatchClip({ project, scene, route, tag }) {
mode: 'image',
sourceImageFile: scene.referenceImageId,
...(byBackend[route.mode] || {}),
- // A style-break end frame (#11173), when this fal model takes one.
- ...sceneEndFrameRequestFields(scene, { backend: route.mode, falModelId: byBackend.fal?.falModelId }),
+ // A style-break end frame (#11173), when this fal or local model takes one.
+ ...sceneEndFrameRequestFields(scene, {
+ backend: route.mode, falModelId: byBackend.fal?.falModelId, mode: byBackend[route.mode]?.mode || 'image', localRuntime,
+ }),
musicVideo: tag,
}, {});
return { jobId };
diff --git a/server/services/musicVideo/productionDispatch.test.js b/server/services/musicVideo/productionDispatch.test.js
index d365565e72..1f8b576b3d 100644
--- a/server/services/musicVideo/productionDispatch.test.js
+++ b/server/services/musicVideo/productionDispatch.test.js
@@ -14,6 +14,12 @@ const enqueueJob = vi.fn(async () => ({ jobId: 'job-image' }));
const submitVideoGenJob = vi.fn(async () => ({ jobId: 'job-video' }));
vi.mock('../mediaJobQueue/index.js', () => ({ enqueueJob: (...a) => enqueueJob(...a) }));
vi.mock('../videoGen/submitJob.js', () => ({ submitVideoGenJob: (...a) => submitVideoGenJob(...a) }));
+// Which local model a route renders on: an LTX-2.5 pin, else the install default.
+vi.mock('../videoGen/modelSelection.js', () => ({
+ resolveVideoModelSelection: async (modelId) => (modelId === 'ltx25_example'
+ ? { modelId, model: { id: modelId, runtime: 'ltx25' } }
+ : { modelId: 'fastmetal_example', model: { id: 'fastmetal_example', runtime: 'fastmetal' } }),
+}));
vi.mock('./productionService.js', () => ({ assertProductionSubmission: vi.fn(async () => {}) }));
const assertRevisionOpen = vi.fn(async () => {});
vi.mock('./revisionService.js', () => ({ assertRevisionOpen: (...a) => assertRevisionOpen(...a) }));
@@ -196,5 +202,16 @@ describe('production scene dispatch (#9066)', () => {
await dispatchProductionStep({ stepKind: 'clip', project: onModel('fal-ai/kling-video/v3/pro/image-to-video'), scene, route: fal, tag: clipTag, settings });
for (const call of submitVideoGenJob.mock.calls.slice(1)) expect(call[0]).not.toHaveProperty('lastImageFile');
});
-});
+ // Uniquely catches a local LTX clip that drops a shot's end frame (#11258),
+ // or one sent to a single-frame local model that would ignore it anyway.
+ it('ends a local clip on the end frame only on a model that anchors the last frame', async () => {
+ const clipTag = { ...tag, productionStepKey: 'clip:mvs-a:base:1' };
+ const styleBreak = { ...scene, endImageId: 'end-example.png' };
+ await dispatchProductionStep({ stepKind: 'clip', project, scene: styleBreak, route: { kind: 'video', mode: 'local', model: 'ltx25_example' }, tag: clipTag, settings });
+ expect(submitVideoGenJob.mock.calls[0][0]).toMatchObject({ backend: 'local', mode: 'fflf', sourceImageFile: 'frame-example.png', lastImageFile: 'end-example.png' });
+ await dispatchProductionStep({ stepKind: 'clip', project, scene: styleBreak, route: { kind: 'video', mode: 'local', model: null }, tag: clipTag, settings });
+ expect(submitVideoGenJob.mock.calls[1][0]).toMatchObject({ backend: 'local', mode: 'image' });
+ expect(submitVideoGenJob.mock.calls[1][0]).not.toHaveProperty('lastImageFile');
+ });
+});
diff --git a/server/services/videoGen/runtimes.js b/server/services/videoGen/runtimes.js
index c28afdc433..005b598ca5 100644
--- a/server/services/videoGen/runtimes.js
+++ b/server/services/videoGen/runtimes.js
@@ -17,7 +17,7 @@ import { PATHS } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
import { createSingleFlight } from '../../lib/singleFlight.js';
-import { MINIMAX_H3_RUNTIMES, LTX2_FAMILY_RUNTIMES } from '../../lib/runners.js';
+import { LAST_FRAME_ANCHORED_RUNTIMES } from '../../lib/runners.js';
// Path to the dgrauet/ltx-2-mlx venv populated by `INSTALL_LTX2=1
// scripts/setup-image-video.sh`. Used when a model entry has
@@ -395,7 +395,7 @@ export const routesToWindowsHelper = (model) => (
// resize in local.js (wasted ffmpeg work otherwise), and the client's
// "last frame is advisory" note, which reads it off the model payload via
// `lastFrameAnchored`.
-export const LAST_FRAME_ANCHORED_RUNTIMES = Object.freeze(new Set([...LTX2_FAMILY_RUNTIMES, ...MINIMAX_H3_RUNTIMES]));
+export { LAST_FRAME_ANCHORED_RUNTIMES };
export const modelAnchorsLastFrame = (model) => LAST_FRAME_ANCHORED_RUNTIMES.has(model?.runtime);