Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Merge remote-tracking branch 'origin/main' into claude/fal-audio-image-to-video-b6ozol

Claudebd96b5d52582
diff --cc client/src/components/musicVideo/SceneCard.jsx
index 0f9109110e,1183925608..c8d7ed291e
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@@ -517,13 -471,12 +476,12 @@@ export default function SceneCard(
</button>
</div>
)}
- <button onClick={() => onGenerateVideo(scene)}
- disabled={settingsSaving || (performance && (speakerDirty || speakerSaving)) || !scene.referenceImageId || !!generatingVideo || !!videoBlockedReason || !!performanceBlocked}
+ <button onClick={generateVideo}
+ disabled={!!videoRefusal}
className="flex items-center gap-1 bg-port-border hover:bg-port-border/70 disabled:opacity-50 rounded px-2 py-1.5 text-xs min-h-[44px] sm:min-h-0 whitespace-nowrap"
- title={videoBlockedReason || performanceBlocked
- || (!scene.referenceImageId ? 'Generate a reference frame first'
- : performance ? `${capability.style === 'music' ? "Animate this scene's frame to the song" : "Lip-sync this scene's frame to the song"} via ${capability.label} — ${performanceCost}`
- : "Generate this scene's video from its reference frame (i2v)")}>
+ title={videoRefusal
- || (performance ? `Lip-sync this scene's frame to the song via ${capability.label} — ${performanceCost}`
++ || (performance ? `${capability.style === 'music' ? "Animate this scene's frame to the song" : "Lip-sync this scene's frame to the song"} via ${capability.label} — ${performanceCost}`
+ : "Generate this scene's video from its reference frame (i2v)")}>
{generatingVideo ? <Activity size={14} className="animate-spin" /> : <Video size={14} />}
{generatingVideo ? 'Generating video…' : (scene.videoHistoryId ? 'New video take' : 'Generate video')}
</button>
diff --cc client/src/components/musicVideo/SceneCard.test.jsx
index e9f710fdd5,112427f2b3..2e54b6e68f
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@@ -48,29 -48,55 +48,78 @@@ describe('performance speaker save boun
});
});
+describe('music-driven performance', () => {
+ it('switches a shot between cutaway, lip-sync and music-driven, and plans the music-driven take on the audio-to-video model', () => {
+ const onSave = vi.fn();
+ function Harness() {
+ const [scene, setScene] = useState({ sceneId: 's1', shotMode: 'cutaway', startSec: 10, endSec: 13.5, referenceImageId: 'frame.png', takes: [] });
+ return <SceneCard scene={scene} index={0} lipSyncBackend="fal" songDurationSec={60} falVideoSettings={{}}
+ onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
+ }
+ render(<Harness />);
+ const shot = screen.getByLabelText('Shot');
+ fireEvent.change(shot, { target: { value: 'music' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: 'music' });
+ expect(shot.value).toBe('music');
+ // The shot's own 3.5s, not lip-sync's 5s minimum window.
+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Music-driven via fal\.ai LTX-2\.3 audio-to-video \(fal-ai\/ltx-2\.3-22b\/audio-to-video\) · song 0:10–0:13 · est\. \$0\.30 \(3\.5s at 1080p\)/);
+ fireEvent.change(shot, { target: { value: 'performance' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: null });
+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Lip-sync via/);
+ fireEvent.change(shot, { target: { value: 'cutaway' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'cutaway' });
+ });
+});
+
+ describe('failed video retry behind the speaker save gate (#11040)', () => {
+ function Harness({ onSave, onGenerateVideo, onGenerateFrame = vi.fn(), lastFailure }) {
+ const [scene, setScene] = useState({ sceneId: 's1', shotMode: 'performance', startSec: 0, endSec: 5, referenceImageId: 'frame.png', takes: [], lastFailure });
+ return <SceneCard scene={scene} index={0} lipSyncBackend="fal" songDurationSec={30}
+ onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave}
+ onGenerateVideo={onGenerateVideo} onGenerateFrame={onGenerateFrame} />;
+ }
+ const videoFailure = { lane: 'video', error: 'Example failure', at: '2026-10-05T00:00:00.000Z' };
+
+ it('refuses the video Retry while the speaker edit is dirty, saving, or failed to save, then recovers', async () => {
+ let settle;
+ const onSave = vi.fn(() => new Promise((resolve) => { settle = resolve; }));
+ const onGenerateVideo = vi.fn();
+ render(<Harness onSave={onSave} onGenerateVideo={onGenerateVideo} lastFailure={videoFailure} />);
+ const input = screen.getByLabelText('Speaker / singer');
+ const retry = screen.getByRole('button', { name: 'Retry' });
+ const generate = screen.getByRole('button', { name: 'Generate video' });
+ expect(retry.disabled).toBe(false);
+ fireEvent.change(input, { target: { value: 'Example Singer' } });
+ expect([generate.disabled, retry.disabled]).toEqual([true, true]);
+ fireEvent.click(retry);
+ fireEvent.blur(input);
+ expect([generate.disabled, retry.disabled]).toEqual([true, true]);
+ fireEvent.click(retry);
+ settle(false);
+ await waitFor(() => expect(onSave).toHaveBeenCalledTimes(1));
+ expect([generate.disabled, retry.disabled]).toEqual([true, true]);
+ fireEvent.click(retry);
+ expect(retry.title).toBe(generate.title);
+ expect(onGenerateVideo).not.toHaveBeenCalled();
+ fireEvent.blur(input);
+ settle(true);
+ await waitFor(() => expect(retry.disabled).toBe(false));
+ fireEvent.click(retry);
+ expect(onGenerateVideo).toHaveBeenCalledTimes(1);
+ });
+
+ it('leaves the frame Retry available while the speaker edit is unsaved', () => {
+ const onGenerateFrame = vi.fn();
+ const onGenerateVideo = vi.fn();
+ render(<Harness onSave={vi.fn()} onGenerateVideo={onGenerateVideo} onGenerateFrame={onGenerateFrame}
+ lastFailure={{ ...videoFailure, lane: 'image' }} />);
+ fireEvent.change(screen.getByLabelText('Speaker / singer'), { target: { value: 'Example Singer' } });
+ fireEvent.click(screen.getByRole('button', { name: 'Retry' }));
+ expect(onGenerateFrame).toHaveBeenCalledTimes(1);
+ expect(onGenerateVideo).not.toHaveBeenCalled();
+ });
+ });
+
describe('persisted render failure chip (#10154)', () => {
const failedScene = (lane) => ({ sceneId: 's1', startSec: 0, endSec: 5, referenceImageId: 'frame.png', takes: [],
lastFailure: { lane, error: 'CUDA out of memory', at: '2026-10-05T00:00:00.000Z' } });
diff --cc client/src/components/musicVideo/SceneTimingPlan.jsx
index 0000000000,b5fe2ab960..100caebdec
mode 000000,100644..100644
--- a/client/src/components/musicVideo/SceneTimingPlan.jsx
+++ b/client/src/components/musicVideo/SceneTimingPlan.jsx
@@@ -1,0 -1,53 +1,53 @@@
+ import { AlertTriangle, Merge, Scissors } from 'lucide-react';
+ import { formatDurationSec } from '../../utils/formatters.js';
+ import { getFalVideoModel } from '../../lib/falVideoModels.js';
+
+ /** Stateless provider plan and scene split/merge controls, without wrapper DOM. */
+ export default function SceneTimingPlan({ timing, sceneId, lipSyncBackend, onSplit, onMergeNext }) {
+ const {
+ performance, performanceBlocked, capability, plan, performanceCost,
+ falTake, falCost, grokPlan, timedSpan, canSplit, splitLimit, canMerge,
+ } = timing;
+ return (
+ <>
+ {performance && performanceBlocked && (
+ <div role="alert" className="flex items-start gap-2 rounded border border-port-warning/40 bg-port-warning/10 px-2 py-1.5 text-xs text-port-warning">
+ <AlertTriangle size={13} className="shrink-0 mt-0.5" />
+ <span className="min-w-0 break-words">{performanceBlocked}</span>
+ </div>
+ )}
+ {performance && !performanceBlocked && (
+ <p className="text-[11px] text-port-text-muted break-words" data-testid="performance-plan">
- Lip-sync via {capability.label} ({capability.modelId}) · song {formatDurationSec(plan.windowStartSec)}–{formatDurationSec(plan.windowEndSec)}
++ {capability.style === 'music' ? 'Music-driven' : 'Lip-sync'} via {capability.label} ({capability.modelId}) · song {formatDurationSec(plan.windowStartSec)}–{formatDurationSec(plan.windowEndSec)}
+ {plan.editInSec > 0 ? ` · shot starts ${plan.editInSec.toFixed(2)}s into the take` : ''} · {performanceCost}
+ </p>
+ )}
+ {falTake && !falTake.performance && (
+ <p className="text-[11px] text-port-text-muted break-words" data-testid="fal-cutaway-plan">
+ fal.ai renders {falTake.seconds != null ? `a ${falTake.seconds}s ` : 'a '}clip on {getFalVideoModel(falTake.modelId)?.label || falTake.modelId}
+ {' · '}{falCost || 'cost unknown — billed to your fal.ai account'}
+ </p>
+ )}
+ {grokPlan && (
+ <p className={`text-[11px] break-words ${grokPlan.needsSplit ? 'text-port-warning' : 'text-port-text-muted'}`}>
+ Grok renders a {grokPlan.requestSec}s clip for this {timedSpan.toFixed(1)}s cutaway
+ {grokPlan.needsSplit ? ` — ${grokPlan.uncoveredSec.toFixed(1)}s uncovered; split the scene rather than loop it` : ''}. Motion timing in the prompt is approximate.
+ </p>
+ )}
+ {canSplit && onSplit && (
+ <button type="button" onClick={() => onSplit(sceneId, lipSyncBackend || null)}
+ className="inline-flex items-center gap-1 rounded bg-port-bg border border-port-border hover:bg-port-border/40 px-2 py-1 text-xs min-h-[44px] sm:min-h-0"
+ title={`Cut this ${timedSpan.toFixed(1)}s shot into scenes of at most ${splitLimit.toFixed(2)}s at lyric pauses or phrase boundaries — nothing is looped or stretched`}>
+ <Scissors size={13} /> Split on lyric boundaries
+ </button>
+ )}
+ {canMerge && (
+ <button type="button" onClick={() => onMergeNext(sceneId, lipSyncBackend || null)}
+ className="inline-flex items-center gap-1 rounded bg-port-bg border border-port-border hover:bg-port-border/40 px-2 py-1 text-xs min-h-[44px] sm:min-h-0"
+ title="Join this shot with the next one into a single shot — the inverse of split">
+ <Merge size={13} /> Merge with next
+ </button>
+ )}
+ </>
+ );
+ }
diff --cc client/src/lib/README.md
index 866f9a6dca,0ae0faee8c..218ef6c509
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@@ -66,7 -66,8 +66,9 @@@ grep -i "what you want to do" client/sr
| `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoSceneAttention.js` | Music Video per-scene attention model (#10152): `sceneAttention(scene, ctx)` returns the codes (`missing-frame`, `missing-clip`, `under-covered`, `contract-problem`, `perf-blocked`, `candidates-pending`, `plate-unverified`, `no-span`, `last-failure`) behind the collapsed scene-row chips, the Board `?scenes=attention\|missing\|all` filter (`parseSceneFilter`, `sceneMatchesFilter`) and the Produce per-shot strip; `SCENE_ATTENTION_LABELS`. Used by `components/musicVideo/SceneCard.jsx`, `stages/BoardStage.jsx` and `stages/ProduceStage.jsx`. |
| `musicVideoCoverOverlay.js` | Cover-art lettering preview (#10345): re-export of `server/lib/musicVideoCoverOverlay.js` (the design vocabulary, `normalizeCoverDesign`, `coverOverlaySvg`) plus `canvasCoverWidths(design, texts, fonts)`, which measures text with a canvas so the live preview wraps and sizes the title the way the server's measured render does. |
+| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes and performance styles, the verified source-audio lip-sync and music-driven capability tables, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. |
+ | `musicVideoSceneTiming.js` | `buildSceneTimingPresentation`: pure provider timing, cost, coverage and split/merge projection for the Music Video scene card, reusing canonical shot-timing policy. |
+ | `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes, the verified source-audio lip-sync capability table, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. |
| `musicVideoStages.js` | Music Video page stage model: `MUSIC_VIDEO_STAGES` (six steps: Song `setup` → Look `cast-sets` → Storyboard `board` → Make `produce` → Final render `review` → Publish; a retired `compose` link resolves to Make), `resolveStageParam(value)` for the `:stage` route param, `deriveStages(project)` (per-stage `done`/`active`/`blocked`/`todo` + `current`, the first incomplete step; a live production run pins Make), `deriveNextAction(project, ctx)` (the one primary action the sticky header offers — approve Cast & Sets, verify lyric timing, stop/resume production, render final, …), `describeProjectStatus` (the header line and the step that needs the director), `autopilotStatus` (the header Autopilot button), `projectSpend(project)` (paid generation used vs. cap), `projectShotSummary(project)` (document shots or scene counts), `stageChecklist(stageId, project, readiness)` (what one step still needs, with the open approval's reason and a scroll anchor; an `optional` item, such as Make's animated proof, is listed but not counted), `summarizeStoryboardProblems(problems)` (folds the server's one-sentence-per-shot storyboard problems into one count line per kind), `stepNotes(project, readiness, publish)` (one fact per step for the step list), `listPreviewSources` (final, live composition, drafts, then the storyboard animatic), `projectServicesSummary(project)` (the saved image/video render services), `compareMusicVideoProjectsNewestFirst(a, b)` (sort projects newest created / updated first for project cards and pickers), and `currentProductionRun`; `productionReviewStopGuidance` separates current approval blockers from historical run stops. Used by `components/musicVideo/MusicVideoLayout.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoAttention.js` | Music Video "Needs attention" model (#9940): `deriveAttentionItems(project, live)` returns the banner rows for server-held states a client-orchestrated workflow can strand — an open/rendering revision, an interrupted Cast & Sets stage, a running auto-review waiting on its revised sections, a final render nobody is watching — derived from the persisted record so they survive a reload (healthy work this tab can see progressing is not flagged); `openRevisionOf(project)`; `ATTENTION_ANCHOR_ID` + `revealAttention()` scroll a refusal toast to the banner. Used by `components/musicVideo/NeedsAttentionBanner.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoProjectList.js` | Music Video project index model (#10167): `projectRunPill(project)` (Running / Needs you / Interrupted / Done, from the same next-action and attention logic as the header), `groupMusicVideoProjects(projects, query)` (collapse forks under their root, newest version first, name filter, groups ordered by last touch), and `compareMusicVideoProjectsRecentlyTouched`. Used by `pages/MusicVideo.jsx` and `components/musicVideo/MusicVideoProjectCard.jsx`. |
diff --cc client/src/lib/musicVideoSceneTiming.js
index 0000000000,9cef3bd31e..781899644a
mode 000000,100644..100644
--- a/client/src/lib/musicVideoSceneTiming.js
+++ b/client/src/lib/musicVideoSceneTiming.js
@@@ -1,0 -1,44 +1,44 @@@
+ import { formatUsd } from '../utils/formatters.js';
+ import {
+ falSceneTake, grokCoverage, isPerformanceScene, performanceBlockedReason, performanceCapability, planPerformanceWindow, shotSplitLimit,
+ } from './musicVideoShotTiming.js';
+
+ // "est. $0.81 (5.05s at 1080P)" — or null when the take cannot be priced.
+ const falEstimate = (take) => (take?.costUsd == null ? null
+ : `est. ${formatUsd(take.costUsd)} (${take.seconds}s${take.resolution ? ` at ${take.resolution}` : ''})`);
+
+ /**
+ * Provider timing and cost presentation for one scene. Domain policy remains
+ * in the canonical shot-timing helpers shared with media submission.
+ */
+ export function buildSceneTimingPresentation({
+ scene, layer, lipSyncBackend, songDurationSec, falVideoSettings,
+ nextEndSec, isLast, documentComposition, hasMergeNext,
+ }) {
+ const performance = isPerformanceScene(scene);
- const capability = performance ? performanceCapability(lipSyncBackend) : null;
++ const capability = performance ? performanceCapability(lipSyncBackend, { scene, videoSettings: falVideoSettings }) : null;
+ const timedSpan = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' && scene.endSec > scene.startSec
+ ? scene.endSec - scene.startSec
+ : null;
+ const plan = capability
+ ? planPerformanceWindow({ startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability })
+ : null;
+ const performanceBlocked = performance
+ ? (capability ? (plan.ok ? null : plan.message) : performanceBlockedReason(lipSyncBackend))
+ : null;
+ const grokPlan = !performance && lipSyncBackend === 'grok' && timedSpan != null ? grokCoverage(timedSpan) : null;
+ const falTake = lipSyncBackend === 'fal' && layer === 'footage'
+ ? falSceneTake({ scene, videoSettings: falVideoSettings || {}, songDurationSec })
+ : null;
+ const falCost = falEstimate(falTake);
+ const performanceCost = falTake?.performance ? (falCost || capability?.costLabel) : capability?.costLabel;
- const splitLimit = layer === 'footage' ? shotSplitLimit(scene, lipSyncBackend) : null;
++ const splitLimit = layer === 'footage' ? shotSplitLimit(scene, lipSyncBackend, falVideoSettings) : null;
+ const canSplit = splitLimit != null && timedSpan != null && timedSpan > splitLimit + 1e-6;
+ const mergedSpan = typeof scene.startSec === 'number' && typeof nextEndSec === 'number' ? nextEndSec - scene.startSec : null;
+ const canMerge = hasMergeNext && !isLast && !documentComposition
+ && (splitLimit == null || (mergedSpan != null && mergedSpan <= splitLimit + 1e-6));
+ return {
+ performance, capability, timedSpan, plan, performanceBlocked, grokPlan,
+ falTake, falCost, performanceCost, splitLimit, canSplit, mergedSpan, canMerge,
+ };
+ }