Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Lip-sync takes line up with the song when the provider pads the clip
Claudea82cbd8a2860
diff --git a/client/src/components/musicVideo/PerformanceEvidence.jsx b/client/src/components/musicVideo/PerformanceEvidence.jsxindex 567269838d..aa9ed6f6fb 100644--- a/client/src/components/musicVideo/PerformanceEvidence.jsx+++ b/client/src/components/musicVideo/PerformanceEvidence.jsx@@ -1,5 +1,6 @@import { useEffect, useRef, useState } from 'react';import { formatTimecode, formatUsd } from '../../utils/formatters.js';+import { clipAudioOffsetSec } from '../../lib/musicVideoShotTiming.js';/** One transport: muted output picture against the take's recorded source.* Word windows are clip-relative; temporal evidence is excerpt-relative.@@ -13,6 +14,8 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = nconst filename = conditioning?.filename;const source = filename ? `/data/music/${encodeURIComponent(filename)}` : null;const windowStart = instruction?.audioWindow?.startSec;+ // Word windows and the source are window time; an aligned clip's window starts this far in.+ const offset = clipAudioOffsetSec(instruction);const canSync = source && Number.isFinite(windowStart);const cues = Array.isArray(instruction?.cues) ? instruction.cues : [];@@ -25,7 +28,7 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = nconst sync = () => {if (!canSync || !audio.current || !video.current) return;- const target = windowStart + video.current.currentTime;+ const target = windowStart + video.current.currentTime - offset;if (Math.abs(audio.current.currentTime - target) > 0.05) audio.current.currentTime = target;};const pause = () => audio.current?.pause();@@ -55,10 +58,10 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n{error && <p role="status" className="text-port-warning">{error}</p>}<div className="flex flex-wrap gap-1" aria-label="Word windows">{cues.flatMap((cue, index) => (cue.words?.length ? cue.words : [cue]).map((word, i) => (- <button type="button" key={`${index}-${i}`} onClick={() => seek(word.startSec)}+ <button type="button" key={`${index}-${i}`} onClick={() => seek(word.startSec + offset)}className="rounded border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0"- title={`${formatTimecode(word.startSec)}–${formatTimecode(word.endSec)} (clip time)`}>- {word.text} · {formatTimecode(word.startSec)}–{formatTimecode(word.endSec)}+ title={`${formatTimecode(word.startSec + offset)}–${formatTimecode(word.endSec + offset)} (clip time)`}>+ {word.text} · {formatTimecode(word.startSec + offset)}–{formatTimecode(word.endSec + offset)}</button>)))}</div>diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex 9cb38a13bc..d831e4ce90 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -67,7 +67,7 @@ grep -i "what you want to do" client/src/lib/README.md| `musicVideoSceneAttention.js` | Music Video per-scene attention model (#10152): `sceneAttention(scene, ctx)` returns the codes (`missing-frame`, `missing-clip`, `under-covered`, `contract-problem`, `perf-blocked`, `candidates-pending`, `plate-unverified`, `no-span`, `last-failure`) behind the collapsed scene-row chips, the Board `?scenes=attention\|missing\|all` filter (`parseSceneFilter`, `sceneMatchesFilter`) and the Produce per-shot strip; `SCENE_ATTENTION_LABELS`; `sceneMediaKind` (`clip` / `image` / `code` / `card`) and `SCENE_MEDIA_LABELS` for the Board thumbnail badges. Used by `components/musicVideo/SceneCard.jsx`, `stages/BoardStage.jsx` and `stages/ProduceStage.jsx`. || `musicVideoCoverOverlay.js` | Cover-art lettering preview (#10345): re-export of `server/lib/musicVideoCoverOverlay.js` (the design vocabulary, `normalizeCoverDesign`, `coverOverlaySvg`) plus `canvasCoverWidths(design, texts, fonts)`, which measures text with a canvas so the live preview wraps and sizes the title the way the server's measured render does. || `musicVideoSceneTiming.js` | `buildSceneTimingPresentation`: pure provider timing, cost, coverage and split/merge projection for the Music Video scene card, reusing canonical shot-timing policy. |-| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes and performance styles, the verified source-audio lip-sync and music-driven capability tables, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. |+| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes and performance styles, the verified source-audio lip-sync and music-driven capability tables, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). `clipAudioOffsetSec` (how far into an aligned take's clip its audio window starts, #11210). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. || `musicVideoStages.js` | Music Video page stage model: `MUSIC_VIDEO_STAGES` (six steps: Song `setup` → Look `cast-sets` → Storyboard `board` → Make `produce` → Final render `review` → Publish; a retired `compose` link resolves to Make), `resolveStageParam(value)` for the `:stage` route param, `deriveStages(project)` (per-stage `done`/`active`/`blocked`/`todo` + `current`, the first incomplete step; a live production run pins Make), `deriveNextAction(project, ctx)` (the one primary action the sticky header offers — approve Cast & Sets, verify lyric timing, stop/resume production, render final, …), `describeProjectStatus` (the header line and the step that needs the director), `autopilotStatus` (the header Autopilot button), `projectSpend(project)` (paid generation used vs. cap), `projectShotSummary(project)` (document shots or scene counts), `stageChecklist(stageId, project, readiness)` (what one step still needs, with the open approval's reason and a scroll anchor; an `optional` item, such as Make's animated proof, is listed but not counted), `summarizeStoryboardProblems(problems)` (folds the server's one-sentence-per-shot storyboard problems into one count line per kind), `stepNotes(project, readiness, publish)` (one fact per step for the step list), `listPreviewSources` (final, live composition, drafts, then the storyboard animatic), `projectServicesSummary(project)` (the saved image/video render services), `compareMusicVideoProjectsNewestFirst(a, b)` (sort projects newest created / updated first for project cards and pickers), and `currentProductionRun`; `productionReviewStopGuidance` separates current approval blockers from historical run stops. Used by `components/musicVideo/MusicVideoLayout.jsx` and `pages/MusicVideo.jsx`. || `musicVideoAttention.js` | Music Video "Needs attention" model (#9940): `deriveAttentionItems(project, live)` returns the banner rows for server-held states a client-orchestrated workflow can strand — an open/rendering revision, an interrupted Cast & Sets stage, a running auto-review waiting on its revised sections, a final render nobody is watching — derived from the persisted record so they survive a reload (healthy work this tab can see progressing is not flagged); `openRevisionOf(project)`; `ATTENTION_ANCHOR_ID` + `revealAttention()` scroll a refusal toast to the banner. Used by `components/musicVideo/NeedsAttentionBanner.jsx` and `pages/MusicVideo.jsx`. || `musicVideoProjectList.js` | Music Video project index model (#10167): `projectRunPill(project)` (Running / Needs you / Interrupted / Done, from the same next-action and attention logic as the header), `groupMusicVideoProjects(projects, query)` (collapse forks under their root, newest version first, name filter, groups ordered by last touch), and `compareMusicVideoProjectsRecentlyTouched`. Used by `pages/MusicVideo.jsx` and `components/musicVideo/MusicVideoProjectCard.jsx`. |diff --git a/client/src/lib/musicVideoShotTiming.js b/client/src/lib/musicVideoShotTiming.jsindex 33d8cc89c3..6c1529ba3c 100644--- a/client/src/lib/musicVideoShotTiming.js+++ b/client/src/lib/musicVideoShotTiming.js@@ -11,6 +11,7 @@ export {SOURCE_AUDIO_LIPSYNC,SOURCE_AUDIO_MUSIC,approximateMotionCues,+ clipAudioOffsetSec,falSceneTake,falTakeRequestFields,grokCoverage,diff --git a/server/lib/README.md b/server/lib/README.mdindex 6f5806a06a..e75a0cfb72 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -198,7 +198,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `comicScriptParser.js` | Marvel/DC-format comic script parser. || `aspectRatio.js` | Describe a pixel canvas as the ratio phrase a prompt can use — `aspectRatioTerms(w, h)` (bounded best rational approximation, so 1096x1536 reads as 5:7 rather than 137:192) and `aspectRatioPhrase(w, h)` → `"5:7 portrait"` / `"16:9 landscape"` / `"1:1 square"`. Use it wherever a prompt has to state a framing the canvas already fixes, so the sentence and the canvas cannot drift. Pure. || `audioBeatDsp.js` | Shared beat/tempo DSP core (#9424) used by both `beatGrid.js` and `services/musicVideo/audioAnalysis.js`: `decodeAudioToPcm` (ffmpeg → mono f32 PCM, capped at `MAX_ANALYSIS_SEC`), Hann/FFT helpers, `onsetEnvelope` (spectral flux + RMS energy), `estimateTempo` (full-track autocorrelation over 70-180 BPM with a windowed, clustered fallback for tracks with a variable intro/outro), `fitBeats`, `pickDownbeats`. Exported as the `audioBeatDsp` namespace from the barrel. |-| `audioFingerprint.js` | Loudness fingerprint of a stretch of a song (#9266): `computeRmsEnvelope(path)` decodes a file once to a 100 Hz integer-dBFS RMS envelope on the song's own frame grid, `windowFingerprint(envelope, { startSec, endSec })` stores a window of it (base64 Int8Array, ≤3 KB for 30 s), and `compareWindowFingerprint(fp, envelope)` says whether a newer master still carries that window (Pearson ≥ 0.98 and mean \|ΔdB\| ≤ 1.5 above a −60 dB codec-noise floor). Lets a lip-sync take survive a re-master that changed audio outside its window. |+| `audioFingerprint.js` | Loudness fingerprint of a stretch of a song (#9266): `computeRmsEnvelope(path)` decodes a file once to a 100 Hz integer-dBFS RMS envelope on the song's own frame grid, `windowFingerprint(envelope, { startSec, endSec })` stores a window of it (base64 Int8Array, ≤3 KB for 30 s), and `compareWindowFingerprint(fp, envelope)` says whether a newer master still carries that window (Pearson ≥ 0.98 and mean \|ΔdB\| ≤ 1.5 above a −60 dB codec-noise floor). Lets a lip-sync take survive a re-master that changed audio outside its window. `envelopeOffsetSec(reference, candidate)` cross-correlates two envelopes and returns `{ offsetSec, correlation }` (where the reference's sound starts inside the candidate, refined below a frame) or null on a weak match — used to line a padded provider clip up with its source window (#11210). || `composeStyledPrompt.js` | Compose user prompt + negative with an optional style preset, or an array of presets layered in order. Re-exported (unmodified) by `client/src/lib/composeStyledPrompt.js`. || `scriptVideoCompiler.js` | Compiles a scene script + bible into beat-level continuous-video clip specs (duration/frame-grid snapping, byte-stable descriptor injection, chain-length fresh-cut rule). || `creativeDirectorPresets.js` | Locked-at-creation aspect ratio + quality presets for the Creative Director. |@@ -326,7 +326,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `grokVideoClip.js` | Clip lengths grok's `image_to_video` actually delivers — `GROK_VIDEO_DURATIONS` (`[6, 10]`, measured in #3022: a 2s/3s request returns the same 6.04s clip), `GROK_VIDEO_DEFAULT_DURATION`, `resolveGrokDuration()` (validate an already-grok-shaped request, else default), and `nearestGrokDuration()` (round a length authored against another backend's continuous contract UP to the shortest clip that covers it — for translating a CD/commission `targetDurationSeconds`). Dependency-free so `validation.js` / `routes/videoGen.js` can derive their schemas from the same list the services gate on. || `fontMetadata.js` | `fontContainer(buf)` (sfnt / woff2 / null) and `fontFamilyName(buf)` — a font file's family name read from its `name` table (TrueType, OpenType, and WOFF2's brotli stream) with no font library; null for anything unreadable (#10345). || `musicVideoCoverOverlay.js` | Music Video cover lettering (#10345): `COVER_DESIGN_OPTIONS` / `DEFAULT_COVER_DESIGN`, `normalizeCoverDesign(design, { fonts })` (accepts every stored design; a `font:<id>` typeface only when that uploaded font is listed), `coverOverlaySvg({ title, tag, design, widths, fonts })` and the `coverFontSpec` / `fontAttrs` / `titleLinesFor` / `tagTextFor` helpers the width probe reads. Pure; the server render (`coverArtCompose.js`) and the client's live Lettering preview run this one layout. |-| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `MUSIC_VIDEO_PERFORMANCE_STYLES` (`lipsync`/`music`) + `isLipSyncScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `SOURCE_AUDIO_MUSIC` (music-driven lanes — fal.ai audio-to-video models, picked by `videoSettings.falMusicModelId`, default OmniHuman v1.5 at 1080p), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend, { scene, videoSettings })` (lip-sync lane, or the music-driven one for a music-driven scene), `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. |+| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `MUSIC_VIDEO_PERFORMANCE_STYLES` (`lipsync`/`music`) + `isLipSyncScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `clipAudioOffsetSec(instruction)` (provider lead padding a landed take's edit points were moved by, #11210), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `SOURCE_AUDIO_MUSIC` (music-driven lanes — fal.ai audio-to-video models, picked by `videoSettings.falMusicModelId`, default OmniHuman v1.5 at 1080p), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend, { scene, videoSettings })` (lip-sync lane, or the music-driven one for a music-driven scene), `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. || `reactorStartingFrame.js` | Fits a reactor.inc `starting_frame` to the fast-h3 session canvas before it is uploaded. `readImageSize(path)` → `{width,height}` or `null` (never a guessed dimension), and `prepareReactorStartingFrame(sourceImagePath, requestedAspect, outputPath)` → `{ aspect, canvas, framePath, fittedPath }`: an absent `requestedAspect` DERIVES the canvas from the frame's own shape (`nearestReactorAspect`), then centre cover-crops the frame to that canvas's exact pixel size so reactor's own "fitted to the session canvas" step is a no-op. `fittedPath` is the temp file the caller deletes; `null` means the original was uploaded unchanged. A frame sharp cannot decode (or a resize that fails) passes through untouched on the default canvas rather than failing a render PortOS merely could not measure — reactor decodes more containers than sharp does. || `reactorVideoClip.js` | The reactor.inc `fast-h3` clip contract, dependency-free so the route schema, the FableLoom prompt compiler and the VideoGen picker all read one copy — `REACTOR_MAX_PROMPT_LENGTH` (800; the API rejects a longer prompt rather than truncating it), `REACTOR_MIN_CLIP_SECONDS` / `REACTOR_MAX_CLIP_SECONDS` (5.167–14.375, i.e. 124–345 frames at the `REACTOR_CLIP_FPS` 24 it delivers), `REACTOR_MAX_CLIP_ID_LENGTH`, and the picker list `REACTOR_CLIP_LENGTHS` + `REACTOR_DEFAULT_CLIP_LENGTH` + `reactorClipLengthLabel()` (the two exact endpoints plus every whole second between them, since the accepted range is continuous but starts and ends off a round number). Also the CANVAS contract `set_canvas` accepts: `REACTOR_CANVASES` (`16:9` 1344×768, `4:3` 1024×768, `1:1` 768×768, `9:16` 768×1344 — every one a 768px short edge, so the aspect string is the whole choice), `REACTOR_ASPECTS`, `REACTOR_DEFAULT_ASPECT`, `reactorCanvas(aspect)` and `nearestReactorAspect(width, height)` (closest canvas in log-ratio space; anything unmeasurable answers the default rather than deriving one). Mirrored to `client/src/lib/reactorVideoClip.js`, parity pinned by `reactorVideoClip.test.js`. || `kimi.js` | Moonshot AI Kimi Code (`kimi`) provider helpers — id constants (`KIMI_CLI_ID`/`KIMI_TUI_ID`), `isKimiCommand`/`isKimiCliProvider`/`isKimiTuiProvider` predicates, `ensureKimiHeadlessArgs` (model flag only — non-interactive mode is implicit in `--prompt`, and kimi has no `--print`/`--afk` and refuses approval flags alongside `--prompt`) / `ensureKimiTuiArgs` (`--yolo`) argv builders (model selection uses the `KIMI_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Grok/Antigravity), and `prepareKimiPrompt` (delivers the prompt as the `--prompt <value>` argv, not stdin). |diff --git a/server/lib/audioFingerprint.js b/server/lib/audioFingerprint.jsindex 889b03780c..0620d62937 100644--- a/server/lib/audioFingerprint.js+++ b/server/lib/audioFingerprint.js@@ -161,3 +161,63 @@ export function compareWindowFingerprint(fingerprint, envelope) {const same = !localChange && meanDeltaDb <= FINGERPRINT_MAX_MEAN_DELTA_DB && (correlation == null || correlation >= FINGERPRINT_MIN_CORRELATION);return { same, correlation, meanDeltaDb };}++// A lip-sync provider's lead padding is a fraction of a second; a clip that+// starts early is rarer and smaller. Searching only this span keeps one+// syllable from matching a repeat of itself a bar away.+const OFFSET_MIN_LAG_SEC = -0.5;+const OFFSET_MAX_LAG_SEC = 1.5;+// Below this the two contours are not the same performance shifted in time.+export const OFFSET_MIN_CORRELATION = 0.6;++/**+ * Where `reference`'s sound sits inside `candidate`: the lag (seconds) at which+ * `candidate[t + lag]` best matches `reference[t]`, by Pearson correlation of+ * the two 100 Hz envelopes above the codec-noise floor, refined to a fraction+ * of a frame. Positive = the candidate's sound starts later (leading padding).+ * Returns `{ offsetSec, correlation }`, or null when nothing matches well+ * enough (a flat or silent window, a different performance).+ */+export function envelopeOffsetSec(reference, candidate, { minLagSec = OFFSET_MIN_LAG_SEC, maxLagSec = OFFSET_MAX_LAG_SEC } = {}) {+ const ref = Float64Array.from(reference, (v) => Math.max(COMPARE_FLOOR_DB, v));+ const cand = Float64Array.from(candidate, (v) => Math.max(COMPARE_FLOOR_DB, v));+ const minOverlap = Math.max(FINGERPRINT_RATE_HZ, Math.ceil(ref.length * 0.6));+ const scores = new Map();+ const score = (lag) => {+ const from = Math.max(0, -lag);+ const to = Math.min(ref.length, cand.length - lag);+ const n = to - from;+ if (n < minOverlap) return null;+ let sumA = 0;+ let sumB = 0;+ for (let i = from; i < to; i++) { sumA += ref[i]; sumB += cand[i + lag]; }+ const meanA = sumA / n;+ const meanB = sumB / n;+ let cov = 0;+ let varA = 0;+ let varB = 0;+ for (let i = from; i < to; i++) {+ const a = ref[i] - meanA;+ const b = cand[i + lag] - meanB;+ cov += a * b;+ varA += a * a;+ varB += b * b;+ }+ if (Math.sqrt(varA / n) < MIN_SHAPE_STD_DB || Math.sqrt(varB / n) < MIN_SHAPE_STD_DB) return null;+ return cov / Math.sqrt(varA * varB);+ };+ let best = null;+ for (let lag = Math.round(minLagSec * FINGERPRINT_RATE_HZ); lag <= Math.round(maxLagSec * FINGERPRINT_RATE_HZ); lag++) {+ const r = score(lag);+ scores.set(lag, r);+ if (r != null && (best == null || r > best.r)) best = { lag, r };+ }+ if (!best || best.r < OFFSET_MIN_CORRELATION) return null;+ // Parabolic peak through the neighbors: the envelope is 10 ms frames, a+ // video frame is ~42 ms, so the refinement only keeps rounding honest.+ const left = scores.get(best.lag - 1);+ const right = scores.get(best.lag + 1);+ const curve = left != null && right != null ? left - 2 * best.r + right : 0;+ const shift = curve < 0 ? Math.max(-0.5, Math.min(0.5, (left - right) / (2 * curve))) : 0;+ return { offsetSec: Math.round((best.lag + shift) / FINGERPRINT_RATE_HZ * 1000) / 1000, correlation: Math.round(best.r * 1000) / 1000 };+}diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.jsindex af8d856ecc..4442bfd7ae 100644--- a/server/lib/musicVideoShotTiming.js+++ b/server/lib/musicVideoShotTiming.js@@ -90,6 +90,13 @@ export function selectedPerformanceInstruction(scene) {return edit.inSec >= 0 && edit.outSec > edit.inSec ? instruction : null;}+/**+ * How far into a take's clip its audio window really starts (#11210): the+ * provider's leading padding the take's edit points were moved by when it+ * landed, or 0. Clip time t plays song time `audioWindow.startSec + t - offset`.+ */+export const clipAudioOffsetSec = (instruction) => (Number.isFinite(instruction?.alignment?.offsetSec) ? instruction.alignment.offsetSec : 0);+// Safety margin kept inside the provider's documented bounds so ffmpeg's// sample rounding can never land a window a hair under the minimum (rejected)// or over the maximum (silently clipped by the provider).@@ -528,8 +535,10 @@ export function planPerformanceRepair({ scene, temporal, excerptStartSec, backen}if (Math.abs(cursor - scene.endSec) > 0.001 || failedAt === null || failedAt <= scene.startSec) return no('Review needed: no accepted prefix and failed suffix were measured');const windowStart = instruction.audioWindow?.startSec;- if (!Number.isFinite(windowStart) || Math.abs(windowStart + instruction.edit.inSec - scene.startSec) > 0.001- || Math.abs(windowStart + instruction.edit.outSec - scene.endSec) > 0.001) return no('Review needed: the source audio timebase is missing');+ // Song time of clip time 0: an aligned take's window starts `offset` into its clip.+ const clipStart = windowStart - clipAudioOffsetSec(instruction);+ if (!Number.isFinite(windowStart) || Math.abs(clipStart + instruction.edit.inSec - scene.startSec) > 0.001+ || Math.abs(clipStart + instruction.edit.outSec - scene.endSec) > 0.001) return no('Review needed: the source audio timebase is missing');const cues = (instruction.cues || []).filter((cue) => cue.endSec + windowStart > scene.startSec && cue.startSec + windowStart < scene.endSec);if (!cues.length || cues.some((cue) => !cue.words?.length)) return no('Review needed: word timings are required to choose a safe boundary');const words = cues.flatMap((cue) => cue.words).slice().sort((a, b) => a.startSec - b.startSec);diff --git a/server/lib/musicVideoShotTiming.test.js b/server/lib/musicVideoShotTiming.test.jsindex 9506fee87d..b67a4382f0 100644--- a/server/lib/musicVideoShotTiming.test.js+++ b/server/lib/musicVideoShotTiming.test.js@@ -127,6 +127,13 @@ describe('accepted-prefix repair planning', () => {expect(planPerformanceRepair({ scene, temporal, excerptStartSec: 8, backend: 'fal' })).toMatchObject({ ok: true, boundarySec: 14, endSec: 24, suffixSec: 10, referenceClipSec: 6, costUsd: 1.6 });});+ it('plans a take whose edit points were moved onto padded clip audio (#11210) in clip time', async () => {+ const { planPerformanceRepair } = await import('./musicVideoShotTiming.js');+ const aligned = structuredClone(scene);+ Object.assign(aligned.takes[0].shotInstruction, { edit: { inSec: 2.42, outSec: 16.42 }, alignment: { offsetSec: 0.42 } });+ expect(planPerformanceRepair({ scene: aligned, temporal, excerptStartSec: 8, backend: 'fal' }))+ .toMatchObject({ ok: true, boundarySec: 14, referenceClipSec: 6.42 });+ });it('rejects inconclusive/partial evidence, unsafe word gaps, short suffixes and unsupported routes', async () => {const { planPerformanceRepair } = await import('./musicVideoShotTiming.js');const plan = (overrides = {}) => planPerformanceRepair({ scene, temporal, excerptStartSec: 8, backend: 'fal', ...overrides });diff --git a/server/services/musicVideo/takeAudioAlignment.js b/server/services/musicVideo/takeAudioAlignment.jsnew file mode 100644index 0000000000..113955ee26--- /dev/null+++ b/server/services/musicVideo/takeAudioAlignment.js@@ -0,0 +1,71 @@+/**+ * Line a performance take's picture up with the song by its own soundtrack+ * (#11210).+ *+ * A performance take's edit in/out points are planned on the assumption that+ * the provider's clip starts exactly where the submitted audio window starts.+ * Some providers pad the clip instead: fal OmniHuman v1.5 returns a clip a+ * second longer than its audio with the speech starting ~0.4–0.7 s in, so a+ * take placed by the planned in-point sings late. The clip's mouth motion+ * follows the clip's OWN audio track, so that track — matched against the+ * window it was conditioned on — says where the song really sits in the clip.+ *+ * `alignPerformanceInstruction` runs when a take lands (the scene-video hook),+ * for any provider whose clip carries audio. It shifts `edit.inSec` /+ * `edit.outSec` by the measured offset — every renderer, preview and handoff+ * reads those as clip times, and every later trim (beat snap, planned span,+ * intercut, repair prefix) is taken relative to `edit.inSec`, so they all stay+ * aligned — and records `alignment` with the planned points it replaced. Code+ * that maps clip time back to song time subtracts `clipAudioOffsetSec`.+ *+ * Best-effort: no audio track, an unreadable file, or a weak match leaves the+ * instruction exactly as planned.+ */++import { existsSync } from 'fs';+import { PATHS } from '../../lib/fileUtils.js';+import { hasAudioStream, safeUnder } from '../../lib/ffmpeg.js';+import { computeRmsEnvelope, envelopeOffsetSec, FINGERPRINT_RATE_HZ } from '../../lib/audioFingerprint.js';++export const TAKE_ALIGNMENT_VERSION = 1;+// Under a frame at 24 fps the planned points already sit on the song.+const APPLY_MIN_SEC = 0.02;++const round3 = (n) => Math.round(n * 1000) / 1000;++/**+ * `instruction` with its edit points moved onto the clip's audio, or+ * `instruction` unchanged. `clipPath` is the finished take's file.+ */+export async function alignPerformanceInstruction(instruction, clipPath) {+ const edit = instruction?.edit;+ const window = instruction?.audioWindow;+ const sourceName = instruction?.audio?.conditioning?.filename;+ if (instruction?.shotMode !== 'performance' || instruction.alignment || !Number.isFinite(edit?.inSec) || !Number.isFinite(edit?.outSec)+ || !Number.isFinite(window?.startSec) || !Number.isFinite(window?.endSec) || typeof sourceName !== 'string') return instruction;+ const sourcePath = safeUnder(PATHS.music, sourceName);+ if (!sourcePath || !existsSync(sourcePath) || !clipPath || !existsSync(clipPath) || !(await hasAudioStream(clipPath))) return instruction;++ const [song, clip] = await Promise.all([computeRmsEnvelope(sourcePath), computeRmsEnvelope(clipPath)]);+ const startFrame = Math.ceil(window.startSec * FINGERPRINT_RATE_HZ - 1e-6);+ const endFrame = Math.min(song.length, Math.floor(window.endSec * FINGERPRINT_RATE_HZ + 1e-6));+ if (!(endFrame > startFrame)) return instruction;+ const match = envelopeOffsetSec(song.subarray(startFrame, endFrame), clip);+ if (!match) return instruction;+ // The envelope grid starts a few ms after the window does.+ const offsetSec = round3(match.offsetSec - (startFrame / FINGERPRINT_RATE_HZ - window.startSec));+ // A clip that starts early can only be pulled back to its first frame.+ const appliedSec = Math.abs(offsetSec) < APPLY_MIN_SEC ? 0 : round3(Math.max(offsetSec, -edit.inSec));+ return {+ ...instruction,+ edit: { ...edit, inSec: round3(edit.inSec + appliedSec), outSec: round3(edit.outSec + appliedSec) },+ alignment: {+ version: TAKE_ALIGNMENT_VERSION,+ method: 'rms-envelope-xcorr',+ offsetSec: appliedSec,+ measuredSec: offsetSec,+ correlation: match.correlation,+ planned: { inSec: edit.inSec, outSec: edit.outSec },+ },+ };+}diff --git a/server/services/musicVideo/takeAudioAlignment.test.js b/server/services/musicVideo/takeAudioAlignment.test.jsnew file mode 100644index 0000000000..c181bdef87--- /dev/null+++ b/server/services/musicVideo/takeAudioAlignment.test.js@@ -0,0 +1,100 @@+/**+ * Performance-take alignment against real ffmpeg (#11210): a provider clip+ * whose soundtrack is the submitted window behind leading padding gets its+ * edit points moved by that padding; a clip without matching audio keeps the+ * planned points.+ */+import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest';+import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'fs';+import { tmpdir } from 'os';+import { join } from 'path';++const ROOT = mkdtempSync(join(tmpdir(), 'mv-take-align-'));+vi.mock('../../lib/fileUtils.js', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, PATHS: { ...actual.PATHS, music: join(ROOT, 'music') } };+});++const { findFfmpeg, runFfmpegProcess } = await import('../../lib/ffmpeg.js');+const { alignPerformanceInstruction } = await import('./takeAudioAlignment.js');+const ffmpeg = await findFfmpeg();++const RATE = 48000;+const SONG_SEC = 12;+// A sung line: a tone whose loudness jumps at irregular 60–180 ms steps, so+// one stretch of the song never resembles another a fraction of a second off.+function songSamples() {+ let seed = 7;+ const random = () => { seed = (seed * 1103515245 + 12345) % 2147483648; return seed / 2147483648; };+ const samples = new Int16Array(SONG_SEC * RATE);+ let amp = 0;+ let next = 0;+ for (let i = 0; i < samples.length; i++) {+ if (i >= next) { amp = random() < 0.2 ? 0 : 0.05 + random() * 0.6; next = i + Math.round((0.06 + random() * 0.12) * RATE); }+ samples[i] = Math.round(amp * 30000 * Math.sin(2 * Math.PI * 220 * i / RATE));+ }+ return samples;+}++function wav(samples) {+ const buf = Buffer.alloc(44 + samples.length * 2);+ buf.write('RIFF', 0); buf.writeUInt32LE(36 + samples.length * 2, 4); buf.write('WAVE', 8);+ buf.write('fmt ', 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22);+ buf.writeUInt32LE(RATE, 24); buf.writeUInt32LE(RATE * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34);+ buf.write('data', 36); buf.writeUInt32LE(samples.length * 2, 40);+ samples.forEach((v, i) => buf.writeInt16LE(v, 44 + i * 2));+ return buf;+}++const song = songSamples();+const SONG = join(ROOT, 'music', 'song.wav');+const WINDOW = { startSec: 2.003, endSec: 6.203, durationSec: 4.2 };+const instruction = () => ({+ shotMode: 'performance',+ audio: { conditioning: { source: 'master', filename: 'song.wav' } },+ audioWindow: { ...WINDOW },+ edit: { inSec: 0, outSec: 4.2, targetSec: 4.2 },+});++// A 5.05 s provider clip whose audio is `[from, from + 4.2)` of the song after `padSec` of silence.+async function clip(name, { padSec = 0, from = WINDOW.startSec, audio = true }) {+ const out = join(ROOT, name);+ const args = ['-y', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc=size=160x90:rate=24:duration=5.05'];+ if (audio) {+ const track = new Int16Array(Math.round(5.05 * RATE));+ const start = Math.round(from * RATE);+ track.set(song.subarray(start, start + Math.round(4.2 * RATE)), Math.round(padSec * RATE));+ writeFileSync(`${out}.wav`, wav(track));+ args.push('-i', `${out}.wav`, '-c:a', 'aac');+ }+ args.push('-pix_fmt', 'yuv420p', out);+ const made = await runFfmpegProcess({ bin: ffmpeg, args });+ expect(made.ok).toBe(true);+ return out;+}++describe.skipIf(!ffmpeg)('alignPerformanceInstruction', () => {+ beforeAll(() => {+ mkdirSync(join(ROOT, 'music'), { recursive: true });+ writeFileSync(SONG, wav(song));+ });+ afterAll(() => rmSync(ROOT, { recursive: true, force: true }));++ it('moves the edit points past the clip\'s leading padding and records what it replaced', async () => {+ const aligned = await alignPerformanceInstruction(instruction(), await clip('padded.mp4', { padSec: 0.42 }));+ expect(aligned.alignment).toMatchObject({ method: 'rms-envelope-xcorr', planned: { inSec: 0, outSec: 4.2 } });+ expect(aligned.alignment.correlation).toBeGreaterThan(0.9);+ expect(aligned.alignment.offsetSec).toBeCloseTo(0.42, 1);+ expect(aligned.edit.inSec).toBe(aligned.alignment.offsetSec);+ expect(aligned.edit.outSec).toBeCloseTo(4.2 + aligned.alignment.offsetSec, 6);+ expect(aligned.edit.targetSec).toBe(4.2);+ });++ it('keeps the planned points for a clip in sync, without audio, or with other audio', async () => {+ const inSync = await alignPerformanceInstruction(instruction(), await clip('sync.mp4', { padSec: 0 }));+ expect(inSync.edit).toMatchObject({ inSec: 0, outSec: 4.2 });+ expect(inSync.alignment.offsetSec).toBe(0);+ expect(await alignPerformanceInstruction(instruction(), await clip('silent.mp4', { audio: false }))).toEqual(instruction());+ expect(await alignPerformanceInstruction(instruction(), await clip('other.mp4', { padSec: 0.3, from: 7.5 }))).toEqual(instruction());+ });+});diff --git a/server/services/musicVideoSceneVideoHook.js b/server/services/musicVideoSceneVideoHook.jsindex eeea852c72..d67a89f624 100644--- a/server/services/musicVideoSceneVideoHook.js+++ b/server/services/musicVideoSceneVideoHook.js@@ -35,6 +35,24 @@ import { basename } from 'path';import { appendSceneTakes } from './musicVideo/projects.js';import { musicVideoEvents } from './musicVideo/events.js';import { recordSceneLastFailure } from './musicVideo/sceneFailure.js';+import { alignPerformanceInstruction } from './musicVideo/takeAudioAlignment.js';+import { getHistoryItem } from './videoGen/history.js';+import { PATHS } from '../lib/fileUtils.js';+import { safeUnder } from '../lib/ffmpeg.js';++// #11210: a provider that pads the clip's start (fal OmniHuman) would sing+// late at the planned in-point; the clip's own audio says where the song is.+async function alignedShotInstruction(instruction, videoHistoryId) {+ if (instruction?.shotMode !== 'performance') return instruction ?? null;+ const entry = await getHistoryItem(videoHistoryId).catch(() => null);+ const clipPath = entry?.filename ? safeUnder(PATHS.videos, entry.filename) : null;+ const aligned = await alignPerformanceInstruction(instruction, clipPath).catch((err) => {+ console.warn(`⚠️ music-video take ${videoHistoryId.slice(0, 8)}: could not align its audio to the song: ${err.message}`);+ return instruction;+ });+ if (aligned.alignment?.offsetSec) console.log(`🎤 music-video take ${videoHistoryId.slice(0, 8)} aligned to the song: speech starts ${aligned.alignment.offsetSec.toFixed(3)}s into the clip (r=${aligned.alignment.correlation})`);+ return aligned;+}const hook = createMediaJobImageHook({label: 'music-video scene-video',@@ -62,7 +80,7 @@ const hook = createMediaJobImageHook({// takes, and none of them can displace a selection.describe: ({ projectId, sceneId }) => `${projectId}/${sceneId}`,// A deleted project/scene 404s here, so a late completion can't resurrect it.- attach: ({ projectId, sceneId, videoHistoryId, job }) => appendSceneTakes(projectId, sceneId, [{+ attach: async ({ projectId, sceneId, videoHistoryId, job }) => appendSceneTakes(projectId, sceneId, [{kind: 'video',assetId: videoHistoryId,source: 'generated',@@ -75,7 +93,7 @@ const hook = createMediaJobImageHook({// #8977: a performance take keeps the immutable record of the song slice,// edit in/out points and capability it was generated against — the// renderer places the take by it, so a later retime can't misalign lips.- shotInstruction: job.params?.shotInstruction ?? null,+ shotInstruction: await alignedShotInstruction(job.params?.shotInstruction, videoHistoryId),dependencies: job.params?.musicVideoDependencies ?? null,// #10157: the estimate this clip was priced at; a revision-tagged job is// an auto-review generation, the rest are manual takes.