Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Test envelope offset matching directly, including early clips

Claude05fe29ef3c8a
diff --git a/server/lib/audioFingerprint.test.js b/server/lib/audioFingerprint.test.js
index c909af16ab..e371cf3a80 100644
--- a/server/lib/audioFingerprint.test.js
+++ b/server/lib/audioFingerprint.test.js
@@ -8,7 +8,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { findFfmpeg } from './ffmpeg.js';
-import { compareWindowFingerprint, computeRmsEnvelope, windowFingerprint } from './audioFingerprint.js';
+import { compareWindowFingerprint, computeRmsEnvelope, envelopeOffsetSec, windowFingerprint } from './audioFingerprint.js';
const ffmpeg = await findFfmpeg();
const RATE = 48000;
@@ -106,3 +106,21 @@ describe.skipIf(!ffmpeg)('audio window fingerprints', () => {
expect(compareWindowFingerprint(null, envelope).same).toBe(false);
}, 30_000);
});
+
+describe('envelopeOffsetSec', () => {
+ // An irregular contour: no stretch resembles another a few frames off.
+ let seed = 3;
+ const contour = Int8Array.from({ length: 400 }, () => { seed = (seed * 1103515245 + 12345) % 2147483648; return -10 - Math.floor((seed / 2147483648) * 40); });
+ const shifted = (frames) => Int8Array.from({ length: 500 }, (_, i) => (i - frames >= 0 && i - frames < contour.length ? contour[i - frames] : -90));
+
+ it('finds the reference late (padding) or early inside the candidate', () => {
+ expect(envelopeOffsetSec(contour, shifted(42))).toMatchObject({ offsetSec: 0.42, correlation: 1 });
+ expect(envelopeOffsetSec(contour.subarray(30), shifted(0)).offsetSec).toBe(0.3);
+ expect(envelopeOffsetSec(contour.subarray(0, 370), Int8Array.from(contour.subarray(20)))).toMatchObject({ offsetSec: -0.2 });
+ });
+
+ it('returns null for a flat window or a contour that does not match', () => {
+ expect(envelopeOffsetSec(new Int8Array(400).fill(-20), shifted(10))).toBeNull();
+ expect(envelopeOffsetSec(contour, Int8Array.from(contour).reverse())).toBeNull();
+ });
+});
diff --git a/server/services/musicVideo/takeAudioAlignment.test.js b/server/services/musicVideo/takeAudioAlignment.test.js
index c181bdef87..1195d660b6 100644
--- a/server/services/musicVideo/takeAudioAlignment.test.js
+++ b/server/services/musicVideo/takeAudioAlignment.test.js
@@ -94,6 +94,10 @@ describe.skipIf(!ffmpeg)('alignPerformanceInstruction', () => {
const inSync = await alignPerformanceInstruction(instruction(), await clip('sync.mp4', { padSec: 0 }));
expect(inSync.edit).toMatchObject({ inSec: 0, outSec: 4.2 });
expect(inSync.alignment.offsetSec).toBe(0);
+ // A clip that starts early (its first 0.2 s of the window missing) cannot be pulled before its first frame.
+ const early = await alignPerformanceInstruction(instruction(), await clip('early.mp4', { from: WINDOW.startSec + 0.2 }));
+ expect(early.alignment.measuredSec).toBeCloseTo(-0.2, 1);
+ expect(early.edit).toMatchObject({ inSec: 0, outSec: 4.2 });
expect(await alignPerformanceInstruction(instruction(), await clip('silent.mp4', { audio: false }))).toEqual(instruction());
expect(await alignPerformanceInstruction(instruction(), await clip('other.mp4', { padSec: 0.3, from: 7.5 }))).toEqual(instruction());
});