Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Crop fal clip frames to the music video's aspect
Claude3edff0b5bda6
diff --git a/server/services/imageGen/aspectConform.js b/server/services/imageGen/aspectConform.jsnew file mode 100644index 0000000000..cbb6f39682--- /dev/null+++ b/server/services/imageGen/aspectConform.js@@ -0,0 +1,64 @@+/**+ * Image Gen — conform a provider's output to the requested aspect ratio.+ *+ * Codex's built-in image_gen tool takes no size argument: the requested canvas+ * rides the prompt as a hint and the model picks one of its native sizes. Most+ * of the time that lands on the asked-for shape, but not always — a 16:9 music+ * video production came back with 1254x1254 squares (mostly performance+ * close-ups), near-4:3 frames and 1536x1024 3:2 frames mixed in with its+ * 1672x941 widescreen stills, and every clip lane downstream inherited the+ * wrong shape (an OmniHuman lip-sync clip from a square still is 1080x1080).+ *+ * So the provider's completion handler runs the written frame through here+ * before the sidecar is written: a frame within `ASPECT_TOLERANCE` of the+ * requested ratio is left byte-for-byte alone; anything else is cropped to the+ * largest rectangle of the requested ratio, positioned by sharp's attention+ * strategy so a close-up keeps the face rather than a geometric centre. The+ * crop never upscales — it keeps the provider's native resolution on the long+ * side it can afford.+ */++import sharp from 'sharp';+import { atomicWrite } from '../../lib/fileUtils.js';++// 2% — 1672x941 (1.7768) is Codex's own 16:9 and must pass untouched against a+// 1.7778 request; the nearest wrong shape it returns (3:2, 1.5) is ~16% off.+export const ASPECT_TOLERANCE = 0.02;++const positive = (n) => Number.isFinite(Number(n)) && Number(n) > 0;++/**+ * The crop box that conforms a `srcW`x`srcH` frame to `targetW:targetH`, or+ * `null` when the frame is already within tolerance (or the input is unusable).+ * Pure — exported for the unit test.+ *+ * @returns {{ width: number, height: number }|null}+ */+export function aspectCropSize(srcW, srcH, targetW, targetH, tolerance = ASPECT_TOLERANCE) {+ if (![srcW, srcH, targetW, targetH].every(positive)) return null;+ const target = targetW / targetH;+ const actual = srcW / srcH;+ if (Math.abs(actual / target - 1) <= tolerance) return null;+ // Too wide → keep full height, narrow the width; too tall → the reverse.+ const width = actual > target ? Math.round(srcH * target) : srcW;+ const height = actual > target ? srcH : Math.round(srcW / target);+ return { width: Math.max(1, width), height: Math.max(1, height) };+}++/**+ * Crop the PNG at `pngPath` in place to the requested aspect ratio when it is+ * off. Returns `{ from, to }` (`"WxH"` strings) when it cropped, else `null`.+ * A frame sharp cannot read is left for the existing invalid-image paths.+ */+export async function conformImageAspect(pngPath, targetW, targetH) {+ if (!positive(targetW) || !positive(targetH)) return null;+ const meta = await sharp(pngPath).metadata().catch(() => null);+ const crop = aspectCropSize(meta?.width, meta?.height, Number(targetW), Number(targetH));+ if (!crop) return null;+ const buffer = await sharp(pngPath)+ .resize(crop.width, crop.height, { fit: 'cover', position: sharp.strategy.attention })+ .png()+ .toBuffer();+ await atomicWrite(pngPath, buffer);+ return { from: `${meta.width}x${meta.height}`, to: `${crop.width}x${crop.height}` };+}diff --git a/server/services/imageGen/aspectConform.test.js b/server/services/imageGen/aspectConform.test.jsnew file mode 100644index 0000000000..2d953d60b0--- /dev/null+++ b/server/services/imageGen/aspectConform.test.js@@ -0,0 +1,20 @@+import { describe, it, expect } from 'vitest';+import { aspectCropSize } from './aspectConform.js';++// The crop-box arithmetic, over the shapes Codex actually returned for 16:9+// production stills. The end-to-end crop (sharp + sidecar) is covered in+// codex.test.js; this pins the tolerance boundary and both crop directions.+describe('aspectCropSize', () => {+ it.each([+ ['Codex 16:9 passes untouched', [1672, 941, 1920, 1080], null],+ ['square close-up crops height', [1254, 1254, 1920, 1080], { width: 1254, height: 705 }],+ ['near-4:3 crops height', [1448, 1086, 1920, 1080], { width: 1448, height: 815 }],+ ['3:2 crops height', [1536, 1024, 1920, 1080], { width: 1536, height: 864 }],+ ['too-wide frame crops width', [2000, 800, 1920, 1080], { width: 1422, height: 800 }],+ ['portrait request from a landscape frame', [1536, 1024, 1024, 1536], { width: 683, height: 1024 }],+ ['unusable size is left alone', [0, 1024, 1920, 1080], null],+ ['no requested size is left alone', [1254, 1254, null, null], null],+ ])('%s', (_label, [sw, sh, tw, th], expected) => {+ expect(aspectCropSize(sw, sh, tw, th)).toEqual(expected);+ });+});diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.jsindex 3e94cb185e..47b1d5f7ad 100644--- a/server/services/videoGen/fal.js+++ b/server/services/videoGen/fal.js@@ -44,6 +44,7 @@ import { maintenance } from '../../lib/maintenanceAdmission.js';import { mutateVideoHistory } from './history.js';import { getSettings } from '../settings.js';import { nearestAspectRatio } from '../imageGen/modes.js';+import { frameDataUriAtAspect } from './falFrameAspect.js';import {FAL_QUEUE_BASE, awaitFalCompletion, cancelFalRequest, createFalRequestEntry, fileToDataUri,readCompletedFalResult, resolveFalApiKey, submitFalRequest,@@ -145,7 +146,7 @@ export async function generateVideo({sourceImagePath = null, lastImagePath = null, jobId: providedJobId = null,audioFilePath = null, lipSync = null, shotInstruction = null,uploadedTempPath = null, uploadedTempPaths = [],- resolution = null, generateAudio = false,+ resolution = null, generateAudio = false, frameAspect = null,}) {await ensureDir(PATHS.videos);const renderStartedAtMs = Date.now();@@ -251,7 +252,7 @@ export async function generateVideo({runFalVideo(job, jobId, {apiKey, modelId, prompt, negativePrompt, duration, aspectRatio: effectiveAspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,audioFilePath, uploadedTempPath, uploadedTempPaths, enableTranscription: lipSync?.enableTranscription === true,- requestSpec: planned ? requestSpec : null,+ requestSpec: planned ? requestSpec : null, frameAspect,}).catch((err) => {console.error(`❌ fal video run failed [${jobId.slice(0, 8)}]: ${err?.message}`);@@ -281,14 +282,25 @@ function audioMimeType(path) {async function runFalVideo(job, jobId, {apiKey, modelId, prompt, negativePrompt, duration, aspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,audioFilePath = null, uploadedTempPath = null, uploadedTempPaths = [], enableTranscription = false, requestSpec = null,+ frameAspect = null,}) {const entry = createFalRequestEntry(apiKey);const publication = {};activeRequests.set(jobId, entry);const deadline = Date.now() + FAL_RENDER_TIMEOUT_MS;try {- const imageDataUri = sourceImagePath ? await fileToDataUri(sourceImagePath) : null;- const endImageDataUri = lastImagePath ? await fileToDataUri(lastImagePath) : null;+ // A Music Video clip's frames are cropped to its project's aspect, since+ // most routes size the output canvas from the start frame (falFrameAspect.js).+ const frame = sourceImagePath ? await frameDataUriAtAspect(sourceImagePath, frameAspect) : null;+ const endFrame = lastImagePath ? await frameDataUriAtAspect(lastImagePath, frameAspect) : null;+ const imageDataUri = frame?.dataUri ?? null;+ const endImageDataUri = endFrame?.dataUri ?? null;+ for (const [label, fit] of [['start', frame], ['end', endFrame]]) {+ if (fit?.conformed) console.log(`✂️ Cropped fal ${label} frame [${jobId.slice(0, 8)}] ${fit.conformed.from} → ${fit.conformed.to} for ${frameAspect}`);+ }+ if (frame?.conformed || endFrame?.conformed) {+ meta.frameConformed = { aspect: frameAspect, ...(frame?.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };+ }const audioDataUri = audioFilePath? `data:${audioMimeType(audioFilePath)};base64,${(await readFile(audioFilePath)).toString('base64')}`: null;diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.jsindex 1b1b33882f..02d76484da 100644--- a/server/services/videoGen/fal.test.js+++ b/server/services/videoGen/fal.test.js@@ -281,6 +281,32 @@ describe('videoGen/fal — generateVideo', () => {expect(fetchMock).toHaveBeenCalled();});+ it('crops an off-aspect start frame to the named frame aspect before upload, leaving the file alone', async () => {+ const sharp = (await import('sharp')).default;+ const framePath = join(TEST_ROOT, 'square.png');+ const square = await sharp({ create: { width: 640, height: 640, channels: 3, background: { r: 200, g: 120, b: 40 } } }).png().toBuffer();+ await writeFile(framePath, square);+ let sent = null;+ vi.stubGlobal('fetch', vi.fn(async (url, opts) => {+ if (url === 'https://queue.fal.run/fal-ai/x') {+ sent = JSON.parse(opts.body);+ return jsonResponse({ request_id: 'r3', status_url: 'https://queue.fal.run/fal-ai/x/requests/r3/status', response_url: 'https://queue.fal.run/fal-ai/x/requests/r3' });+ }+ if (url.endsWith('/r3/status')) return jsonResponse({ status: 'COMPLETED' });+ if (url.endsWith('/r3')) return jsonResponse({ video: { url: 'https://cdn.fal.ai/out.mp4' } });+ if (url === 'https://cdn.fal.ai/out.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(Buffer.from('bytes')).buffer };+ throw new Error(`unexpected fetch: ${url}`);+ }));++ const job = await fal.generateVideo({ apiKey: 'test-key', modelId: 'fal-ai/x', prompt: 'singer', sourceImagePath: framePath, frameAspect: '16:9' });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });+ const uploaded = Buffer.from(sent.image_url.split(',')[1], 'base64');+ const { width, height } = await sharp(uploaded).metadata();+ expect({ width, height }).toEqual({ width: 640, height: 360 });+ expect((await readFile(framePath)).equals(square)).toBe(true);+ expect((await loadHistory())[0].frameConformed).toEqual({ aspect: '16:9', start: { from: '640x640', to: '640x360' } });+ });+it('resolves the API key from live settings when the caller supplies neither apiKey nor settings (the mediaJobQueue dispatch shape)', async () => {getSettingsMock.mockResolvedValue({ videoGen: { fal: { apiKey: 'live-key' } } });const fetchMock = vi.fn(async (url, opts) => {diff --git a/server/services/videoGen/falFrameAspect.js b/server/services/videoGen/falFrameAspect.jsnew file mode 100644index 0000000000..26ae9e7a10--- /dev/null+++ b/server/services/videoGen/falFrameAspect.js@@ -0,0 +1,39 @@+/**+ * fal.ai clip frames at the project's aspect.+ *+ * Most fal image-to-video and audio-driven routes (OmniHuman, Hailuo-02, Kling)+ * take no aspect parameter: the output canvas follows the start frame. A square+ * still therefore became a 1080x1080 lip-sync clip that pillarboxes in a 16:9+ * music video. A Music Video clip names its project's aspect as `frameAspect`,+ * and the start and end frames are cropped to it here, in memory, just before+ * upload. The gallery image stays untouched, so the take still records the+ * frame the director picked. A frame within the shared tolerance uploads+ * byte-for-byte; anything else is attention-cropped (a close-up keeps its face)+ * to the largest rectangle of that ratio, never upscaled.+ */+import sharp from 'sharp';+import { fileToDataUri } from '../falQueue.js';+import { aspectCropSize } from '../imageGen/aspectConform.js';++const RATIO = /^(\d+):(\d+)$/;++/**+ * The frame at `filePath` as a data URI, cropped to `aspect` (`"16:9"`) when it+ * is off. Returns `{ dataUri, conformed }`; `conformed` is `{ from, to }`+ * (`"WxH"`) when it cropped, else null. No aspect, or a frame sharp cannot+ * read, uploads as is.+ */+export async function frameDataUriAtAspect(filePath, aspect) {+ const ratio = RATIO.exec(typeof aspect === 'string' ? aspect : '');+ const meta = ratio ? await sharp(filePath).metadata().catch(() => null) : null;+ const crop = meta && aspectCropSize(meta.width, meta.height, Number(ratio[1]), Number(ratio[2]));+ if (!crop) return { dataUri: await fileToDataUri(filePath), conformed: null };+ const buffer = await sharp(filePath)+ .resize(crop.width, crop.height, { fit: 'cover', position: sharp.strategy.attention })+ .png()+ .toBuffer();+ return {+ dataUri: `data:image/png;base64,${buffer.toString('base64')}`,+ conformed: { from: `${meta.width}x${meta.height}`, to: `${crop.width}x${crop.height}` },+ };+}diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex 986ccb68d1..96294fe1d0 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -35,6 +35,13 @@ import {import { VIDEO_GEN_LOCAL_ONLY_FIELD_NAMES } from './requestFields.js';import { isTruthyMeta } from '../../lib/metadataFlags.js';++const musicVideoProjectAspect = async (projectId) => {+ const [{ getProject }, { musicVideoAspect }] = await Promise.all([+ import('../musicVideo/projects.js'), import('../../lib/musicVideoAspect.js'),+ ]);+ return musicVideoAspect(await getProject(projectId));+};const submitValidatedVideoGenJob = async (body, uploads) => {let fableLoomRenderSettings = null;if (body.fableLoom) {@@ -348,6 +355,13 @@ const submitValidatedVideoGenJob = async (body, uploads) => {const hosted = HOSTED_VIDEO_SUBMISSIONS[backend];if (hosted) {const { sourceImagePath, uploadedTempPath } = prepared;+ // A Music Video fal clip renders at its project's aspect: requested where+ // the model takes an aspect ratio, and its frames cropped to it where the+ // output follows the start frame (falFrameAspect.js). An explicit+ // visual-conditioning aspect (FableLoom) still wins.+ const projectAspect = backend === VIDEO_GEN_MODE.FAL && body.musicVideo && !body.visualConditioning?.render?.parameters?.aspectRatio+ ? await musicVideoProjectAspect(body.musicVideo.projectId).catch(async (error) => { await cleanupStaged(); throw error; })+ : null;const { jobId, position, status } = await enqueue({// Hosted jobs use mode for queue dispatch and videoMode for text/image.mode: backend,@@ -360,6 +374,7 @@ const submitValidatedVideoGenJob = async (body, uploads) => {...(body.fableLoom ? { fableLoom: body.fableLoom } : {}),...(body.visualConditioning ? { visualConditioning: body.visualConditioning } : {}),...hosted.buildParams(body, prepared),+ ...(projectAspect ? { aspectRatio: projectAspect, frameAspect: projectAspect } : {}),// standalone fal lip sync: the uploaded voice clip (a Music Video performance shot overrides below)...(prepared.audioFilePath ? { audioFilePath: prepared.audioFilePath, uploadedTempPaths: prepared.uploadedTempPaths, lipSync: { enableTranscription: false } } : {}),...performanceParams,diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.jsindex ddb8d02742..a9049762ae 100644--- a/server/services/videoGen/submitJob.test.js+++ b/server/services/videoGen/submitJob.test.js@@ -246,6 +246,19 @@ describe('submitVideoGenJob', () => {expect(params.generateAudio).toBeUndefined();});+ it('asks for the project aspect and crops the clip frames to it (a 9:16 project)', async () => {+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());+ mocks.getMusicVideoProject.mockResolvedValue({ id: 'mv-1', scenes: [{ sceneId: 'mvs-1' }], treatment: { brief: { aspectRatio: '9:16' } } });+ await submitVideoGenJob({ prompt: 'singer', backend: 'fal', falModelId: 'fal-ai/bytedance/omnihuman/v1.5', musicVideo, mode: 'image' }, {});+ expect(mocks.enqueueJob.mock.calls[0][0].params).toMatchObject({ aspectRatio: '9:16', frameAspect: '9:16' });+ });++ it('leaves a clip outside a music video at its own aspect', async () => {+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());+ await submitVideoGenJob({ prompt: 'a fox', backend: 'fal', mode: 'image' }, {});+ expect(mocks.enqueueJob.mock.calls[0][0].params).not.toHaveProperty('frameAspect');+ });+it('refuses a performance shot the backend cannot render and releases what was staged', async () => {const prepared = { ...falPrepared(), backend: 'grok', grok: { grokPath: '/example/grok' } };mocks.prepareVideoGenParams.mockResolvedValue(prepared);