Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Crop fal clip frames to the music video's aspect

Claude3edff0b5bda6
diff --git a/server/services/imageGen/aspectConform.js b/server/services/imageGen/aspectConform.js
new file mode 100644
index 0000000000..cbb6f39682
--- /dev/null
+++ b/server/services/imageGen/aspectConform.js
@@ -0,0 +1,64 @@
+/**
+ * Image Gen — conform a provider's output to the requested aspect ratio.
+ *
+ * Codex's built-in image_gen tool takes no size argument: the requested canvas
+ * rides the prompt as a hint and the model picks one of its native sizes. Most
+ * of the time that lands on the asked-for shape, but not always — a 16:9 music
+ * video production came back with 1254x1254 squares (mostly performance
+ * close-ups), near-4:3 frames and 1536x1024 3:2 frames mixed in with its
+ * 1672x941 widescreen stills, and every clip lane downstream inherited the
+ * wrong shape (an OmniHuman lip-sync clip from a square still is 1080x1080).
+ *
+ * So the provider's completion handler runs the written frame through here
+ * before the sidecar is written: a frame within `ASPECT_TOLERANCE` of the
+ * requested ratio is left byte-for-byte alone; anything else is cropped to the
+ * largest rectangle of the requested ratio, positioned by sharp's attention
+ * strategy so a close-up keeps the face rather than a geometric centre. The
+ * crop never upscales — it keeps the provider's native resolution on the long
+ * side it can afford.
+ */
+
+import sharp from 'sharp';
+import { atomicWrite } from '../../lib/fileUtils.js';
+
+// 2% — 1672x941 (1.7768) is Codex's own 16:9 and must pass untouched against a
+// 1.7778 request; the nearest wrong shape it returns (3:2, 1.5) is ~16% off.
+export const ASPECT_TOLERANCE = 0.02;
+
+const positive = (n) => Number.isFinite(Number(n)) && Number(n) > 0;
+
+/**
+ * The crop box that conforms a `srcW`x`srcH` frame to `targetW:targetH`, or
+ * `null` when the frame is already within tolerance (or the input is unusable).
+ * Pure — exported for the unit test.
+ *
+ * @returns {{ width: number, height: number }|null}
+ */
+export function aspectCropSize(srcW, srcH, targetW, targetH, tolerance = ASPECT_TOLERANCE) {
+ if (![srcW, srcH, targetW, targetH].every(positive)) return null;
+ const target = targetW / targetH;
+ const actual = srcW / srcH;
+ if (Math.abs(actual / target - 1) <= tolerance) return null;
+ // Too wide → keep full height, narrow the width; too tall → the reverse.
+ const width = actual > target ? Math.round(srcH * target) : srcW;
+ const height = actual > target ? srcH : Math.round(srcW / target);
+ return { width: Math.max(1, width), height: Math.max(1, height) };
+}
+
+/**
+ * Crop the PNG at `pngPath` in place to the requested aspect ratio when it is
+ * off. Returns `{ from, to }` (`"WxH"` strings) when it cropped, else `null`.
+ * A frame sharp cannot read is left for the existing invalid-image paths.
+ */
+export async function conformImageAspect(pngPath, targetW, targetH) {
+ if (!positive(targetW) || !positive(targetH)) return null;
+ const meta = await sharp(pngPath).metadata().catch(() => null);
+ const crop = aspectCropSize(meta?.width, meta?.height, Number(targetW), Number(targetH));
+ if (!crop) return null;
+ const buffer = await sharp(pngPath)
+ .resize(crop.width, crop.height, { fit: 'cover', position: sharp.strategy.attention })
+ .png()
+ .toBuffer();
+ await atomicWrite(pngPath, buffer);
+ return { from: `${meta.width}x${meta.height}`, to: `${crop.width}x${crop.height}` };
+}
diff --git a/server/services/imageGen/aspectConform.test.js b/server/services/imageGen/aspectConform.test.js
new file mode 100644
index 0000000000..2d953d60b0
--- /dev/null
+++ b/server/services/imageGen/aspectConform.test.js
@@ -0,0 +1,20 @@
+import { describe, it, expect } from 'vitest';
+import { aspectCropSize } from './aspectConform.js';
+
+// The crop-box arithmetic, over the shapes Codex actually returned for 16:9
+// production stills. The end-to-end crop (sharp + sidecar) is covered in
+// codex.test.js; this pins the tolerance boundary and both crop directions.
+describe('aspectCropSize', () => {
+ it.each([
+ ['Codex 16:9 passes untouched', [1672, 941, 1920, 1080], null],
+ ['square close-up crops height', [1254, 1254, 1920, 1080], { width: 1254, height: 705 }],
+ ['near-4:3 crops height', [1448, 1086, 1920, 1080], { width: 1448, height: 815 }],
+ ['3:2 crops height', [1536, 1024, 1920, 1080], { width: 1536, height: 864 }],
+ ['too-wide frame crops width', [2000, 800, 1920, 1080], { width: 1422, height: 800 }],
+ ['portrait request from a landscape frame', [1536, 1024, 1024, 1536], { width: 683, height: 1024 }],
+ ['unusable size is left alone', [0, 1024, 1920, 1080], null],
+ ['no requested size is left alone', [1254, 1254, null, null], null],
+ ])('%s', (_label, [sw, sh, tw, th], expected) => {
+ expect(aspectCropSize(sw, sh, tw, th)).toEqual(expected);
+ });
+});
diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.js
index 3e94cb185e..47b1d5f7ad 100644
--- a/server/services/videoGen/fal.js
+++ b/server/services/videoGen/fal.js
@@ -44,6 +44,7 @@ import { maintenance } from '../../lib/maintenanceAdmission.js';
import { mutateVideoHistory } from './history.js';
import { getSettings } from '../settings.js';
import { nearestAspectRatio } from '../imageGen/modes.js';
+import { frameDataUriAtAspect } from './falFrameAspect.js';
import {
FAL_QUEUE_BASE, awaitFalCompletion, cancelFalRequest, createFalRequestEntry, fileToDataUri,
readCompletedFalResult, resolveFalApiKey, submitFalRequest,
@@ -145,7 +146,7 @@ export async function generateVideo({
sourceImagePath = null, lastImagePath = null, jobId: providedJobId = null,
audioFilePath = null, lipSync = null, shotInstruction = null,
uploadedTempPath = null, uploadedTempPaths = [],
- resolution = null, generateAudio = false,
+ resolution = null, generateAudio = false, frameAspect = null,
}) {
await ensureDir(PATHS.videos);
const renderStartedAtMs = Date.now();
@@ -251,7 +252,7 @@ export async function generateVideo({
runFalVideo(job, jobId, {
apiKey, modelId, prompt, negativePrompt, duration, aspectRatio: effectiveAspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,
audioFilePath, uploadedTempPath, uploadedTempPaths, enableTranscription: lipSync?.enableTranscription === true,
- requestSpec: planned ? requestSpec : null,
+ requestSpec: planned ? requestSpec : null, frameAspect,
})
.catch((err) => {
console.error(`❌ fal video run failed [${jobId.slice(0, 8)}]: ${err?.message}`);
@@ -281,14 +282,25 @@ function audioMimeType(path) {
async function runFalVideo(job, jobId, {
apiKey, modelId, prompt, negativePrompt, duration, aspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,
audioFilePath = null, uploadedTempPath = null, uploadedTempPaths = [], enableTranscription = false, requestSpec = null,
+ frameAspect = null,
}) {
const entry = createFalRequestEntry(apiKey);
const publication = {};
activeRequests.set(jobId, entry);
const deadline = Date.now() + FAL_RENDER_TIMEOUT_MS;
try {
- const imageDataUri = sourceImagePath ? await fileToDataUri(sourceImagePath) : null;
- const endImageDataUri = lastImagePath ? await fileToDataUri(lastImagePath) : null;
+ // A Music Video clip's frames are cropped to its project's aspect, since
+ // most routes size the output canvas from the start frame (falFrameAspect.js).
+ const frame = sourceImagePath ? await frameDataUriAtAspect(sourceImagePath, frameAspect) : null;
+ const endFrame = lastImagePath ? await frameDataUriAtAspect(lastImagePath, frameAspect) : null;
+ const imageDataUri = frame?.dataUri ?? null;
+ const endImageDataUri = endFrame?.dataUri ?? null;
+ for (const [label, fit] of [['start', frame], ['end', endFrame]]) {
+ if (fit?.conformed) console.log(`✂️ Cropped fal ${label} frame [${jobId.slice(0, 8)}] ${fit.conformed.from} → ${fit.conformed.to} for ${frameAspect}`);
+ }
+ if (frame?.conformed || endFrame?.conformed) {
+ meta.frameConformed = { aspect: frameAspect, ...(frame?.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };
+ }
const audioDataUri = audioFilePath
? `data:${audioMimeType(audioFilePath)};base64,${(await readFile(audioFilePath)).toString('base64')}`
: null;
diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.js
index 1b1b33882f..02d76484da 100644
--- a/server/services/videoGen/fal.test.js
+++ b/server/services/videoGen/fal.test.js
@@ -281,6 +281,32 @@ describe('videoGen/fal — generateVideo', () => {
expect(fetchMock).toHaveBeenCalled();
});
+ it('crops an off-aspect start frame to the named frame aspect before upload, leaving the file alone', async () => {
+ const sharp = (await import('sharp')).default;
+ const framePath = join(TEST_ROOT, 'square.png');
+ const square = await sharp({ create: { width: 640, height: 640, channels: 3, background: { r: 200, g: 120, b: 40 } } }).png().toBuffer();
+ await writeFile(framePath, square);
+ let sent = null;
+ vi.stubGlobal('fetch', vi.fn(async (url, opts) => {
+ if (url === 'https://queue.fal.run/fal-ai/x') {
+ sent = JSON.parse(opts.body);
+ return jsonResponse({ request_id: 'r3', status_url: 'https://queue.fal.run/fal-ai/x/requests/r3/status', response_url: 'https://queue.fal.run/fal-ai/x/requests/r3' });
+ }
+ if (url.endsWith('/r3/status')) return jsonResponse({ status: 'COMPLETED' });
+ if (url.endsWith('/r3')) return jsonResponse({ video: { url: 'https://cdn.fal.ai/out.mp4' } });
+ if (url === 'https://cdn.fal.ai/out.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(Buffer.from('bytes')).buffer };
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+
+ const job = await fal.generateVideo({ apiKey: 'test-key', modelId: 'fal-ai/x', prompt: 'singer', sourceImagePath: framePath, frameAspect: '16:9' });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
+ const uploaded = Buffer.from(sent.image_url.split(',')[1], 'base64');
+ const { width, height } = await sharp(uploaded).metadata();
+ expect({ width, height }).toEqual({ width: 640, height: 360 });
+ expect((await readFile(framePath)).equals(square)).toBe(true);
+ expect((await loadHistory())[0].frameConformed).toEqual({ aspect: '16:9', start: { from: '640x640', to: '640x360' } });
+ });
+
it('resolves the API key from live settings when the caller supplies neither apiKey nor settings (the mediaJobQueue dispatch shape)', async () => {
getSettingsMock.mockResolvedValue({ videoGen: { fal: { apiKey: 'live-key' } } });
const fetchMock = vi.fn(async (url, opts) => {
diff --git a/server/services/videoGen/falFrameAspect.js b/server/services/videoGen/falFrameAspect.js
new file mode 100644
index 0000000000..26ae9e7a10
--- /dev/null
+++ b/server/services/videoGen/falFrameAspect.js
@@ -0,0 +1,39 @@
+/**
+ * fal.ai clip frames at the project's aspect.
+ *
+ * Most fal image-to-video and audio-driven routes (OmniHuman, Hailuo-02, Kling)
+ * take no aspect parameter: the output canvas follows the start frame. A square
+ * still therefore became a 1080x1080 lip-sync clip that pillarboxes in a 16:9
+ * music video. A Music Video clip names its project's aspect as `frameAspect`,
+ * and the start and end frames are cropped to it here, in memory, just before
+ * upload. The gallery image stays untouched, so the take still records the
+ * frame the director picked. A frame within the shared tolerance uploads
+ * byte-for-byte; anything else is attention-cropped (a close-up keeps its face)
+ * to the largest rectangle of that ratio, never upscaled.
+ */
+import sharp from 'sharp';
+import { fileToDataUri } from '../falQueue.js';
+import { aspectCropSize } from '../imageGen/aspectConform.js';
+
+const RATIO = /^(\d+):(\d+)$/;
+
+/**
+ * The frame at `filePath` as a data URI, cropped to `aspect` (`"16:9"`) when it
+ * is off. Returns `{ dataUri, conformed }`; `conformed` is `{ from, to }`
+ * (`"WxH"`) when it cropped, else null. No aspect, or a frame sharp cannot
+ * read, uploads as is.
+ */
+export async function frameDataUriAtAspect(filePath, aspect) {
+ const ratio = RATIO.exec(typeof aspect === 'string' ? aspect : '');
+ const meta = ratio ? await sharp(filePath).metadata().catch(() => null) : null;
+ const crop = meta && aspectCropSize(meta.width, meta.height, Number(ratio[1]), Number(ratio[2]));
+ if (!crop) return { dataUri: await fileToDataUri(filePath), conformed: null };
+ const buffer = await sharp(filePath)
+ .resize(crop.width, crop.height, { fit: 'cover', position: sharp.strategy.attention })
+ .png()
+ .toBuffer();
+ return {
+ dataUri: `data:image/png;base64,${buffer.toString('base64')}`,
+ conformed: { from: `${meta.width}x${meta.height}`, to: `${crop.width}x${crop.height}` },
+ };
+}
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index 986ccb68d1..96294fe1d0 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -35,6 +35,13 @@ import {
import { VIDEO_GEN_LOCAL_ONLY_FIELD_NAMES } from './requestFields.js';
import { isTruthyMeta } from '../../lib/metadataFlags.js';
+
+const musicVideoProjectAspect = async (projectId) => {
+ const [{ getProject }, { musicVideoAspect }] = await Promise.all([
+ import('../musicVideo/projects.js'), import('../../lib/musicVideoAspect.js'),
+ ]);
+ return musicVideoAspect(await getProject(projectId));
+};
const submitValidatedVideoGenJob = async (body, uploads) => {
let fableLoomRenderSettings = null;
if (body.fableLoom) {
@@ -348,6 +355,13 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
const hosted = HOSTED_VIDEO_SUBMISSIONS[backend];
if (hosted) {
const { sourceImagePath, uploadedTempPath } = prepared;
+ // A Music Video fal clip renders at its project's aspect: requested where
+ // the model takes an aspect ratio, and its frames cropped to it where the
+ // output follows the start frame (falFrameAspect.js). An explicit
+ // visual-conditioning aspect (FableLoom) still wins.
+ const projectAspect = backend === VIDEO_GEN_MODE.FAL && body.musicVideo && !body.visualConditioning?.render?.parameters?.aspectRatio
+ ? await musicVideoProjectAspect(body.musicVideo.projectId).catch(async (error) => { await cleanupStaged(); throw error; })
+ : null;
const { jobId, position, status } = await enqueue({
// Hosted jobs use mode for queue dispatch and videoMode for text/image.
mode: backend,
@@ -360,6 +374,7 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
...(body.fableLoom ? { fableLoom: body.fableLoom } : {}),
...(body.visualConditioning ? { visualConditioning: body.visualConditioning } : {}),
...hosted.buildParams(body, prepared),
+ ...(projectAspect ? { aspectRatio: projectAspect, frameAspect: projectAspect } : {}),
// standalone fal lip sync: the uploaded voice clip (a Music Video performance shot overrides below)
...(prepared.audioFilePath ? { audioFilePath: prepared.audioFilePath, uploadedTempPaths: prepared.uploadedTempPaths, lipSync: { enableTranscription: false } } : {}),
...performanceParams,
diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.js
index ddb8d02742..a9049762ae 100644
--- a/server/services/videoGen/submitJob.test.js
+++ b/server/services/videoGen/submitJob.test.js
@@ -246,6 +246,19 @@ describe('submitVideoGenJob', () => {
expect(params.generateAudio).toBeUndefined();
});
+ it('asks for the project aspect and crops the clip frames to it (a 9:16 project)', async () => {
+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());
+ mocks.getMusicVideoProject.mockResolvedValue({ id: 'mv-1', scenes: [{ sceneId: 'mvs-1' }], treatment: { brief: { aspectRatio: '9:16' } } });
+ await submitVideoGenJob({ prompt: 'singer', backend: 'fal', falModelId: 'fal-ai/bytedance/omnihuman/v1.5', musicVideo, mode: 'image' }, {});
+ expect(mocks.enqueueJob.mock.calls[0][0].params).toMatchObject({ aspectRatio: '9:16', frameAspect: '9:16' });
+ });
+
+ it('leaves a clip outside a music video at its own aspect', async () => {
+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());
+ await submitVideoGenJob({ prompt: 'a fox', backend: 'fal', mode: 'image' }, {});
+ expect(mocks.enqueueJob.mock.calls[0][0].params).not.toHaveProperty('frameAspect');
+ });
+
it('refuses a performance shot the backend cannot render and releases what was staged', async () => {
const prepared = { ...falPrepared(), backend: 'grok', grok: { grokPath: '/example/grok' } };
mocks.prepareVideoGenParams.mockResolvedValue(prepared);