Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Codex stills are cropped to the requested aspect ratio (#11203)

[._.]/59cd30b70222
diff --git a/server/services/imageGen/aspectConform.js b/server/services/imageGen/aspectConform.js
index b6c351a6a3..dc00bfbf2e 100644
--- a/server/services/imageGen/aspectConform.js
+++ b/server/services/imageGen/aspectConform.js
@@ -1,25 +1,36 @@
/**
- * Image Gen — the crop box that conforms a frame to a requested aspect ratio.
+ * Image Gen — conform a provider's output to the requested aspect ratio.
*
- * Image backends do not always return the asked-for shape (Codex answered a
- * 16:9 music video request with 1254x1254 close-ups, near-4:3 and 3:2 frames),
- * and every clip lane downstream inherits that shape: an OmniHuman lip-sync clip
- * from a square still is 1080x1080. A frame within `ASPECT_TOLERANCE` of the
- * requested ratio is left alone; anything else is cropped to the largest
- * rectangle of that ratio, never upscaled. The fal clip lane applies it to the
- * frames it uploads (videoGen/falFrameAspect.js).
+ * Codex's built-in image_gen tool takes no size argument: the requested canvas
+ * rides the prompt as a hint and the model picks one of its native sizes. Most
+ * of the time that lands on the asked-for shape, but not always — a 16:9 music
+ * video production came back with 1254x1254 squares (mostly performance
+ * close-ups), near-4:3 frames and 1536x1024 3:2 frames mixed in with its
+ * 1672x941 widescreen stills, and every clip lane downstream inherited the
+ * wrong shape (an OmniHuman lip-sync clip from a square still is 1080x1080).
+ *
+ * So the provider's completion handler runs the written frame through here
+ * before the sidecar is written: a frame within `ASPECT_TOLERANCE` of the
+ * requested ratio is left byte-for-byte alone; anything else is cropped to the
+ * largest rectangle of the requested ratio, positioned by sharp's attention
+ * strategy so a close-up keeps the face rather than a geometric centre. The
+ * crop never upscales — it keeps the provider's native resolution on the long
+ * side it can afford. The fal clip lane applies the same crop box to the frames
+ * it uploads (videoGen/falFrameAspect.js).
*/
+import sharp from 'sharp';
+import { atomicWrite } from '../../lib/fileUtils.js';
+
// 2% — 1672x941 (1.7768) is Codex's own 16:9 and must pass untouched against a
// 1.7778 request; the nearest wrong shape it returns (3:2, 1.5) is ~16% off.
-export const ASPECT_TOLERANCE = 0.02;
+const ASPECT_TOLERANCE = 0.02;
const positive = (n) => Number.isFinite(Number(n)) && Number(n) > 0;
/**
* The crop box that conforms a `srcW`x`srcH` frame to `targetW:targetH`, or
* `null` when the frame is already within tolerance (or the input is unusable).
- * Pure — exported for the unit test.
*
* @returns {{ width: number, height: number }|null}
*/
@@ -33,3 +44,21 @@ export function aspectCropSize(srcW, srcH, targetW, targetH, tolerance = ASPECT_
const height = actual > target ? srcH : Math.round(srcW / target);
return { width: Math.max(1, width), height: Math.max(1, height) };
}
+
+/**
+ * Crop the PNG at `pngPath` in place to the requested aspect ratio when it is
+ * off. Returns `{ from, to }` (`"WxH"` strings) when it cropped, else `null`.
+ * A frame sharp cannot read is left for the existing invalid-image paths.
+ */
+export async function conformImageAspect(pngPath, targetW, targetH) {
+ if (!positive(targetW) || !positive(targetH)) return null;
+ const meta = await sharp(pngPath).metadata().catch(() => null);
+ const crop = aspectCropSize(meta?.width, meta?.height, Number(targetW), Number(targetH));
+ if (!crop) return null;
+ const buffer = await sharp(pngPath)
+ .resize(crop.width, crop.height, { fit: 'cover', position: sharp.strategy.attention })
+ .png()
+ .toBuffer();
+ await atomicWrite(pngPath, buffer);
+ return { from: `${meta.width}x${meta.height}`, to: `${crop.width}x${crop.height}` };
+}
diff --git a/server/services/imageGen/aspectConform.test.js b/server/services/imageGen/aspectConform.test.js
index 0204ac86e1..c6b3259352 100644
--- a/server/services/imageGen/aspectConform.test.js
+++ b/server/services/imageGen/aspectConform.test.js
@@ -1,20 +1,48 @@
-import { describe, it, expect } from 'vitest';
-import { aspectCropSize } from './aspectConform.js';
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { mkdtemp, rm, writeFile, readFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import sharp from 'sharp';
+import { conformImageAspect } from './aspectConform.js';
+
+// The crop over the shapes Codex actually returned for 16:9 production stills:
+// pins the tolerance boundary (its own 1672x941 must pass byte-for-byte) and
+// both crop directions. The provider wiring + sidecar is covered in codex.test.js,
+// the clip-lane use of the same crop box in videoGen/fal.test.js.
+describe('conformImageAspect', () => {
+ let dir;
+ beforeAll(async () => { dir = await mkdtemp(join(tmpdir(), 'aspect-conform-')); });
+ afterAll(async () => { await rm(dir, { recursive: true, force: true }); });
+
+ // Label-free file names: a test label like "near-4:3" is not a legal
+ // Windows path segment.
+ let seq = 0;
+ const frame = async (width, height) => {
+ const path = join(dir, `frame-${seq++}.png`);
+ await writeFile(path, await sharp({ create: { width, height, channels: 3, background: { r: 90, g: 60, b: 30 } } }).png().toBuffer());
+ return path;
+ };
+
+ it.each([
+ ['square close-up crops height', [1254, 1254, 1920, 1080], { from: '1254x1254', to: '1254x705' }],
+ ['near-4:3 crops height', [1448, 1086, 1920, 1080], { from: '1448x1086', to: '1448x815' }],
+ ['3:2 crops height', [1536, 1024, 1920, 1080], { from: '1536x1024', to: '1536x864' }],
+ ['too-wide frame crops width', [2000, 800, 1920, 1080], { from: '2000x800', to: '1422x800' }],
+ ['portrait request from a landscape frame', [1536, 1024, 1024, 1536], { from: '1536x1024', to: '683x1024' }],
+ ])('%s', async (_label, [sw, sh, tw, th], expected) => {
+ const path = await frame(sw, sh);
+ expect(await conformImageAspect(path, tw, th)).toEqual(expected);
+ const [w, h] = expected.to.split('x').map(Number);
+ expect(await sharp(path).metadata()).toMatchObject({ width: w, height: h });
+ });
-// The crop-box arithmetic, over the shapes Codex actually returned for 16:9
-// production stills. The end-to-end crop is covered in videoGen/fal.test.js;
-// this pins the tolerance boundary and both crop directions.
-describe('aspectCropSize', () => {
it.each([
- ['Codex 16:9 passes untouched', [1672, 941, 1920, 1080], null],
- ['square close-up crops height', [1254, 1254, 1920, 1080], { width: 1254, height: 705 }],
- ['near-4:3 crops height', [1448, 1086, 1920, 1080], { width: 1448, height: 815 }],
- ['3:2 crops height', [1536, 1024, 1920, 1080], { width: 1536, height: 864 }],
- ['too-wide frame crops width', [2000, 800, 1920, 1080], { width: 1422, height: 800 }],
- ['portrait request from a landscape frame', [1536, 1024, 1024, 1536], { width: 683, height: 1024 }],
- ['unusable size is left alone', [0, 1024, 1920, 1080], null],
- ['no requested size is left alone', [1254, 1254, null, null], null],
- ])('%s', (_label, [sw, sh, tw, th], expected) => {
- expect(aspectCropSize(sw, sh, tw, th)).toEqual(expected);
+ ['Codex 16:9 within tolerance', [1672, 941], [1920, 1080]],
+ ['no requested size', [1254, 1254], [null, null]],
+ ])('leaves %s untouched', async (_label, [sw, sh], [tw, th]) => {
+ const path = await frame(sw, sh);
+ const before = await readFile(path);
+ expect(await conformImageAspect(path, tw, th)).toBeNull();
+ expect(Buffer.compare(await readFile(path), before)).toBe(0);
});
});
diff --git a/server/services/imageGen/codex.js b/server/services/imageGen/codex.js
index 04898481c3..4620d8c1fb 100644
--- a/server/services/imageGen/codex.js
+++ b/server/services/imageGen/codex.js
@@ -51,6 +51,8 @@ import {
} from './modes.js';
import { buildNoImageReason } from './noImageReason.js';
import { rejectDegenerateFrame } from './frameGuard.js';
+import { conformImageAspect } from './aspectConform.js';
+import { aspectRatioPhrase } from '../../lib/aspectRatio.js';
import { resolveInputImages } from './inputImages.js';
import { cloudPromptRequired } from './cloudProviderConfig.js';
@@ -238,7 +240,11 @@ export async function generateImage({
// gpt-image-2 supports up to 4K output; the "(high quality)" suffix
// pushes it off its 1024 default toward whatever native size best fits
// the requested aspect ratio.
- const sizeHint = (width && height) ? ` (${width}x${height})` : '';
+ // The ratio is named as well as the pixels: a bare "(1672x941)" left the
+ // model free to return a square close-up, so the frame is also conformed
+ // after harvest (aspectConform.js) — the hint just makes that crop rarer.
+ const ratioPhrase = (width && height) ? aspectRatioPhrase(Number(width), Number(height)) : '';
+ const sizeHint = (width && height) ? ` (${ratioPhrase ? `${ratioPhrase} frame, ` : ''}${width}x${height})` : '';
const qualityHint = (width >= 1536 || height >= 1536) ? ' (high quality)' : '';
const avoidHint = negativePrompt?.trim() ? `\nAvoid: ${negativePrompt.trim()}` : '';
const editPrefix = buildCodexAttachmentPrefix({
@@ -429,6 +435,10 @@ function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA
if (emptyFrame) {
return finalizeJobFailure(job, jobId, proc, emptyFrame);
}
+ // Codex picks its own output size; crop an off-aspect frame to the
+ // requested ratio before anything downstream inherits its shape.
+ const aspectConformed = await conformImageAspect(outputPath, meta.width, meta.height);
+ if (aspectConformed) console.log(`✂️ Conformed codex frame [${jobId.slice(0, 8)}] ${aspectConformed.from} → ${aspectConformed.to}`);
// Sidecar metadata so the gallery can recover prompt/seed/etc. The
// codex sessionId is the closest analogue to a seed for gpt-image-2
// (which doesn't expose one) — uniquely identifies the run and is
@@ -437,7 +447,7 @@ function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA
// `job.renderStartedAtMs` is the queue-ingestion instant generateImage
// captured, so the spread measures the render itself — not the time the
// job spent queued behind other renders.
- await atomicWrite(sidecar, { ...meta, codexSessionId: sessionId, ...renderTimingFields(job.renderStartedAtMs) });
+ await atomicWrite(sidecar, { ...meta, codexSessionId: sessionId, ...(aspectConformed ? { aspectConformed } : {}), ...renderTimingFields(job.renderStartedAtMs) });
// Cleaners run BEFORE the SSE complete + completed events so subscribers
// see the cleaned bytes. codex output is the highest-value target for
// C2PA stripping because gpt-image is the one provider that embeds
diff --git a/server/services/imageGen/codex.test.js b/server/services/imageGen/codex.test.js
index 6ba83cf352..8667ced974 100644
--- a/server/services/imageGen/codex.test.js
+++ b/server/services/imageGen/codex.test.js
@@ -4,6 +4,7 @@ import { existsSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { EventEmitter } from 'events';
+import sharp from 'sharp';
import { ChildProcess } from 'child_process';
import { pinPlatform } from '../../lib/testHelper.js';
@@ -194,7 +195,7 @@ describe('codex provider — generateImage', () => {
it('appends a "(high quality)" hint when width or height ≥ 1536', async () => {
await codex.generateImage({ prompt: 'a fox', width: 1536, height: 1024 });
const promptArg = spawnCalls[0].args[spawnCalls[0].args.length - 1];
- expect(promptArg).toContain('(1536x1024)');
+ expect(promptArg).toContain('(3:2 landscape frame, 1536x1024)');
expect(promptArg).toContain('(high quality)');
spawnCalls[0].child.exitCode = 1;
spawnCalls[0].child.emit('close', 1, null);
@@ -204,7 +205,7 @@ describe('codex provider — generateImage', () => {
it('does not append "(high quality)" for sub-1536 dimensions', async () => {
await codex.generateImage({ prompt: 'a fox', width: 1024, height: 1024 });
const promptArg = spawnCalls[0].args[spawnCalls[0].args.length - 1];
- expect(promptArg).toContain('(1024x1024)');
+ expect(promptArg).toContain('(1:1 square frame, 1024x1024)');
expect(promptArg).not.toContain('high quality');
spawnCalls[0].child.exitCode = 1;
spawnCalls[0].child.emit('close', 1, null);
@@ -454,6 +455,36 @@ describe('codex provider — image harvest', () => {
expect(existsSync(sidecar)).toBe(true);
});
+ it('crops an off-aspect frame to the requested ratio and records it in the sidecar', async () => {
+ const sessionId = '019dd59e-a8da-7bd2-a7d8-4ac6f46e7b99';
+ const codexDir = join(TEST_HOME, '.codex', 'generated_images', sessionId);
+ await mkdir(codexDir, { recursive: true });
+ // Codex answered a 16:9 request with a square close-up.
+ const square = await sharp({ create: { width: 640, height: 640, channels: 3, background: { r: 200, g: 120, b: 40 } } })
+ .composite([{ input: { create: { width: 200, height: 200, channels: 3, background: { r: 20, g: 40, b: 220 } } }, top: 100, left: 220 }])
+ .png().toBuffer();
+ await writeFile(join(codexDir, 'ig_square.png'), square);
+
+ const completedListener = vi.fn();
+ imageGenEvents.on('completed', completedListener);
+ const job = await codex.generateImage({ prompt: 'a singer close-up', width: 1600, height: 900 });
+ const child = spawnCalls[0].child;
+ child.stderr.emit('data', Buffer.from(`session id: ${sessionId}\n`));
+ child.exitCode = 0;
+ child.emit('close', 0, null);
+ const deadline = Date.now() + 3000;
+ while (Date.now() < deadline && completedListener.mock.calls.length === 0) {
+ await new Promise((r) => setTimeout(r, 50));
+ }
+ imageGenEvents.off('completed', completedListener);
+
+ expect(completedListener).toHaveBeenCalledTimes(1);
+ const { width, height } = await sharp(join(FAKE_IMAGES_DIR, job.filename)).metadata();
+ expect({ width, height }).toEqual({ width: 640, height: 360 });
+ const sidecar = JSON.parse(await readFile(join(FAKE_IMAGES_DIR, `${job.generationId}.metadata.json`), 'utf8'));
+ expect(sidecar.aspectConformed).toEqual({ from: '640x640', to: '640x360' });
+ });
+
it('harvests the exec-named PNG emitted by newer Codex image tools', async () => {
const sessionId = '33333333-3333-4333-8333-333333333333';
const codexDir = join(TEST_HOME, '.codex', 'generated_images', sessionId);