Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Codex stills are cropped to the requested aspect ratio (#11203)
[._.]/59cd30b70222
diff --git a/server/services/imageGen/aspectConform.js b/server/services/imageGen/aspectConform.jsindex b6c351a6a3..dc00bfbf2e 100644--- a/server/services/imageGen/aspectConform.js+++ b/server/services/imageGen/aspectConform.js@@ -1,25 +1,36 @@/**- * Image Gen — the crop box that conforms a frame to a requested aspect ratio.+ * Image Gen — conform a provider's output to the requested aspect ratio.*- * Image backends do not always return the asked-for shape (Codex answered a- * 16:9 music video request with 1254x1254 close-ups, near-4:3 and 3:2 frames),- * and every clip lane downstream inherits that shape: an OmniHuman lip-sync clip- * from a square still is 1080x1080. A frame within `ASPECT_TOLERANCE` of the- * requested ratio is left alone; anything else is cropped to the largest- * rectangle of that ratio, never upscaled. The fal clip lane applies it to the- * frames it uploads (videoGen/falFrameAspect.js).+ * Codex's built-in image_gen tool takes no size argument: the requested canvas+ * rides the prompt as a hint and the model picks one of its native sizes. Most+ * of the time that lands on the asked-for shape, but not always — a 16:9 music+ * video production came back with 1254x1254 squares (mostly performance+ * close-ups), near-4:3 frames and 1536x1024 3:2 frames mixed in with its+ * 1672x941 widescreen stills, and every clip lane downstream inherited the+ * wrong shape (an OmniHuman lip-sync clip from a square still is 1080x1080).+ *+ * So the provider's completion handler runs the written frame through here+ * before the sidecar is written: a frame within `ASPECT_TOLERANCE` of the+ * requested ratio is left byte-for-byte alone; anything else is cropped to the+ * largest rectangle of the requested ratio, positioned by sharp's attention+ * strategy so a close-up keeps the face rather than a geometric centre. The+ * crop never upscales — it keeps the provider's native resolution on the long+ * side it can afford. The fal clip lane applies the same crop box to the frames+ * it uploads (videoGen/falFrameAspect.js).*/+import sharp from 'sharp';+import { atomicWrite } from '../../lib/fileUtils.js';+// 2% — 1672x941 (1.7768) is Codex's own 16:9 and must pass untouched against a// 1.7778 request; the nearest wrong shape it returns (3:2, 1.5) is ~16% off.-export const ASPECT_TOLERANCE = 0.02;+const ASPECT_TOLERANCE = 0.02;const positive = (n) => Number.isFinite(Number(n)) && Number(n) > 0;/*** The crop box that conforms a `srcW`x`srcH` frame to `targetW:targetH`, or* `null` when the frame is already within tolerance (or the input is unusable).- * Pure — exported for the unit test.** @returns {{ width: number, height: number }|null}*/@@ -33,3 +44,21 @@ export function aspectCropSize(srcW, srcH, targetW, targetH, tolerance = ASPECT_const height = actual > target ? srcH : Math.round(srcW / target);return { width: Math.max(1, width), height: Math.max(1, height) };}++/**+ * Crop the PNG at `pngPath` in place to the requested aspect ratio when it is+ * off. Returns `{ from, to }` (`"WxH"` strings) when it cropped, else `null`.+ * A frame sharp cannot read is left for the existing invalid-image paths.+ */+export async function conformImageAspect(pngPath, targetW, targetH) {+ if (!positive(targetW) || !positive(targetH)) return null;+ const meta = await sharp(pngPath).metadata().catch(() => null);+ const crop = aspectCropSize(meta?.width, meta?.height, Number(targetW), Number(targetH));+ if (!crop) return null;+ const buffer = await sharp(pngPath)+ .resize(crop.width, crop.height, { fit: 'cover', position: sharp.strategy.attention })+ .png()+ .toBuffer();+ await atomicWrite(pngPath, buffer);+ return { from: `${meta.width}x${meta.height}`, to: `${crop.width}x${crop.height}` };+}diff --git a/server/services/imageGen/aspectConform.test.js b/server/services/imageGen/aspectConform.test.jsindex 0204ac86e1..c6b3259352 100644--- a/server/services/imageGen/aspectConform.test.js+++ b/server/services/imageGen/aspectConform.test.js@@ -1,20 +1,48 @@-import { describe, it, expect } from 'vitest';-import { aspectCropSize } from './aspectConform.js';+import { describe, it, expect, beforeAll, afterAll } from 'vitest';+import { mkdtemp, rm, writeFile, readFile } from 'fs/promises';+import { tmpdir } from 'os';+import { join } from 'path';+import sharp from 'sharp';+import { conformImageAspect } from './aspectConform.js';++// The crop over the shapes Codex actually returned for 16:9 production stills:+// pins the tolerance boundary (its own 1672x941 must pass byte-for-byte) and+// both crop directions. The provider wiring + sidecar is covered in codex.test.js,+// the clip-lane use of the same crop box in videoGen/fal.test.js.+describe('conformImageAspect', () => {+ let dir;+ beforeAll(async () => { dir = await mkdtemp(join(tmpdir(), 'aspect-conform-')); });+ afterAll(async () => { await rm(dir, { recursive: true, force: true }); });++ // Label-free file names: a test label like "near-4:3" is not a legal+ // Windows path segment.+ let seq = 0;+ const frame = async (width, height) => {+ const path = join(dir, `frame-${seq++}.png`);+ await writeFile(path, await sharp({ create: { width, height, channels: 3, background: { r: 90, g: 60, b: 30 } } }).png().toBuffer());+ return path;+ };++ it.each([+ ['square close-up crops height', [1254, 1254, 1920, 1080], { from: '1254x1254', to: '1254x705' }],+ ['near-4:3 crops height', [1448, 1086, 1920, 1080], { from: '1448x1086', to: '1448x815' }],+ ['3:2 crops height', [1536, 1024, 1920, 1080], { from: '1536x1024', to: '1536x864' }],+ ['too-wide frame crops width', [2000, 800, 1920, 1080], { from: '2000x800', to: '1422x800' }],+ ['portrait request from a landscape frame', [1536, 1024, 1024, 1536], { from: '1536x1024', to: '683x1024' }],+ ])('%s', async (_label, [sw, sh, tw, th], expected) => {+ const path = await frame(sw, sh);+ expect(await conformImageAspect(path, tw, th)).toEqual(expected);+ const [w, h] = expected.to.split('x').map(Number);+ expect(await sharp(path).metadata()).toMatchObject({ width: w, height: h });+ });-// The crop-box arithmetic, over the shapes Codex actually returned for 16:9-// production stills. The end-to-end crop is covered in videoGen/fal.test.js;-// this pins the tolerance boundary and both crop directions.-describe('aspectCropSize', () => {it.each([- ['Codex 16:9 passes untouched', [1672, 941, 1920, 1080], null],- ['square close-up crops height', [1254, 1254, 1920, 1080], { width: 1254, height: 705 }],- ['near-4:3 crops height', [1448, 1086, 1920, 1080], { width: 1448, height: 815 }],- ['3:2 crops height', [1536, 1024, 1920, 1080], { width: 1536, height: 864 }],- ['too-wide frame crops width', [2000, 800, 1920, 1080], { width: 1422, height: 800 }],- ['portrait request from a landscape frame', [1536, 1024, 1024, 1536], { width: 683, height: 1024 }],- ['unusable size is left alone', [0, 1024, 1920, 1080], null],- ['no requested size is left alone', [1254, 1254, null, null], null],- ])('%s', (_label, [sw, sh, tw, th], expected) => {- expect(aspectCropSize(sw, sh, tw, th)).toEqual(expected);+ ['Codex 16:9 within tolerance', [1672, 941], [1920, 1080]],+ ['no requested size', [1254, 1254], [null, null]],+ ])('leaves %s untouched', async (_label, [sw, sh], [tw, th]) => {+ const path = await frame(sw, sh);+ const before = await readFile(path);+ expect(await conformImageAspect(path, tw, th)).toBeNull();+ expect(Buffer.compare(await readFile(path), before)).toBe(0);});});diff --git a/server/services/imageGen/codex.js b/server/services/imageGen/codex.jsindex 04898481c3..4620d8c1fb 100644--- a/server/services/imageGen/codex.js+++ b/server/services/imageGen/codex.js@@ -51,6 +51,8 @@ import {} from './modes.js';import { buildNoImageReason } from './noImageReason.js';import { rejectDegenerateFrame } from './frameGuard.js';+import { conformImageAspect } from './aspectConform.js';+import { aspectRatioPhrase } from '../../lib/aspectRatio.js';import { resolveInputImages } from './inputImages.js';import { cloudPromptRequired } from './cloudProviderConfig.js';@@ -238,7 +240,11 @@ export async function generateImage({// gpt-image-2 supports up to 4K output; the "(high quality)" suffix// pushes it off its 1024 default toward whatever native size best fits// the requested aspect ratio.- const sizeHint = (width && height) ? ` (${width}x${height})` : '';+ // The ratio is named as well as the pixels: a bare "(1672x941)" left the+ // model free to return a square close-up, so the frame is also conformed+ // after harvest (aspectConform.js) — the hint just makes that crop rarer.+ const ratioPhrase = (width && height) ? aspectRatioPhrase(Number(width), Number(height)) : '';+ const sizeHint = (width && height) ? ` (${ratioPhrase ? `${ratioPhrase} frame, ` : ''}${width}x${height})` : '';const qualityHint = (width >= 1536 || height >= 1536) ? ' (high quality)' : '';const avoidHint = negativePrompt?.trim() ? `\nAvoid: ${negativePrompt.trim()}` : '';const editPrefix = buildCodexAttachmentPrefix({@@ -429,6 +435,10 @@ function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PAif (emptyFrame) {return finalizeJobFailure(job, jobId, proc, emptyFrame);}+ // Codex picks its own output size; crop an off-aspect frame to the+ // requested ratio before anything downstream inherits its shape.+ const aspectConformed = await conformImageAspect(outputPath, meta.width, meta.height);+ if (aspectConformed) console.log(`✂️ Conformed codex frame [${jobId.slice(0, 8)}] ${aspectConformed.from} → ${aspectConformed.to}`);// Sidecar metadata so the gallery can recover prompt/seed/etc. The// codex sessionId is the closest analogue to a seed for gpt-image-2// (which doesn't expose one) — uniquely identifies the run and is@@ -437,7 +447,7 @@ function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA// `job.renderStartedAtMs` is the queue-ingestion instant generateImage// captured, so the spread measures the render itself — not the time the// job spent queued behind other renders.- await atomicWrite(sidecar, { ...meta, codexSessionId: sessionId, ...renderTimingFields(job.renderStartedAtMs) });+ await atomicWrite(sidecar, { ...meta, codexSessionId: sessionId, ...(aspectConformed ? { aspectConformed } : {}), ...renderTimingFields(job.renderStartedAtMs) });// Cleaners run BEFORE the SSE complete + completed events so subscribers// see the cleaned bytes. codex output is the highest-value target for// C2PA stripping because gpt-image is the one provider that embedsdiff --git a/server/services/imageGen/codex.test.js b/server/services/imageGen/codex.test.jsindex 6ba83cf352..8667ced974 100644--- a/server/services/imageGen/codex.test.js+++ b/server/services/imageGen/codex.test.js@@ -4,6 +4,7 @@ import { existsSync } from 'fs';import { join } from 'path';import { tmpdir } from 'os';import { EventEmitter } from 'events';+import sharp from 'sharp';import { ChildProcess } from 'child_process';import { pinPlatform } from '../../lib/testHelper.js';@@ -194,7 +195,7 @@ describe('codex provider — generateImage', () => {it('appends a "(high quality)" hint when width or height ≥ 1536', async () => {await codex.generateImage({ prompt: 'a fox', width: 1536, height: 1024 });const promptArg = spawnCalls[0].args[spawnCalls[0].args.length - 1];- expect(promptArg).toContain('(1536x1024)');+ expect(promptArg).toContain('(3:2 landscape frame, 1536x1024)');expect(promptArg).toContain('(high quality)');spawnCalls[0].child.exitCode = 1;spawnCalls[0].child.emit('close', 1, null);@@ -204,7 +205,7 @@ describe('codex provider — generateImage', () => {it('does not append "(high quality)" for sub-1536 dimensions', async () => {await codex.generateImage({ prompt: 'a fox', width: 1024, height: 1024 });const promptArg = spawnCalls[0].args[spawnCalls[0].args.length - 1];- expect(promptArg).toContain('(1024x1024)');+ expect(promptArg).toContain('(1:1 square frame, 1024x1024)');expect(promptArg).not.toContain('high quality');spawnCalls[0].child.exitCode = 1;spawnCalls[0].child.emit('close', 1, null);@@ -454,6 +455,36 @@ describe('codex provider — image harvest', () => {expect(existsSync(sidecar)).toBe(true);});+ it('crops an off-aspect frame to the requested ratio and records it in the sidecar', async () => {+ const sessionId = '019dd59e-a8da-7bd2-a7d8-4ac6f46e7b99';+ const codexDir = join(TEST_HOME, '.codex', 'generated_images', sessionId);+ await mkdir(codexDir, { recursive: true });+ // Codex answered a 16:9 request with a square close-up.+ const square = await sharp({ create: { width: 640, height: 640, channels: 3, background: { r: 200, g: 120, b: 40 } } })+ .composite([{ input: { create: { width: 200, height: 200, channels: 3, background: { r: 20, g: 40, b: 220 } } }, top: 100, left: 220 }])+ .png().toBuffer();+ await writeFile(join(codexDir, 'ig_square.png'), square);++ const completedListener = vi.fn();+ imageGenEvents.on('completed', completedListener);+ const job = await codex.generateImage({ prompt: 'a singer close-up', width: 1600, height: 900 });+ const child = spawnCalls[0].child;+ child.stderr.emit('data', Buffer.from(`session id: ${sessionId}\n`));+ child.exitCode = 0;+ child.emit('close', 0, null);+ const deadline = Date.now() + 3000;+ while (Date.now() < deadline && completedListener.mock.calls.length === 0) {+ await new Promise((r) => setTimeout(r, 50));+ }+ imageGenEvents.off('completed', completedListener);++ expect(completedListener).toHaveBeenCalledTimes(1);+ const { width, height } = await sharp(join(FAKE_IMAGES_DIR, job.filename)).metadata();+ expect({ width, height }).toEqual({ width: 640, height: 360 });+ const sidecar = JSON.parse(await readFile(join(FAKE_IMAGES_DIR, `${job.generationId}.metadata.json`), 'utf8'));+ expect(sidecar.aspectConformed).toEqual({ from: '640x640', to: '640x360' });+ });+it('harvests the exec-named PNG emitted by newer Codex image tools', async () => {const sessionId = '33333333-3333-4333-8333-333333333333';const codexDir = join(TEST_HOME, '.codex', 'generated_images', sessionId);