Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: settle image CLI jobs when launch preparation throws (#10749)
Adam Eivy16d69590ed40
diff --git a/server/services/imageGen/agy.js b/server/services/imageGen/agy.jsindex 2b26707b1f..a977a60460 100644--- a/server/services/imageGen/agy.js+++ b/server/services/imageGen/agy.js@@ -299,7 +299,6 @@ export async function generateImage({const outputPath = join(PATHS.images, filename);const scratchDir = join(tmpdir(), `portos-agy-${jobId}`);const stagingPath = join(scratchDir, 'output.png');- await mkdir(scratchDir, { recursive: true });const fullPrompt = buildAgyPrompt({prompt, negativePrompt, width, height, stagingPath, inputImages, initImageStrength,@@ -327,6 +326,9 @@ export async function generateImage({...(visualConditioning ? { visualConditioning } : {}),createdAt: new Date().toISOString(),};+ // The scratch dir is the last fallible step before the job exists, so a+ // refusal above leaves nothing behind; from here on runAgy owns it.+ await mkdir(scratchDir, { recursive: true });const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };jobs.set(jobId, job);activeJobs.set(jobId, {@@ -349,8 +351,6 @@ export async function generateImage({meta,cleanC2PA,denoise,- }).catch((err) => {- console.error(`❌ agy run failed [${jobId.slice(0, 8)}]: ${err?.message}`);});return {@@ -364,7 +364,12 @@ export async function generateImage({};}-async function runAgy(job, jobId, bin, args, {+// Launch is synchronous so every failure has one owner: a refusal before a+// child exists (shim resolution, or Node rejecting an invalid argv/bin+// synchronously) releases the scratch dir and settles the job here; a setup+// failure after spawn terminates the child and leaves the close handler to+// release the scratch dir once it has physically exited.+function runAgy(job, jobId, bin, args, {scratchDir,stagingPath,outputPath,@@ -373,35 +378,31 @@ async function runAgy(job, jobId, bin, args, {cleanC2PA,denoise,}) {- const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;- const { command, args: spawnArgs } = prepareCliSpawn(resolvedBin, args);- // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). agy reads- // process.cwd(), so this is defensive rather than a live fix; it keeps every- // scratch-dir spawn telling the child one consistent story about where it is.- const proc = spawn(command, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: ['ignore', 'pipe', 'pipe'] });- activeProcs.set(jobId, proc);+ // A failed removal keeps the maintenance settlement unsettled rather than+ // claiming the scratch dir was released; removal is idempotent.const removeScratch = () => maintenance.continueSettlement(() => rmGuarded(scratchDir, { recursive: true, force: true }),).catch((err) => {maintenance.markCurrentUnsettled();console.error(`❌ agy scratch cleanup failed: ${err.message}`);});+ let proc;+ try {+ const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;+ const { command, args: spawnArgs } = prepareCliSpawn(resolvedBin, args);+ // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). agy reads+ // process.cwd(), so this is defensive rather than a live fix; it keeps every+ // scratch-dir spawn telling the child one consistent story about where it is.+ proc = spawn(command, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: ['ignore', 'pipe', 'pipe'] });+ } catch (err) {+ removeScratch();+ return finalizeJobFailure(job, jobId, null, `Failed to spawn ${bin}: ${err?.message || err}`);+ }+ activeProcs.set(jobId, proc);let stdoutTail = '';let stderrTail = '';- const timeoutTimer = setTimeout(() => {- if (activeProcs.get(jobId) === proc) {- console.log(`⏱️ agy timed out after ${AGY_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);- terminate(jobId, proc);- }- }, AGY_TIMEOUT_MS);-- proc.stdout.on('data', (chunk) => {- stdoutTail = `${stdoutTail}${chunk}`.slice(-8192);- broadcastSse(job, { type: 'status', message: 'Running…' });- });- proc.stderr.on('data', (chunk) => {- stderrTail = `${stderrTail}${chunk}`.slice(-32768);- });+ let timeoutTimer = null;+ let setupError = null;let processError;proc.on('error', (err) => {processError = err;@@ -415,6 +416,10 @@ async function runAgy(job, jobId, bin, args, {await withBackupAssetPublication(async () => {clearTimeout(timeoutTimer);try {+ if (setupError) {+ removeScratch();+ return finalizeJobFailure(job, jobId, proc, `Agy launch setup failed: ${setupError?.message || setupError}`);+ }if (code !== 0 || processError) {removeScratch();const reason = processError?.message || (signal ? `Killed by signal ${signal}` : `Exit code ${code}`);@@ -488,6 +493,30 @@ async function runAgy(job, jobId, bin, args, {finalizeJobFailure(job, jobId, proc, `Agy publication admission failed: ${err?.message || err}`);});});++ // Post-spawn setup. The error/close owners above are already installed, so a+ // throw here must not settle the job or release its scratch dir under a live+ // child: record it, terminate the child, and let close settle after exit.+ try {+ timeoutTimer = setTimeout(() => {+ if (activeProcs.get(jobId) === proc) {+ console.log(`⏱️ agy timed out after ${AGY_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);+ terminate(jobId, proc);+ }+ }, AGY_TIMEOUT_MS);++ proc.stdout.on('data', (chunk) => {+ stdoutTail = `${stdoutTail}${chunk}`.slice(-8192);+ broadcastSse(job, { type: 'status', message: 'Running…' });+ });+ proc.stderr.on('data', (chunk) => {+ stderrTail = `${stderrTail}${chunk}`.slice(-32768);+ });+ } catch (err) {+ setupError = err;+ clearTimeout(timeoutTimer);+ terminate(jobId, proc);+ }}const finalizeJobFailure = createJobFailureFinalizer({diff --git a/server/services/imageGen/agy.test.js b/server/services/imageGen/agy.test.jsindex e9e888f3fc..5c14631536 100644--- a/server/services/imageGen/agy.test.js+++ b/server/services/imageGen/agy.test.js@@ -51,18 +51,32 @@ vi.mock('../providers.js', () => ({const TEST_ROOT = join(tmpdir(), `portos-agy-test-${process.pid}-${Date.now()}`);const FAKE_IMAGES_DIR = join(TEST_ROOT, 'data-images');+const FAKE_DATA_DIR = join(TEST_ROOT, 'data');vi.mock('../../lib/fileUtils.js', async () => {const actual = await vi.importActual('../../lib/fileUtils.js');actual.PATHS.images = FAKE_IMAGES_DIR;return {...actual,ensureDir: vi.fn(async (dir) => mkdir(dir, { recursive: true })),+ rmGuarded: vi.fn(actual.rmGuarded),};});+// A test-rooted maintenance journal, so cleanup-failure ownership is observable.+const fixture = vi.hoisted(() => ({ admission: null }));+vi.mock('../../lib/maintenanceAdmission.js', async importOriginal => ({+ ...await importOriginal(),+ maintenance: new Proxy({}, { get: (_target, property) => fixture.admission[property] }),+}));const agy = await import('./agy.js');const { AGY_IMAGEGEN_DEFAULT_MODEL } = await import('../../lib/imageGenCapabilities.js');const { imageGenEvents } = await import('../imageGenEvents.js');+const { spawn } = await import('../../lib/childProcess.js');+const { spawn: realSpawn } = await vi.importActual('../../lib/childProcess.js');+const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');+const { createMaintenanceAdmission } = await import('../../lib/maintenanceAdmission.js');+const { rmGuarded } = await import('../../lib/fileUtils.js');+const realFiles = await vi.importActual('../../lib/fileUtils.js');const flush = () => new Promise((resolve) => setImmediate(resolve));const stagingPathFor = (jobId) => join(tmpdir(), `portos-agy-${jobId}`, 'output.png');@@ -78,10 +92,13 @@ beforeEach(async () => {imageGenEvents.removeAllListeners();agy._internals.setHarvestTimeoutForTests(10);await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});- await mkdir(TEST_ROOT, { recursive: true });+ await mkdir(FAKE_DATA_DIR, { recursive: true });+ fixture.admission = createMaintenanceAdmission(FAKE_DATA_DIR);+ rmGuarded.mockReset().mockImplementation(realFiles.rmGuarded);});afterEach(async () => {+ vi.useRealTimers();agy._internals.setHarvestTimeoutForTests();await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});});@@ -410,3 +427,87 @@ it('settles admission rejection from the actual close event and clears the activresponse.req.emit('close');} finally { admissionFault.error = null; }});++// #10749 — a launch that throws must settle through the job's own finalizer and+// release the job-owned scratch dir, not leave a phantom 'running' job whose SSE+// stream never terminates and whose scratch dir leaks.+describe('agy image provider — launch refusal settlement', () => {+ const scratchDirFor = (jobId) => join(tmpdir(), `portos-agy-${jobId}`);+ const sseClient = () => ({ writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: new EventEmitter() });++ it.each([+ // Node validates argv synchronously and throws before any child exists.+ ['a real Node argument refusal', { agyPath: 'agy\0probe' }, () => spawn.mockImplementationOnce(realSpawn), /Failed to spawn/],+ ['a synthetic launch refusal', {}, () => spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); }), /synthetic launch refusal/],+ ])('settles %s once and removes its scratch dir', async (_label, params, arrange, reason) => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ arrange();+ const job = await agy.generateImage({ prompt: 'synthetic', ...params });+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));++ expect(spawnCalls).toHaveLength(0);+ expect(failed).toHaveBeenCalledTimes(1);+ expect(failed.mock.calls[0][0]).toMatchObject({ mode: 'agy', generationId: job.jobId, error: expect.stringMatching(reason) });+ expect(agy.getActiveJob()).toBeNull();+ expect(agy.cancel(job.jobId)).toBe(false);+ const client = sseClient();+ expect(agy.attachSseClient(job.jobId, client)).toBe(true);+ expect(JSON.parse(client.write.mock.calls[0][0].replace(/^data: /, ''))).toMatchObject({ type: 'error', error: expect.stringMatching(reason) });+ client.req.emit('close');++ vi.advanceTimersByTime(SSE_CLEANUP_DELAY_MS);+ expect(agy.attachSseClient(job.jobId, sseClient())).toBe(false);+ });++ it('retains an unsettled maintenance blocker when the refused launch cannot remove its scratch dir', async () => {+ const jobId = 'agy-launch-refusal-cleanup';+ rmGuarded.mockRejectedValueOnce(Object.assign(new Error('Cleanup denied'), { code: 'EACCES' }));+ spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ const permit = fixture.admission.admit('image', jobId);+ await permit.run(() => agy.generateImage({ jobId, prompt: 'synthetic' }));+ fixture.admission.begin({ reason: 'Drain renderer', owner: 'Operator' });+ await vi.waitFor(() => expect(rmGuarded).toHaveBeenCalledTimes(1));+ await flush();+ await permit.finish();+ expect(failed).toHaveBeenCalledTimes(1);+ expect(fixture.admission.status()).toMatchObject({+ state: 'draining', blockers: expect.arrayContaining([expect.objectContaining({ resource: jobId, unsettled: true })]),+ });+ await realFiles.rmGuarded(scratchDirFor(jobId), { recursive: true, force: true });+ });++ it('terminates a live child whose setup fails and releases its scratch dir only after it physically closes', async () => {+ spawn.mockImplementationOnce((bin, args) => {+ const child = makeFakeChild();+ child.stdout.on = () => { throw new Error('synthetic stream setup failure'); };+ spawnCalls.push({ bin, args, child });+ return child;+ });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ const job = await agy.generateImage({ prompt: 'synthetic' });+ const { child } = spawnCalls[0];+ child.pid = 4242;++ expect(child.kill).toHaveBeenCalledWith('SIGTERM');+ child.emit('error', new Error('signal delivery failed'));+ await flush();+ expect(failed).not.toHaveBeenCalled();+ expect(existsSync(scratchDirFor(job.jobId))).toBe(true);+ expect(agy.getActiveJob()).toMatchObject({ generationId: job.jobId });++ child.exitCode = 0;+ child.emit('close', 0, null);+ child.emit('close', 0, null);+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));+ await flush();+ expect(failed).toHaveBeenCalledTimes(1);+ expect(failed.mock.calls[0][0].error).toMatch(/launch setup failed: synthetic stream setup failure/);+ expect(agy.getActiveJob()).toBeNull();+ expect(agy.cancel(job.jobId)).toBe(false);+ });+});diff --git a/server/services/imageGen/codex.js b/server/services/imageGen/codex.jsindex 80d0bf54be..8407dcc879 100644--- a/server/services/imageGen/codex.js+++ b/server/services/imageGen/codex.js@@ -285,10 +285,9 @@ export async function generateImage({// generateImage returns a job descriptor synchronously; the actual codex// child runs out-of-band so the HTTP response can ship while the client- // attaches to the per-job SSE stream (mirrors local.js).- runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA, denoise }).catch((err) => {- console.error(`❌ codex run failed [${jobId.slice(0, 8)}]: ${err?.message}`);- });+ // attaches to the per-job SSE stream (mirrors local.js). runCodex settles+ // every launch failure through the job's own failure finalizer.+ runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA, denoise });return {jobId, filename, path: `/data/images/${filename}`, generationId: jobId,@@ -322,8 +321,17 @@ export const noImageReason = (stdoutTail = '') => buildNoImageReason(stdoutTail,),});-async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA = false, denoise = false } = {}) {- const proc = spawn(bin, args, { shell: false, stdio: ['ignore', 'pipe', 'pipe'] });+// Launch is synchronous so every failure has one owner: a refusal before a+// child exists (Node rejects an invalid argv/bin synchronously) settles the job+// here, and a setup failure after spawn terminates the child and leaves the+// close handler to settle once it has physically exited.+function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA = false, denoise = false } = {}) {+ let proc;+ try {+ proc = spawn(bin, args, { shell: false, stdio: ['ignore', 'pipe', 'pipe'] });+ } catch (err) {+ return finalizeJobFailure(job, jobId, null, `Failed to spawn ${bin}: ${err?.message || err}`);+ }activeProcs.set(jobId, proc);let sessionId = null;@@ -334,13 +342,8 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cle// can surface the model's actual words instead of a generic "no image" guess.let stdoutTail = '';const STDOUT_TAIL_BYTES = 8 * 1024;- let timeoutTimer = setTimeout(() => {- if (activeProcs.get(jobId) === proc) {- console.log(`⏱️ codex timed out after ${CODEX_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);- proc.kill('SIGTERM');- setTimeout(() => { if (proc.exitCode === null) proc.kill('SIGKILL'); }, 5000);- }- }, CODEX_TIMEOUT_MS);+ let timeoutTimer = null;+ let setupError = null;// Banner is roughly 12 lines / ~500 bytes — keep a small rolling// buffer so a session-id line that gets split across chunk boundaries@@ -377,26 +380,6 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { clefinalizeJobFailure(job, jobId, proc, `Failed to spawn ${bin}: ${err.message}`);});- proc.stdout.on('data', (chunk) => {- // Codex prints the `session id:` banner on stderr only — don't feed- // stdout into bannerBuf. A stdout chunk arriving between two stderr- // chunks of the banner can split the session-id line with unrelated- // text and break the regex match.- // We DO retain stdout here (the model's turn narration) so a no-image- // finish can report why — a content decline, or a "generated" claim with- // no file (tool unavailable / image-gen rate-limited on the account).- stdoutTail += chunk.toString();- if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);- broadcastSse(job, { type: 'status', message: 'Running…' });- });-- proc.stderr.on('data', (chunk) => {- const text = chunk.toString();- captureSession(text);- stderrTail += text;- if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);- });-proc.on('close', async (code, signal) => {await withBackupAssetPublication(async () => {clearTimeout(timeoutTimer);@@ -410,6 +393,9 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cle// unhandled rejection (process-killing on Node ≥15) and the job would// be stuck in 'running' forever with no SSE error to the client.try {+ if (setupError) {+ return finalizeJobFailure(job, jobId, proc, `Codex launch setup failed: ${setupError?.message || setupError}`);+ }if (code !== 0 || processError) {const reason = processError?.message || (signal ? `Killed by signal ${signal}` : `Exit code ${code}`);const tail = stderrTail.trim().split('\n').slice(-6).join('\n');@@ -474,6 +460,43 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { clefinalizeJobFailure(job, jobId, proc, `Codex publication admission failed: ${err?.message || err}`);});});++ // Post-spawn setup. The error/close owners above are already installed, so a+ // throw here must not settle the job under a live child: record it,+ // terminate the child, and let close settle once it has physically exited.+ try {+ timeoutTimer = setTimeout(() => {+ if (activeProcs.get(jobId) === proc) {+ console.log(`⏱️ codex timed out after ${CODEX_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);+ proc.kill('SIGTERM');+ setTimeout(() => { if (proc.exitCode === null) proc.kill('SIGKILL'); }, 5000);+ }+ }, CODEX_TIMEOUT_MS);++ proc.stdout.on('data', (chunk) => {+ // Codex prints the `session id:` banner on stderr only — don't feed+ // stdout into bannerBuf. A stdout chunk arriving between two stderr+ // chunks of the banner can split the session-id line with unrelated+ // text and break the regex match.+ // We DO retain stdout here (the model's turn narration) so a no-image+ // finish can report why — a content decline, or a "generated" claim with+ // no file (tool unavailable / image-gen rate-limited on the account).+ stdoutTail += chunk.toString();+ if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);+ broadcastSse(job, { type: 'status', message: 'Running…' });+ });++ proc.stderr.on('data', (chunk) => {+ const text = chunk.toString();+ captureSession(text);+ stderrTail += text;+ if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);+ });+ } catch (err) {+ setupError = err;+ clearTimeout(timeoutTimer);+ sigtermWithEscalation(jobId, proc);+ }}const finalizeJobFailure = createJobFailureFinalizer({diff --git a/server/services/imageGen/codex.test.js b/server/services/imageGen/codex.test.jsindex 1c0ade4cdc..c0474fc1af 100644--- a/server/services/imageGen/codex.test.js+++ b/server/services/imageGen/codex.test.js@@ -71,6 +71,9 @@ vi.mock('../../lib/fileUtils.js', async () => {const { acquireBackupSnapshotCut } = await import('../../lib/backupSnapshotBoundary.js');const codex = await import('./codex.js');const { imageGenEvents } = await import('../imageGenEvents.js');+const { spawn } = await import('../../lib/childProcess.js');+const { spawn: realSpawn } = await vi.importActual('../../lib/childProcess.js');+const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');const flush = () => new Promise((r) => setImmediate(r));@@ -764,3 +767,67 @@ it('settles admission rejection from the actual close event and clears the activresponse.req.emit('close');} finally { admissionFault.error = null; }});++// #10749 — a launch that throws must settle through the job's own finalizer,+// not leave a phantom 'running' job, a never-terminating SSE stream and a+// cancel that cannot find a process.+describe('codex provider — launch refusal settlement', () => {+ const sseClient = () => ({ writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: new EventEmitter() });++ it.each([+ // Node validates argv synchronously and throws before any child exists.+ ['a real Node argument refusal', { codexPath: 'codex\0probe' }, () => spawn.mockImplementationOnce(realSpawn), /without null bytes/],+ ['a synthetic launch refusal', {}, () => spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); }), /synthetic launch refusal/],+ ])('settles %s exactly once with a replayable terminal frame', async (_label, params, arrange, reason) => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ arrange();+ const job = await codex.generateImage({ prompt: 'synthetic', ...params });++ expect(spawnCalls).toHaveLength(0);+ expect(failed).toHaveBeenCalledTimes(1);+ expect(failed.mock.calls[0][0]).toMatchObject({ mode: 'codex', generationId: job.jobId, error: expect.stringMatching(reason) });+ expect(codex.getActiveJob()).toBeNull();+ expect(codex.cancel(job.jobId)).toBe(false);+ const client = sseClient();+ expect(codex.attachSseClient(job.jobId, client)).toBe(true);+ expect(JSON.parse(client.write.mock.calls[0][0].replace(/^data: /, ''))).toMatchObject({ type: 'error', error: expect.stringMatching(reason) });+ client.req.emit('close');++ vi.advanceTimersByTime(SSE_CLEANUP_DELAY_MS);+ expect(codex.attachSseClient(job.jobId, sseClient())).toBe(false);+ });++ it('terminates a live child whose setup fails and settles once, only after it physically closes', async () => {+ spawn.mockImplementationOnce((bin, args, options) => {+ const child = makeFakeChild();+ child.stdout.on = () => { throw new Error('synthetic stream setup failure'); };+ spawnCalls.push({ bin, args, options, child });+ return child;+ });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ const job = await codex.generateImage({ prompt: 'synthetic' });+ const { child } = spawnCalls[0];+ child.pid = 4242;++ expect(child.kill).toHaveBeenCalledWith('SIGTERM');+ expect(failed).not.toHaveBeenCalled();+ expect(codex.getActiveJob()).toMatchObject({ generationId: job.jobId });++ // A live-child error is not exit; the close that follows settles exactly once.+ child.emit('error', new Error('signal delivery failed'));+ await flush();+ expect(failed).not.toHaveBeenCalled();+ child.exitCode = 0;+ child.emit('close', 0, null);+ child.emit('close', 0, null);+ await vi.waitFor(() => expect(failed).toHaveBeenCalledTimes(1));+ await flush();+ expect(failed).toHaveBeenCalledTimes(1);+ expect(failed.mock.calls[0][0].error).toMatch(/launch setup failed: synthetic stream setup failure/);+ expect(codex.getActiveJob()).toBeNull();+ expect(codex.cancel(job.jobId)).toBe(false);+ });+});diff --git a/server/services/imageGen/grok.js b/server/services/imageGen/grok.jsindex b2d24a0de8..6cc57475ea 100644--- a/server/services/imageGen/grok.js+++ b/server/services/imageGen/grok.js@@ -247,7 +247,6 @@ export async function generateImage({// terminal path removes the whole dir.const scratchDir = join(tmpdir(), `portos-grok-${jobId}`);const stagingPath = join(scratchDir, 'output.png');- await mkdir(scratchDir, { recursive: true });// Caller dimensions win (mapped to the closest supported ratio); the saved// per-provider default applies when no dimensions were sent; otherwise the@@ -268,7 +267,6 @@ export async function generateImage({// the agent model is whatever the local `grok` install defaults to (see// server/lib/grok.js: PortOS does not pick a grok model).const baseArgs = ensureGrokHeadlessArgs([], null);- const { args, useStdin, cleanup: cleanupPromptFile } = prepareGrokPromptFile(baseArgs, fullPrompt);const meta = {id: jobId, prompt: prompt.trim(), negativePrompt: negativePrompt || '',@@ -278,6 +276,9 @@ export async function generateImage({...(effectiveRatio ? { aspectRatio: effectiveRatio } : {}),createdAt: new Date().toISOString(),};+ // The scratch dir is the last fallible step before the job exists, so a+ // refusal above leaves nothing behind; from here on runGrok owns it.+ await mkdir(scratchDir, { recursive: true });const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };jobs.set(jobId, job);@@ -288,12 +289,11 @@ export async function generateImage({// generateImage returns a job descriptor synchronously; the actual grok// child runs out-of-band so the HTTP response can ship while the client- // attaches to the per-job SSE stream (mirrors codex.js/local.js).- runGrok(job, jobId, bin, args, {- useStdin, fullPrompt, cleanupPromptFile, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA, denoise,+ // attaches to the per-job SSE stream (mirrors codex.js/local.js). runGrok+ // settles every launch failure through the job's own failure finalizer.+ runGrok(job, jobId, bin, baseArgs, {+ fullPrompt, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA, denoise,toolName: grokImageTool(inputImages.paths.length > 0),- }).catch((err) => {- console.error(`❌ grok run failed [${jobId.slice(0, 8)}]: ${err?.message}`);});return {@@ -305,49 +305,59 @@ export async function generateImage({};}-async function runGrok(job, jobId, bin, args, {- useStdin, fullPrompt, cleanupPromptFile, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA = false, denoise = false,+// Launch is synchronous so every failure has one owner: a refusal before a+// child exists (prompt-file preparation, shim resolution, or Node rejecting an+// invalid argv/bin synchronously) releases the scratch dir and prompt file and+// settles the job here; a setup failure after spawn terminates the child and+// leaves the close handler to release resources once it has physically exited.+function runGrok(job, jobId, bin, baseArgs, {+ fullPrompt, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA = false, denoise = false,toolName = grokImageTool(false),}) {- // A path-shaped grokPath (contains a separator) must resolve against the- // PortOS working directory NOW — the child spawns with cwd set to the- // scratch dir, where a relative "./node_modules/.bin/grok" would ENOENT.- // Bare names stay bare for PATH lookup. prepareCliSpawn then resolves the- // Windows .cmd shim of an npm-installed grok and wraps it for a safe- // shell:false spawn — a no-op on POSIX.- const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;- const { command: spawnBin, args: spawnArgs } = prepareCliSpawn(resolvedBin, args);- // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). grok reads- // process.cwd(), so this is defensive rather than a live fix; it keeps every- // scratch-dir spawn telling the child one consistent story about where it is.- const proc = spawn(spawnBin, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: [useStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });- activeProcs.set(jobId, proc);- const removeScratch = () => maintenance.continueSettlement(- () => rmGuarded(scratchDir, { recursive: true, force: true }),- ).catch((err) => {+ let cleanupPromptFile = () => {};+ // Every terminal path releases both job-owned temp resources. A failed+ // removal keeps the maintenance settlement unsettled rather than claiming+ // the resources were released; both removals are idempotent.+ const removeScratch = () => maintenance.continueSettlement(async () => {+ let promptError = null;+ try { cleanupPromptFile({ throwOnError: true }); } catch (err) { promptError = err; }+ await rmGuarded(scratchDir, { recursive: true, force: true });+ if (promptError) throw promptError;+ }).catch((err) => {maintenance.markCurrentUnsettled();console.error(`❌ grok scratch cleanup failed: ${err.message}`);});- if (useStdin) {- // POSIX: grok reads the prompt via --prompt-file /dev/stdin. EPIPE fires- // when the child dies before consuming stdin — the close handler reports- // the real failure, so just swallow the write error.- proc.stdin.on('error', () => {});- proc.stdin.write(fullPrompt);- proc.stdin.end();+ let proc;+ let useStdin;+ try {+ const prepared = prepareGrokPromptFile(baseArgs, fullPrompt);+ cleanupPromptFile = prepared.cleanup;+ useStdin = prepared.useStdin;+ // A path-shaped grokPath (contains a separator) must resolve against the+ // PortOS working directory NOW — the child spawns with cwd set to the+ // scratch dir, where a relative "./node_modules/.bin/grok" would ENOENT.+ // Bare names stay bare for PATH lookup. prepareCliSpawn then resolves the+ // Windows .cmd shim of an npm-installed grok and wraps it for a safe+ // shell:false spawn — a no-op on POSIX.+ const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;+ const { command: spawnBin, args: spawnArgs } = prepareCliSpawn(resolvedBin, prepared.args);+ // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). grok reads+ // process.cwd(), so this is defensive rather than a live fix; it keeps every+ // scratch-dir spawn telling the child one consistent story about where it is.+ proc = spawn(spawnBin, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: [useStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });+ } catch (err) {+ removeScratch();+ return finalizeJobFailure(job, jobId, null, `Failed to spawn ${bin}: ${err?.message || err}`);}+ activeProcs.set(jobId, proc);let stdoutTail = '';const STDOUT_TAIL_BYTES = 8 * 1024;let stderrTail = '';const STDERR_TAIL_BYTES = 32 * 1024;- const timeoutTimer = setTimeout(() => {- if (activeProcs.get(jobId) === proc) {- console.log(`⏱️ grok timed out after ${GROK_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);- sigtermWithEscalation(jobId, proc);- }- }, GROK_TIMEOUT_MS);+ let timeoutTimer = null;+ let setupError = null;let processError;proc.on('error', (err) => {@@ -355,30 +365,21 @@ async function runGrok(job, jobId, bin, args, {// A failed kill/send on a live child is not physical exit.if (proc.pid) return;clearTimeout(timeoutTimer);- cleanupPromptFile();removeScratch();finalizeJobFailure(job, jobId, proc, `Failed to spawn ${bin}: ${err.message}`);});- proc.stdout.on('data', (chunk) => {- stdoutTail += chunk.toString();- if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);- broadcastSse(job, { type: 'status', message: 'Running…' });- });-- proc.stderr.on('data', (chunk) => {- stderrTail += chunk.toString();- if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);- });-proc.on('close', async (code, signal) => {await withBackupAssetPublication(async () => {clearTimeout(timeoutTimer);- cleanupPromptFile();// EventEmitter doesn't await async listeners — without this try/catch,// a throw from the harvest/copy would surface as an unhandled rejection// and the job would be stuck in 'running' forever with no SSE error.try {+ if (setupError) {+ removeScratch();+ return finalizeJobFailure(job, jobId, proc, `Grok launch setup failed: ${setupError?.message || setupError}`);+ }if (code !== 0 || processError) {const reason = processError?.message || (signal ? `Killed by signal ${signal}` : `Exit code ${code}`);const tail = stderrTail.trim().split('\n').slice(-6).join('\n');@@ -452,11 +453,46 @@ async function runGrok(job, jobId, bin, args, {// Admission can reject before the publication callback is entered.// EventEmitter does not await this listener, so settle the job here too.clearTimeout(timeoutTimer);- cleanupPromptFile();removeScratch();finalizeJobFailure(job, jobId, proc, `Grok publication admission failed: ${err?.message || err}`);});});++ // Post-spawn setup. The error/close owners above are already installed, so a+ // throw here must not settle the job or release its resources under a live+ // child: record it, terminate the child, and let close settle after exit.+ try {+ if (useStdin) {+ // POSIX: grok reads the prompt via --prompt-file /dev/stdin. EPIPE fires+ // when the child dies before consuming stdin — the close handler reports+ // the real failure, so just swallow the write error.+ proc.stdin.on('error', () => {});+ proc.stdin.write(fullPrompt);+ proc.stdin.end();+ }++ timeoutTimer = setTimeout(() => {+ if (activeProcs.get(jobId) === proc) {+ console.log(`⏱️ grok timed out after ${GROK_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);+ sigtermWithEscalation(jobId, proc);+ }+ }, GROK_TIMEOUT_MS);++ proc.stdout.on('data', (chunk) => {+ stdoutTail += chunk.toString();+ if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);+ broadcastSse(job, { type: 'status', message: 'Running…' });+ });++ proc.stderr.on('data', (chunk) => {+ stderrTail += chunk.toString();+ if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);+ });+ } catch (err) {+ setupError = err;+ clearTimeout(timeoutTimer);+ sigtermWithEscalation(jobId, proc);+ }}const finalizeJobFailure = createJobFailureFinalizer({diff --git a/server/services/imageGen/grok.test.js b/server/services/imageGen/grok.test.jsindex 97499347fc..e9ce67738e 100644--- a/server/services/imageGen/grok.test.js+++ b/server/services/imageGen/grok.test.js@@ -66,17 +66,31 @@ vi.mock('../../lib/childProcess.js', async (importOriginal) => {// a predictable place we can read back (mirrors codex.test.js).const TEST_ROOT = join(tmpdir(), `portos-grok-test-${process.pid}-${Date.now()}`);const FAKE_IMAGES_DIR = join(TEST_ROOT, 'data-images');+const FAKE_DATA_DIR = join(TEST_ROOT, 'data');vi.mock('../../lib/fileUtils.js', async () => {const actual = await vi.importActual('../../lib/fileUtils.js');actual.PATHS.images = FAKE_IMAGES_DIR;return {...actual,ensureDir: vi.fn(async (dir) => mkdir(dir, { recursive: true })),+ rmGuarded: vi.fn(actual.rmGuarded),};});+// A test-rooted maintenance journal, so cleanup-failure ownership is observable.+const fixture = vi.hoisted(() => ({ admission: null }));+vi.mock('../../lib/maintenanceAdmission.js', async importOriginal => ({+ ...await importOriginal(),+ maintenance: new Proxy({}, { get: (_target, property) => fixture.admission[property] }),+}));const grok = await import('./grok.js');const { imageGenEvents } = await import('../imageGenEvents.js');+const { spawn } = await import('../../lib/childProcess.js');+const { spawn: realSpawn } = await vi.importActual('../../lib/childProcess.js');+const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');+const { createMaintenanceAdmission } = await import('../../lib/maintenanceAdmission.js');+const { rmGuarded } = await import('../../lib/fileUtils.js');+const realFiles = await vi.importActual('../../lib/fileUtils.js');const flush = () => new Promise((r) => setImmediate(r));const stagingPathFor = (jobId) => join(tmpdir(), `portos-grok-${jobId}`, 'output.png');@@ -94,7 +108,9 @@ beforeEach(async () => {imageGenEvents.removeAllListeners();grok._internals.setHarvestTimeoutForTests(10);await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});- await mkdir(TEST_ROOT, { recursive: true });+ await mkdir(FAKE_DATA_DIR, { recursive: true });+ fixture.admission = createMaintenanceAdmission(FAKE_DATA_DIR);+ rmGuarded.mockReset().mockImplementation(realFiles.rmGuarded);});afterEach(async () => {@@ -530,3 +546,101 @@ it('settles admission rejection from the actual close event and clears the activresponse.req.emit('close');} finally { admissionFault.error = null; }});++// #10749 — a launch that throws must settle through the job's own finalizer and+// release the job-owned scratch dir / prompt file, not leave a phantom+// 'running' job whose SSE stream never terminates and whose temp files leak.+describe('grok provider — launch refusal settlement', () => {+ const scratchDirFor = (jobId) => join(tmpdir(), `portos-grok-${jobId}`);+ const sseClient = () => ({ writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: new EventEmitter() });+ const promptFileIn = (args = []) => {+ const path = args[args.indexOf('--prompt-file') + 1];+ return path && path !== '/dev/stdin' ? path : null;+ };++ it.each([+ // Node validates argv synchronously and throws before any child exists.+ ['a real Node argument refusal', { grokPath: 'grok\0probe' }, (seen) => spawn.mockImplementationOnce((bin, args, options) => {+ seen.args = args;+ return realSpawn(bin, args, options);+ }), /Failed to spawn/],+ ['a synthetic launch refusal', {}, (seen) => spawn.mockImplementationOnce((bin, args) => {+ seen.args = args;+ throw new Error('synthetic launch refusal');+ }), /synthetic launch refusal/],+ ])('settles %s once and removes its scratch dir and prompt file', async (_label, params, arrange, reason) => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ const seen = {};+ arrange(seen);+ const job = await grok.generateImage({ prompt: 'synthetic', ...params });+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));++ expect(spawnCalls).toHaveLength(0);+ expect(failed).toHaveBeenCalledTimes(1);+ expect(failed.mock.calls[0][0]).toMatchObject({ mode: 'grok', generationId: job.jobId, error: expect.stringMatching(reason) });+ // Windows stages the prompt in a temp file; it must not outlive the refusal.+ const promptFile = promptFileIn(seen.args);+ if (promptFile) expect(existsSync(promptFile)).toBe(false);+ expect(grok.getActiveJob()).toBeNull();+ expect(grok.cancel(job.jobId)).toBe(false);+ const client = sseClient();+ expect(grok.attachSseClient(job.jobId, client)).toBe(true);+ expect(JSON.parse(client.write.mock.calls[0][0].replace(/^data: /, ''))).toMatchObject({ type: 'error', error: expect.stringMatching(reason) });+ client.req.emit('close');++ vi.advanceTimersByTime(SSE_CLEANUP_DELAY_MS);+ expect(grok.attachSseClient(job.jobId, sseClient())).toBe(false);+ });++ it('retains an unsettled maintenance blocker when the refused launch cannot remove its scratch dir', async () => {+ const jobId = 'grok-launch-refusal-cleanup';+ rmGuarded.mockRejectedValueOnce(Object.assign(new Error('Cleanup denied'), { code: 'EACCES' }));+ spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ const permit = fixture.admission.admit('image', jobId);+ await permit.run(() => grok.generateImage({ jobId, prompt: 'synthetic' }));+ fixture.admission.begin({ reason: 'Drain renderer', owner: 'Operator' });+ await vi.waitFor(() => expect(rmGuarded).toHaveBeenCalledTimes(1));+ await flush();+ await permit.finish();+ expect(failed).toHaveBeenCalledTimes(1);+ expect(fixture.admission.status()).toMatchObject({+ state: 'draining', blockers: expect.arrayContaining([expect.objectContaining({ resource: jobId, unsettled: true })]),+ });+ await realFiles.rmGuarded(scratchDirFor(jobId), { recursive: true, force: true });+ });++ it('terminates a live child whose setup fails and releases resources only after it physically closes', async () => {+ spawn.mockImplementationOnce((bin, args) => {+ const child = makeFakeChild();+ child.stdout.on = () => { throw new Error('synthetic stream setup failure'); };+ spawnCalls.push({ bin, args, child, promptFromFile: null });+ return child;+ });+ const failed = vi.fn();+ imageGenEvents.on('failed', failed);+ const job = await grok.generateImage({ prompt: 'synthetic' });+ const { child } = spawnCalls[0];+ child.pid = 4242;++ expect(child.kill).toHaveBeenCalledWith('SIGTERM');+ child.emit('error', new Error('signal delivery failed'));+ await flush();+ expect(failed).not.toHaveBeenCalled();+ expect(existsSync(scratchDirFor(job.jobId))).toBe(true);+ expect(grok.getActiveJob()).toMatchObject({ generationId: job.jobId });++ child.exitCode = 0;+ child.emit('close', 0, null);+ child.emit('close', 0, null);+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));+ await flush();+ expect(failed).toHaveBeenCalledTimes(1);+ expect(failed.mock.calls[0][0].error).toMatch(/launch setup failed: synthetic stream setup failure/);+ expect(grok.getActiveJob()).toBeNull();+ expect(grok.cancel(job.jobId)).toBe(false);+ });+});