Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: settle image CLI jobs when launch preparation throws (#10749)

Adam Eivy16d69590ed40
diff --git a/server/services/imageGen/agy.js b/server/services/imageGen/agy.js
index 2b26707b1f..a977a60460 100644
--- a/server/services/imageGen/agy.js
+++ b/server/services/imageGen/agy.js
@@ -299,7 +299,6 @@ export async function generateImage({
const outputPath = join(PATHS.images, filename);
const scratchDir = join(tmpdir(), `portos-agy-${jobId}`);
const stagingPath = join(scratchDir, 'output.png');
- await mkdir(scratchDir, { recursive: true });
const fullPrompt = buildAgyPrompt({
prompt, negativePrompt, width, height, stagingPath, inputImages, initImageStrength,
@@ -327,6 +326,9 @@ export async function generateImage({
...(visualConditioning ? { visualConditioning } : {}),
createdAt: new Date().toISOString(),
};
+ // The scratch dir is the last fallible step before the job exists, so a
+ // refusal above leaves nothing behind; from here on runAgy owns it.
+ await mkdir(scratchDir, { recursive: true });
const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
jobs.set(jobId, job);
activeJobs.set(jobId, {
@@ -349,8 +351,6 @@ export async function generateImage({
meta,
cleanC2PA,
denoise,
- }).catch((err) => {
- console.error(`❌ agy run failed [${jobId.slice(0, 8)}]: ${err?.message}`);
});
return {
@@ -364,7 +364,12 @@ export async function generateImage({
};
}
-async function runAgy(job, jobId, bin, args, {
+// Launch is synchronous so every failure has one owner: a refusal before a
+// child exists (shim resolution, or Node rejecting an invalid argv/bin
+// synchronously) releases the scratch dir and settles the job here; a setup
+// failure after spawn terminates the child and leaves the close handler to
+// release the scratch dir once it has physically exited.
+function runAgy(job, jobId, bin, args, {
scratchDir,
stagingPath,
outputPath,
@@ -373,35 +378,31 @@ async function runAgy(job, jobId, bin, args, {
cleanC2PA,
denoise,
}) {
- const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;
- const { command, args: spawnArgs } = prepareCliSpawn(resolvedBin, args);
- // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). agy reads
- // process.cwd(), so this is defensive rather than a live fix; it keeps every
- // scratch-dir spawn telling the child one consistent story about where it is.
- const proc = spawn(command, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: ['ignore', 'pipe', 'pipe'] });
- activeProcs.set(jobId, proc);
+ // A failed removal keeps the maintenance settlement unsettled rather than
+ // claiming the scratch dir was released; removal is idempotent.
const removeScratch = () => maintenance.continueSettlement(
() => rmGuarded(scratchDir, { recursive: true, force: true }),
).catch((err) => {
maintenance.markCurrentUnsettled();
console.error(`❌ agy scratch cleanup failed: ${err.message}`);
});
+ let proc;
+ try {
+ const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;
+ const { command, args: spawnArgs } = prepareCliSpawn(resolvedBin, args);
+ // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). agy reads
+ // process.cwd(), so this is defensive rather than a live fix; it keeps every
+ // scratch-dir spawn telling the child one consistent story about where it is.
+ proc = spawn(command, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: ['ignore', 'pipe', 'pipe'] });
+ } catch (err) {
+ removeScratch();
+ return finalizeJobFailure(job, jobId, null, `Failed to spawn ${bin}: ${err?.message || err}`);
+ }
+ activeProcs.set(jobId, proc);
let stdoutTail = '';
let stderrTail = '';
- const timeoutTimer = setTimeout(() => {
- if (activeProcs.get(jobId) === proc) {
- console.log(`⏱️ agy timed out after ${AGY_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);
- terminate(jobId, proc);
- }
- }, AGY_TIMEOUT_MS);
-
- proc.stdout.on('data', (chunk) => {
- stdoutTail = `${stdoutTail}${chunk}`.slice(-8192);
- broadcastSse(job, { type: 'status', message: 'Running…' });
- });
- proc.stderr.on('data', (chunk) => {
- stderrTail = `${stderrTail}${chunk}`.slice(-32768);
- });
+ let timeoutTimer = null;
+ let setupError = null;
let processError;
proc.on('error', (err) => {
processError = err;
@@ -415,6 +416,10 @@ async function runAgy(job, jobId, bin, args, {
await withBackupAssetPublication(async () => {
clearTimeout(timeoutTimer);
try {
+ if (setupError) {
+ removeScratch();
+ return finalizeJobFailure(job, jobId, proc, `Agy launch setup failed: ${setupError?.message || setupError}`);
+ }
if (code !== 0 || processError) {
removeScratch();
const reason = processError?.message || (signal ? `Killed by signal ${signal}` : `Exit code ${code}`);
@@ -488,6 +493,30 @@ async function runAgy(job, jobId, bin, args, {
finalizeJobFailure(job, jobId, proc, `Agy publication admission failed: ${err?.message || err}`);
});
});
+
+ // Post-spawn setup. The error/close owners above are already installed, so a
+ // throw here must not settle the job or release its scratch dir under a live
+ // child: record it, terminate the child, and let close settle after exit.
+ try {
+ timeoutTimer = setTimeout(() => {
+ if (activeProcs.get(jobId) === proc) {
+ console.log(`⏱️ agy timed out after ${AGY_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);
+ terminate(jobId, proc);
+ }
+ }, AGY_TIMEOUT_MS);
+
+ proc.stdout.on('data', (chunk) => {
+ stdoutTail = `${stdoutTail}${chunk}`.slice(-8192);
+ broadcastSse(job, { type: 'status', message: 'Running…' });
+ });
+ proc.stderr.on('data', (chunk) => {
+ stderrTail = `${stderrTail}${chunk}`.slice(-32768);
+ });
+ } catch (err) {
+ setupError = err;
+ clearTimeout(timeoutTimer);
+ terminate(jobId, proc);
+ }
}
const finalizeJobFailure = createJobFailureFinalizer({
diff --git a/server/services/imageGen/agy.test.js b/server/services/imageGen/agy.test.js
index e9e888f3fc..5c14631536 100644
--- a/server/services/imageGen/agy.test.js
+++ b/server/services/imageGen/agy.test.js
@@ -51,18 +51,32 @@ vi.mock('../providers.js', () => ({
const TEST_ROOT = join(tmpdir(), `portos-agy-test-${process.pid}-${Date.now()}`);
const FAKE_IMAGES_DIR = join(TEST_ROOT, 'data-images');
+const FAKE_DATA_DIR = join(TEST_ROOT, 'data');
vi.mock('../../lib/fileUtils.js', async () => {
const actual = await vi.importActual('../../lib/fileUtils.js');
actual.PATHS.images = FAKE_IMAGES_DIR;
return {
...actual,
ensureDir: vi.fn(async (dir) => mkdir(dir, { recursive: true })),
+ rmGuarded: vi.fn(actual.rmGuarded),
};
});
+// A test-rooted maintenance journal, so cleanup-failure ownership is observable.
+const fixture = vi.hoisted(() => ({ admission: null }));
+vi.mock('../../lib/maintenanceAdmission.js', async importOriginal => ({
+ ...await importOriginal(),
+ maintenance: new Proxy({}, { get: (_target, property) => fixture.admission[property] }),
+}));
const agy = await import('./agy.js');
const { AGY_IMAGEGEN_DEFAULT_MODEL } = await import('../../lib/imageGenCapabilities.js');
const { imageGenEvents } = await import('../imageGenEvents.js');
+const { spawn } = await import('../../lib/childProcess.js');
+const { spawn: realSpawn } = await vi.importActual('../../lib/childProcess.js');
+const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');
+const { createMaintenanceAdmission } = await import('../../lib/maintenanceAdmission.js');
+const { rmGuarded } = await import('../../lib/fileUtils.js');
+const realFiles = await vi.importActual('../../lib/fileUtils.js');
const flush = () => new Promise((resolve) => setImmediate(resolve));
const stagingPathFor = (jobId) => join(tmpdir(), `portos-agy-${jobId}`, 'output.png');
@@ -78,10 +92,13 @@ beforeEach(async () => {
imageGenEvents.removeAllListeners();
agy._internals.setHarvestTimeoutForTests(10);
await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});
- await mkdir(TEST_ROOT, { recursive: true });
+ await mkdir(FAKE_DATA_DIR, { recursive: true });
+ fixture.admission = createMaintenanceAdmission(FAKE_DATA_DIR);
+ rmGuarded.mockReset().mockImplementation(realFiles.rmGuarded);
});
afterEach(async () => {
+ vi.useRealTimers();
agy._internals.setHarvestTimeoutForTests();
await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});
});
@@ -410,3 +427,87 @@ it('settles admission rejection from the actual close event and clears the activ
response.req.emit('close');
} finally { admissionFault.error = null; }
});
+
+// #10749 — a launch that throws must settle through the job's own finalizer and
+// release the job-owned scratch dir, not leave a phantom 'running' job whose SSE
+// stream never terminates and whose scratch dir leaks.
+describe('agy image provider — launch refusal settlement', () => {
+ const scratchDirFor = (jobId) => join(tmpdir(), `portos-agy-${jobId}`);
+ const sseClient = () => ({ writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: new EventEmitter() });
+
+ it.each([
+ // Node validates argv synchronously and throws before any child exists.
+ ['a real Node argument refusal', { agyPath: 'agy\0probe' }, () => spawn.mockImplementationOnce(realSpawn), /Failed to spawn/],
+ ['a synthetic launch refusal', {}, () => spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); }), /synthetic launch refusal/],
+ ])('settles %s once and removes its scratch dir', async (_label, params, arrange, reason) => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ arrange();
+ const job = await agy.generateImage({ prompt: 'synthetic', ...params });
+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));
+
+ expect(spawnCalls).toHaveLength(0);
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(failed.mock.calls[0][0]).toMatchObject({ mode: 'agy', generationId: job.jobId, error: expect.stringMatching(reason) });
+ expect(agy.getActiveJob()).toBeNull();
+ expect(agy.cancel(job.jobId)).toBe(false);
+ const client = sseClient();
+ expect(agy.attachSseClient(job.jobId, client)).toBe(true);
+ expect(JSON.parse(client.write.mock.calls[0][0].replace(/^data: /, ''))).toMatchObject({ type: 'error', error: expect.stringMatching(reason) });
+ client.req.emit('close');
+
+ vi.advanceTimersByTime(SSE_CLEANUP_DELAY_MS);
+ expect(agy.attachSseClient(job.jobId, sseClient())).toBe(false);
+ });
+
+ it('retains an unsettled maintenance blocker when the refused launch cannot remove its scratch dir', async () => {
+ const jobId = 'agy-launch-refusal-cleanup';
+ rmGuarded.mockRejectedValueOnce(Object.assign(new Error('Cleanup denied'), { code: 'EACCES' }));
+ spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ const permit = fixture.admission.admit('image', jobId);
+ await permit.run(() => agy.generateImage({ jobId, prompt: 'synthetic' }));
+ fixture.admission.begin({ reason: 'Drain renderer', owner: 'Operator' });
+ await vi.waitFor(() => expect(rmGuarded).toHaveBeenCalledTimes(1));
+ await flush();
+ await permit.finish();
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(fixture.admission.status()).toMatchObject({
+ state: 'draining', blockers: expect.arrayContaining([expect.objectContaining({ resource: jobId, unsettled: true })]),
+ });
+ await realFiles.rmGuarded(scratchDirFor(jobId), { recursive: true, force: true });
+ });
+
+ it('terminates a live child whose setup fails and releases its scratch dir only after it physically closes', async () => {
+ spawn.mockImplementationOnce((bin, args) => {
+ const child = makeFakeChild();
+ child.stdout.on = () => { throw new Error('synthetic stream setup failure'); };
+ spawnCalls.push({ bin, args, child });
+ return child;
+ });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ const job = await agy.generateImage({ prompt: 'synthetic' });
+ const { child } = spawnCalls[0];
+ child.pid = 4242;
+
+ expect(child.kill).toHaveBeenCalledWith('SIGTERM');
+ child.emit('error', new Error('signal delivery failed'));
+ await flush();
+ expect(failed).not.toHaveBeenCalled();
+ expect(existsSync(scratchDirFor(job.jobId))).toBe(true);
+ expect(agy.getActiveJob()).toMatchObject({ generationId: job.jobId });
+
+ child.exitCode = 0;
+ child.emit('close', 0, null);
+ child.emit('close', 0, null);
+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));
+ await flush();
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(failed.mock.calls[0][0].error).toMatch(/launch setup failed: synthetic stream setup failure/);
+ expect(agy.getActiveJob()).toBeNull();
+ expect(agy.cancel(job.jobId)).toBe(false);
+ });
+});
diff --git a/server/services/imageGen/codex.js b/server/services/imageGen/codex.js
index 80d0bf54be..8407dcc879 100644
--- a/server/services/imageGen/codex.js
+++ b/server/services/imageGen/codex.js
@@ -285,10 +285,9 @@ export async function generateImage({
// generateImage returns a job descriptor synchronously; the actual codex
// child runs out-of-band so the HTTP response can ship while the client
- // attaches to the per-job SSE stream (mirrors local.js).
- runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA, denoise }).catch((err) => {
- console.error(`❌ codex run failed [${jobId.slice(0, 8)}]: ${err?.message}`);
- });
+ // attaches to the per-job SSE stream (mirrors local.js). runCodex settles
+ // every launch failure through the job's own failure finalizer.
+ runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA, denoise });
return {
jobId, filename, path: `/data/images/${filename}`, generationId: jobId,
@@ -322,8 +321,17 @@ export const noImageReason = (stdoutTail = '') => buildNoImageReason(stdoutTail,
),
});
-async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA = false, denoise = false } = {}) {
- const proc = spawn(bin, args, { shell: false, stdio: ['ignore', 'pipe', 'pipe'] });
+// Launch is synchronous so every failure has one owner: a refusal before a
+// child exists (Node rejects an invalid argv/bin synchronously) settles the job
+// here, and a setup failure after spawn terminates the child and leaves the
+// close handler to settle once it has physically exited.
+function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA = false, denoise = false } = {}) {
+ let proc;
+ try {
+ proc = spawn(bin, args, { shell: false, stdio: ['ignore', 'pipe', 'pipe'] });
+ } catch (err) {
+ return finalizeJobFailure(job, jobId, null, `Failed to spawn ${bin}: ${err?.message || err}`);
+ }
activeProcs.set(jobId, proc);
let sessionId = null;
@@ -334,13 +342,8 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cle
// can surface the model's actual words instead of a generic "no image" guess.
let stdoutTail = '';
const STDOUT_TAIL_BYTES = 8 * 1024;
- let timeoutTimer = setTimeout(() => {
- if (activeProcs.get(jobId) === proc) {
- console.log(`⏱️ codex timed out after ${CODEX_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);
- proc.kill('SIGTERM');
- setTimeout(() => { if (proc.exitCode === null) proc.kill('SIGKILL'); }, 5000);
- }
- }, CODEX_TIMEOUT_MS);
+ let timeoutTimer = null;
+ let setupError = null;
// Banner is roughly 12 lines / ~500 bytes — keep a small rolling
// buffer so a session-id line that gets split across chunk boundaries
@@ -377,26 +380,6 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cle
finalizeJobFailure(job, jobId, proc, `Failed to spawn ${bin}: ${err.message}`);
});
- proc.stdout.on('data', (chunk) => {
- // Codex prints the `session id:` banner on stderr only — don't feed
- // stdout into bannerBuf. A stdout chunk arriving between two stderr
- // chunks of the banner can split the session-id line with unrelated
- // text and break the regex match.
- // We DO retain stdout here (the model's turn narration) so a no-image
- // finish can report why — a content decline, or a "generated" claim with
- // no file (tool unavailable / image-gen rate-limited on the account).
- stdoutTail += chunk.toString();
- if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);
- broadcastSse(job, { type: 'status', message: 'Running…' });
- });
-
- proc.stderr.on('data', (chunk) => {
- const text = chunk.toString();
- captureSession(text);
- stderrTail += text;
- if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);
- });
-
proc.on('close', async (code, signal) => {
await withBackupAssetPublication(async () => {
clearTimeout(timeoutTimer);
@@ -410,6 +393,9 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cle
// unhandled rejection (process-killing on Node ≥15) and the job would
// be stuck in 'running' forever with no SSE error to the client.
try {
+ if (setupError) {
+ return finalizeJobFailure(job, jobId, proc, `Codex launch setup failed: ${setupError?.message || setupError}`);
+ }
if (code !== 0 || processError) {
const reason = processError?.message || (signal ? `Killed by signal ${signal}` : `Exit code ${code}`);
const tail = stderrTail.trim().split('\n').slice(-6).join('\n');
@@ -474,6 +460,43 @@ async function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cle
finalizeJobFailure(job, jobId, proc, `Codex publication admission failed: ${err?.message || err}`);
});
});
+
+ // Post-spawn setup. The error/close owners above are already installed, so a
+ // throw here must not settle the job under a live child: record it,
+ // terminate the child, and let close settle once it has physically exited.
+ try {
+ timeoutTimer = setTimeout(() => {
+ if (activeProcs.get(jobId) === proc) {
+ console.log(`⏱️ codex timed out after ${CODEX_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);
+ proc.kill('SIGTERM');
+ setTimeout(() => { if (proc.exitCode === null) proc.kill('SIGKILL'); }, 5000);
+ }
+ }, CODEX_TIMEOUT_MS);
+
+ proc.stdout.on('data', (chunk) => {
+ // Codex prints the `session id:` banner on stderr only — don't feed
+ // stdout into bannerBuf. A stdout chunk arriving between two stderr
+ // chunks of the banner can split the session-id line with unrelated
+ // text and break the regex match.
+ // We DO retain stdout here (the model's turn narration) so a no-image
+ // finish can report why — a content decline, or a "generated" claim with
+ // no file (tool unavailable / image-gen rate-limited on the account).
+ stdoutTail += chunk.toString();
+ if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);
+ broadcastSse(job, { type: 'status', message: 'Running…' });
+ });
+
+ proc.stderr.on('data', (chunk) => {
+ const text = chunk.toString();
+ captureSession(text);
+ stderrTail += text;
+ if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);
+ });
+ } catch (err) {
+ setupError = err;
+ clearTimeout(timeoutTimer);
+ sigtermWithEscalation(jobId, proc);
+ }
}
const finalizeJobFailure = createJobFailureFinalizer({
diff --git a/server/services/imageGen/codex.test.js b/server/services/imageGen/codex.test.js
index 1c0ade4cdc..c0474fc1af 100644
--- a/server/services/imageGen/codex.test.js
+++ b/server/services/imageGen/codex.test.js
@@ -71,6 +71,9 @@ vi.mock('../../lib/fileUtils.js', async () => {
const { acquireBackupSnapshotCut } = await import('../../lib/backupSnapshotBoundary.js');
const codex = await import('./codex.js');
const { imageGenEvents } = await import('../imageGenEvents.js');
+const { spawn } = await import('../../lib/childProcess.js');
+const { spawn: realSpawn } = await vi.importActual('../../lib/childProcess.js');
+const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');
const flush = () => new Promise((r) => setImmediate(r));
@@ -764,3 +767,67 @@ it('settles admission rejection from the actual close event and clears the activ
response.req.emit('close');
} finally { admissionFault.error = null; }
});
+
+// #10749 — a launch that throws must settle through the job's own finalizer,
+// not leave a phantom 'running' job, a never-terminating SSE stream and a
+// cancel that cannot find a process.
+describe('codex provider — launch refusal settlement', () => {
+ const sseClient = () => ({ writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: new EventEmitter() });
+
+ it.each([
+ // Node validates argv synchronously and throws before any child exists.
+ ['a real Node argument refusal', { codexPath: 'codex\0probe' }, () => spawn.mockImplementationOnce(realSpawn), /without null bytes/],
+ ['a synthetic launch refusal', {}, () => spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); }), /synthetic launch refusal/],
+ ])('settles %s exactly once with a replayable terminal frame', async (_label, params, arrange, reason) => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ arrange();
+ const job = await codex.generateImage({ prompt: 'synthetic', ...params });
+
+ expect(spawnCalls).toHaveLength(0);
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(failed.mock.calls[0][0]).toMatchObject({ mode: 'codex', generationId: job.jobId, error: expect.stringMatching(reason) });
+ expect(codex.getActiveJob()).toBeNull();
+ expect(codex.cancel(job.jobId)).toBe(false);
+ const client = sseClient();
+ expect(codex.attachSseClient(job.jobId, client)).toBe(true);
+ expect(JSON.parse(client.write.mock.calls[0][0].replace(/^data: /, ''))).toMatchObject({ type: 'error', error: expect.stringMatching(reason) });
+ client.req.emit('close');
+
+ vi.advanceTimersByTime(SSE_CLEANUP_DELAY_MS);
+ expect(codex.attachSseClient(job.jobId, sseClient())).toBe(false);
+ });
+
+ it('terminates a live child whose setup fails and settles once, only after it physically closes', async () => {
+ spawn.mockImplementationOnce((bin, args, options) => {
+ const child = makeFakeChild();
+ child.stdout.on = () => { throw new Error('synthetic stream setup failure'); };
+ spawnCalls.push({ bin, args, options, child });
+ return child;
+ });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ const job = await codex.generateImage({ prompt: 'synthetic' });
+ const { child } = spawnCalls[0];
+ child.pid = 4242;
+
+ expect(child.kill).toHaveBeenCalledWith('SIGTERM');
+ expect(failed).not.toHaveBeenCalled();
+ expect(codex.getActiveJob()).toMatchObject({ generationId: job.jobId });
+
+ // A live-child error is not exit; the close that follows settles exactly once.
+ child.emit('error', new Error('signal delivery failed'));
+ await flush();
+ expect(failed).not.toHaveBeenCalled();
+ child.exitCode = 0;
+ child.emit('close', 0, null);
+ child.emit('close', 0, null);
+ await vi.waitFor(() => expect(failed).toHaveBeenCalledTimes(1));
+ await flush();
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(failed.mock.calls[0][0].error).toMatch(/launch setup failed: synthetic stream setup failure/);
+ expect(codex.getActiveJob()).toBeNull();
+ expect(codex.cancel(job.jobId)).toBe(false);
+ });
+});
diff --git a/server/services/imageGen/grok.js b/server/services/imageGen/grok.js
index b2d24a0de8..6cc57475ea 100644
--- a/server/services/imageGen/grok.js
+++ b/server/services/imageGen/grok.js
@@ -247,7 +247,6 @@ export async function generateImage({
// terminal path removes the whole dir.
const scratchDir = join(tmpdir(), `portos-grok-${jobId}`);
const stagingPath = join(scratchDir, 'output.png');
- await mkdir(scratchDir, { recursive: true });
// Caller dimensions win (mapped to the closest supported ratio); the saved
// per-provider default applies when no dimensions were sent; otherwise the
@@ -268,7 +267,6 @@ export async function generateImage({
// the agent model is whatever the local `grok` install defaults to (see
// server/lib/grok.js: PortOS does not pick a grok model).
const baseArgs = ensureGrokHeadlessArgs([], null);
- const { args, useStdin, cleanup: cleanupPromptFile } = prepareGrokPromptFile(baseArgs, fullPrompt);
const meta = {
id: jobId, prompt: prompt.trim(), negativePrompt: negativePrompt || '',
@@ -278,6 +276,9 @@ export async function generateImage({
...(effectiveRatio ? { aspectRatio: effectiveRatio } : {}),
createdAt: new Date().toISOString(),
};
+ // The scratch dir is the last fallible step before the job exists, so a
+ // refusal above leaves nothing behind; from here on runGrok owns it.
+ await mkdir(scratchDir, { recursive: true });
const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
jobs.set(jobId, job);
@@ -288,12 +289,11 @@ export async function generateImage({
// generateImage returns a job descriptor synchronously; the actual grok
// child runs out-of-band so the HTTP response can ship while the client
- // attaches to the per-job SSE stream (mirrors codex.js/local.js).
- runGrok(job, jobId, bin, args, {
- useStdin, fullPrompt, cleanupPromptFile, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA, denoise,
+ // attaches to the per-job SSE stream (mirrors codex.js/local.js). runGrok
+ // settles every launch failure through the job's own failure finalizer.
+ runGrok(job, jobId, bin, baseArgs, {
+ fullPrompt, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA, denoise,
toolName: grokImageTool(inputImages.paths.length > 0),
- }).catch((err) => {
- console.error(`❌ grok run failed [${jobId.slice(0, 8)}]: ${err?.message}`);
});
return {
@@ -305,49 +305,59 @@ export async function generateImage({
};
}
-async function runGrok(job, jobId, bin, args, {
- useStdin, fullPrompt, cleanupPromptFile, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA = false, denoise = false,
+// Launch is synchronous so every failure has one owner: a refusal before a
+// child exists (prompt-file preparation, shim resolution, or Node rejecting an
+// invalid argv/bin synchronously) releases the scratch dir and prompt file and
+// settles the job here; a setup failure after spawn terminates the child and
+// leaves the close handler to release resources once it has physically exited.
+function runGrok(job, jobId, bin, baseArgs, {
+ fullPrompt, scratchDir, stagingPath, outputPath, filename, meta, cleanC2PA = false, denoise = false,
toolName = grokImageTool(false),
}) {
- // A path-shaped grokPath (contains a separator) must resolve against the
- // PortOS working directory NOW — the child spawns with cwd set to the
- // scratch dir, where a relative "./node_modules/.bin/grok" would ENOENT.
- // Bare names stay bare for PATH lookup. prepareCliSpawn then resolves the
- // Windows .cmd shim of an npm-installed grok and wraps it for a safe
- // shell:false spawn — a no-op on POSIX.
- const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;
- const { command: spawnBin, args: spawnArgs } = prepareCliSpawn(resolvedBin, args);
- // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). grok reads
- // process.cwd(), so this is defensive rather than a live fix; it keeps every
- // scratch-dir spawn telling the child one consistent story about where it is.
- const proc = spawn(spawnBin, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: [useStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
- activeProcs.set(jobId, proc);
- const removeScratch = () => maintenance.continueSettlement(
- () => rmGuarded(scratchDir, { recursive: true, force: true }),
- ).catch((err) => {
+ let cleanupPromptFile = () => {};
+ // Every terminal path releases both job-owned temp resources. A failed
+ // removal keeps the maintenance settlement unsettled rather than claiming
+ // the resources were released; both removals are idempotent.
+ const removeScratch = () => maintenance.continueSettlement(async () => {
+ let promptError = null;
+ try { cleanupPromptFile({ throwOnError: true }); } catch (err) { promptError = err; }
+ await rmGuarded(scratchDir, { recursive: true, force: true });
+ if (promptError) throw promptError;
+ }).catch((err) => {
maintenance.markCurrentUnsettled();
console.error(`❌ grok scratch cleanup failed: ${err.message}`);
});
- if (useStdin) {
- // POSIX: grok reads the prompt via --prompt-file /dev/stdin. EPIPE fires
- // when the child dies before consuming stdin — the close handler reports
- // the real failure, so just swallow the write error.
- proc.stdin.on('error', () => {});
- proc.stdin.write(fullPrompt);
- proc.stdin.end();
+ let proc;
+ let useStdin;
+ try {
+ const prepared = prepareGrokPromptFile(baseArgs, fullPrompt);
+ cleanupPromptFile = prepared.cleanup;
+ useStdin = prepared.useStdin;
+ // A path-shaped grokPath (contains a separator) must resolve against the
+ // PortOS working directory NOW — the child spawns with cwd set to the
+ // scratch dir, where a relative "./node_modules/.bin/grok" would ENOENT.
+ // Bare names stay bare for PATH lookup. prepareCliSpawn then resolves the
+ // Windows .cmd shim of an npm-installed grok and wraps it for a safe
+ // shell:false spawn — a no-op on POSIX.
+ const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;
+ const { command: spawnBin, args: spawnArgs } = prepareCliSpawn(resolvedBin, prepared.args);
+ // Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). grok reads
+ // process.cwd(), so this is defensive rather than a live fix; it keeps every
+ // scratch-dir spawn telling the child one consistent story about where it is.
+ proc = spawn(spawnBin, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: [useStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
+ } catch (err) {
+ removeScratch();
+ return finalizeJobFailure(job, jobId, null, `Failed to spawn ${bin}: ${err?.message || err}`);
}
+ activeProcs.set(jobId, proc);
let stdoutTail = '';
const STDOUT_TAIL_BYTES = 8 * 1024;
let stderrTail = '';
const STDERR_TAIL_BYTES = 32 * 1024;
- const timeoutTimer = setTimeout(() => {
- if (activeProcs.get(jobId) === proc) {
- console.log(`⏱️ grok timed out after ${GROK_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);
- sigtermWithEscalation(jobId, proc);
- }
- }, GROK_TIMEOUT_MS);
+ let timeoutTimer = null;
+ let setupError = null;
let processError;
proc.on('error', (err) => {
@@ -355,30 +365,21 @@ async function runGrok(job, jobId, bin, args, {
// A failed kill/send on a live child is not physical exit.
if (proc.pid) return;
clearTimeout(timeoutTimer);
- cleanupPromptFile();
removeScratch();
finalizeJobFailure(job, jobId, proc, `Failed to spawn ${bin}: ${err.message}`);
});
- proc.stdout.on('data', (chunk) => {
- stdoutTail += chunk.toString();
- if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);
- broadcastSse(job, { type: 'status', message: 'Running…' });
- });
-
- proc.stderr.on('data', (chunk) => {
- stderrTail += chunk.toString();
- if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);
- });
-
proc.on('close', async (code, signal) => {
await withBackupAssetPublication(async () => {
clearTimeout(timeoutTimer);
- cleanupPromptFile();
// EventEmitter doesn't await async listeners — without this try/catch,
// a throw from the harvest/copy would surface as an unhandled rejection
// and the job would be stuck in 'running' forever with no SSE error.
try {
+ if (setupError) {
+ removeScratch();
+ return finalizeJobFailure(job, jobId, proc, `Grok launch setup failed: ${setupError?.message || setupError}`);
+ }
if (code !== 0 || processError) {
const reason = processError?.message || (signal ? `Killed by signal ${signal}` : `Exit code ${code}`);
const tail = stderrTail.trim().split('\n').slice(-6).join('\n');
@@ -452,11 +453,46 @@ async function runGrok(job, jobId, bin, args, {
// Admission can reject before the publication callback is entered.
// EventEmitter does not await this listener, so settle the job here too.
clearTimeout(timeoutTimer);
- cleanupPromptFile();
removeScratch();
finalizeJobFailure(job, jobId, proc, `Grok publication admission failed: ${err?.message || err}`);
});
});
+
+ // Post-spawn setup. The error/close owners above are already installed, so a
+ // throw here must not settle the job or release its resources under a live
+ // child: record it, terminate the child, and let close settle after exit.
+ try {
+ if (useStdin) {
+ // POSIX: grok reads the prompt via --prompt-file /dev/stdin. EPIPE fires
+ // when the child dies before consuming stdin — the close handler reports
+ // the real failure, so just swallow the write error.
+ proc.stdin.on('error', () => {});
+ proc.stdin.write(fullPrompt);
+ proc.stdin.end();
+ }
+
+ timeoutTimer = setTimeout(() => {
+ if (activeProcs.get(jobId) === proc) {
+ console.log(`⏱️ grok timed out after ${GROK_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);
+ sigtermWithEscalation(jobId, proc);
+ }
+ }, GROK_TIMEOUT_MS);
+
+ proc.stdout.on('data', (chunk) => {
+ stdoutTail += chunk.toString();
+ if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);
+ broadcastSse(job, { type: 'status', message: 'Running…' });
+ });
+
+ proc.stderr.on('data', (chunk) => {
+ stderrTail += chunk.toString();
+ if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);
+ });
+ } catch (err) {
+ setupError = err;
+ clearTimeout(timeoutTimer);
+ sigtermWithEscalation(jobId, proc);
+ }
}
const finalizeJobFailure = createJobFailureFinalizer({
diff --git a/server/services/imageGen/grok.test.js b/server/services/imageGen/grok.test.js
index 97499347fc..e9ce67738e 100644
--- a/server/services/imageGen/grok.test.js
+++ b/server/services/imageGen/grok.test.js
@@ -66,17 +66,31 @@ vi.mock('../../lib/childProcess.js', async (importOriginal) => {
// a predictable place we can read back (mirrors codex.test.js).
const TEST_ROOT = join(tmpdir(), `portos-grok-test-${process.pid}-${Date.now()}`);
const FAKE_IMAGES_DIR = join(TEST_ROOT, 'data-images');
+const FAKE_DATA_DIR = join(TEST_ROOT, 'data');
vi.mock('../../lib/fileUtils.js', async () => {
const actual = await vi.importActual('../../lib/fileUtils.js');
actual.PATHS.images = FAKE_IMAGES_DIR;
return {
...actual,
ensureDir: vi.fn(async (dir) => mkdir(dir, { recursive: true })),
+ rmGuarded: vi.fn(actual.rmGuarded),
};
});
+// A test-rooted maintenance journal, so cleanup-failure ownership is observable.
+const fixture = vi.hoisted(() => ({ admission: null }));
+vi.mock('../../lib/maintenanceAdmission.js', async importOriginal => ({
+ ...await importOriginal(),
+ maintenance: new Proxy({}, { get: (_target, property) => fixture.admission[property] }),
+}));
const grok = await import('./grok.js');
const { imageGenEvents } = await import('../imageGenEvents.js');
+const { spawn } = await import('../../lib/childProcess.js');
+const { spawn: realSpawn } = await vi.importActual('../../lib/childProcess.js');
+const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');
+const { createMaintenanceAdmission } = await import('../../lib/maintenanceAdmission.js');
+const { rmGuarded } = await import('../../lib/fileUtils.js');
+const realFiles = await vi.importActual('../../lib/fileUtils.js');
const flush = () => new Promise((r) => setImmediate(r));
const stagingPathFor = (jobId) => join(tmpdir(), `portos-grok-${jobId}`, 'output.png');
@@ -94,7 +108,9 @@ beforeEach(async () => {
imageGenEvents.removeAllListeners();
grok._internals.setHarvestTimeoutForTests(10);
await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});
- await mkdir(TEST_ROOT, { recursive: true });
+ await mkdir(FAKE_DATA_DIR, { recursive: true });
+ fixture.admission = createMaintenanceAdmission(FAKE_DATA_DIR);
+ rmGuarded.mockReset().mockImplementation(realFiles.rmGuarded);
});
afterEach(async () => {
@@ -530,3 +546,101 @@ it('settles admission rejection from the actual close event and clears the activ
response.req.emit('close');
} finally { admissionFault.error = null; }
});
+
+// #10749 — a launch that throws must settle through the job's own finalizer and
+// release the job-owned scratch dir / prompt file, not leave a phantom
+// 'running' job whose SSE stream never terminates and whose temp files leak.
+describe('grok provider — launch refusal settlement', () => {
+ const scratchDirFor = (jobId) => join(tmpdir(), `portos-grok-${jobId}`);
+ const sseClient = () => ({ writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: new EventEmitter() });
+ const promptFileIn = (args = []) => {
+ const path = args[args.indexOf('--prompt-file') + 1];
+ return path && path !== '/dev/stdin' ? path : null;
+ };
+
+ it.each([
+ // Node validates argv synchronously and throws before any child exists.
+ ['a real Node argument refusal', { grokPath: 'grok\0probe' }, (seen) => spawn.mockImplementationOnce((bin, args, options) => {
+ seen.args = args;
+ return realSpawn(bin, args, options);
+ }), /Failed to spawn/],
+ ['a synthetic launch refusal', {}, (seen) => spawn.mockImplementationOnce((bin, args) => {
+ seen.args = args;
+ throw new Error('synthetic launch refusal');
+ }), /synthetic launch refusal/],
+ ])('settles %s once and removes its scratch dir and prompt file', async (_label, params, arrange, reason) => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ const seen = {};
+ arrange(seen);
+ const job = await grok.generateImage({ prompt: 'synthetic', ...params });
+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));
+
+ expect(spawnCalls).toHaveLength(0);
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(failed.mock.calls[0][0]).toMatchObject({ mode: 'grok', generationId: job.jobId, error: expect.stringMatching(reason) });
+ // Windows stages the prompt in a temp file; it must not outlive the refusal.
+ const promptFile = promptFileIn(seen.args);
+ if (promptFile) expect(existsSync(promptFile)).toBe(false);
+ expect(grok.getActiveJob()).toBeNull();
+ expect(grok.cancel(job.jobId)).toBe(false);
+ const client = sseClient();
+ expect(grok.attachSseClient(job.jobId, client)).toBe(true);
+ expect(JSON.parse(client.write.mock.calls[0][0].replace(/^data: /, ''))).toMatchObject({ type: 'error', error: expect.stringMatching(reason) });
+ client.req.emit('close');
+
+ vi.advanceTimersByTime(SSE_CLEANUP_DELAY_MS);
+ expect(grok.attachSseClient(job.jobId, sseClient())).toBe(false);
+ });
+
+ it('retains an unsettled maintenance blocker when the refused launch cannot remove its scratch dir', async () => {
+ const jobId = 'grok-launch-refusal-cleanup';
+ rmGuarded.mockRejectedValueOnce(Object.assign(new Error('Cleanup denied'), { code: 'EACCES' }));
+ spawn.mockImplementationOnce(() => { throw new Error('synthetic launch refusal'); });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ const permit = fixture.admission.admit('image', jobId);
+ await permit.run(() => grok.generateImage({ jobId, prompt: 'synthetic' }));
+ fixture.admission.begin({ reason: 'Drain renderer', owner: 'Operator' });
+ await vi.waitFor(() => expect(rmGuarded).toHaveBeenCalledTimes(1));
+ await flush();
+ await permit.finish();
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(fixture.admission.status()).toMatchObject({
+ state: 'draining', blockers: expect.arrayContaining([expect.objectContaining({ resource: jobId, unsettled: true })]),
+ });
+ await realFiles.rmGuarded(scratchDirFor(jobId), { recursive: true, force: true });
+ });
+
+ it('terminates a live child whose setup fails and releases resources only after it physically closes', async () => {
+ spawn.mockImplementationOnce((bin, args) => {
+ const child = makeFakeChild();
+ child.stdout.on = () => { throw new Error('synthetic stream setup failure'); };
+ spawnCalls.push({ bin, args, child, promptFromFile: null });
+ return child;
+ });
+ const failed = vi.fn();
+ imageGenEvents.on('failed', failed);
+ const job = await grok.generateImage({ prompt: 'synthetic' });
+ const { child } = spawnCalls[0];
+ child.pid = 4242;
+
+ expect(child.kill).toHaveBeenCalledWith('SIGTERM');
+ child.emit('error', new Error('signal delivery failed'));
+ await flush();
+ expect(failed).not.toHaveBeenCalled();
+ expect(existsSync(scratchDirFor(job.jobId))).toBe(true);
+ expect(grok.getActiveJob()).toMatchObject({ generationId: job.jobId });
+
+ child.exitCode = 0;
+ child.emit('close', 0, null);
+ child.emit('close', 0, null);
+ await vi.waitFor(() => expect(existsSync(scratchDirFor(job.jobId))).toBe(false));
+ await flush();
+ expect(failed).toHaveBeenCalledTimes(1);
+ expect(failed.mock.calls[0][0].error).toMatch(/launch setup failed: synthetic stream setup failure/);
+ expect(grok.getActiveJob()).toBeNull();
+ expect(grok.cancel(job.jobId)).toBe(false);
+ });
+});