Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: drain canceled production renders and expose bounded timeout phases (#13278)

[._.]/ Adam Eivy36aec939292b
diff --git a/server/services/codeAnimation/stages.js b/server/services/codeAnimation/stages.js
index fd88b9a32f..dfaa57ea36 100644
--- a/server/services/codeAnimation/stages.js
+++ b/server/services/codeAnimation/stages.js
@@ -11,6 +11,7 @@
* status that keeps accepted artifacts. Nothing runs at boot and a restart
* marks a stranded run interrupted without calling a provider.
*/
+import { channel } from 'node:diagnostics_channel';
import { randomUUID } from 'crypto';
import { join } from 'path';
import { createCodeAnimationPackage } from '../../lib/codeAnimationPackage.js';
@@ -39,6 +40,8 @@ const RESUMABLE = new Set(['interrupted', 'canceled', 'exhausted', 'failed']);
const BUDGET_CODE = 'CODE_ANIMATION_BUDGET_EXHAUSTED';
const CANCEL_CODE = 'CODE_ANIMATION_RUN_CANCELED';
+const lifecycle = channel('portos.code-animation-stage.lifecycle');
+
const activeRuns = trackedRenderMap('code-animation', { noun: 'Code Animation production run', link: () => '/code-animation/production' });
export const activeStageRunIds = () => [...activeRuns.keys()];
@@ -108,7 +111,10 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {
checkBudget(ctx);
const entry = { key, rendererBinding: ctx.state.renderer?.binding ?? null, stageRunId: randomUUID(), revisionId: revision.id, sourceHash: revision.sourceHash, packageHash: revision.packageHash, status: 'running', startedAt: iso() };
ctx.state.stages.push(entry);
+ const report = phase => lifecycle.publish({ runId: ctx.runId, stage: key, phase, elapsedMs: Date.now() - ctx.startedAt });
+ report('persist-start');
await persist(ctx);
+ report('execute');
const { ms, dimension } = remainingMs(ctx);
const deadline = new AbortController();
const timer = setTimeout(() => deadline.abort(exhausted(dimension)), Math.max(1, Math.ceil(ms)));
@@ -130,9 +136,11 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {
throw error;
} finally {
clearTimeout(timer);
+ report('persist-end');
if (renders) ctx.state.spent.renderMs += Date.now() - began;
await persist(ctx).catch(error => console.error(`❌ Code Animation stage ${key} status could not be saved: ${error.message}`));
}
+ report('settled');
return entry;
}
diff --git a/server/services/codeAnimation/stages.realBrowser.db.test.js b/server/services/codeAnimation/stages.realBrowser.db.test.js
index 5158808386..babcd0c342 100644
--- a/server/services/codeAnimation/stages.realBrowser.db.test.js
+++ b/server/services/codeAnimation/stages.realBrowser.db.test.js
@@ -28,7 +28,7 @@ const { checkHealth, ensureSchema, query, close } = await import('../../lib/db.j
const { requireDbOrSkip } = await import('../../lib/dbTestGate.js');
const { createCodeAnimationPackage } = await import('../../lib/codeAnimationPackage.js');
const { createProductionProject, getProductionHistory, importProductionPackage } = await import('./projects.js');
-const { startProductionStageRun } = await import('./stages.js');
+const { cancelProductionStageRun, startProductionStageRun } = await import('./stages.js');
const { renderComposition } = await import('../htmlComposition/index.js');
const { PATHS } = await import('../../lib/fileUtils.js');
@@ -66,10 +66,40 @@ let browser;
let artifacts;
let projectId;
let startup;
+// Keep only bounded phase/counter evidence; never stringify diagnostic payloads,
+// which may later acquire paths or other private context.
+const runs = [];
+let stageProgress;
+let renderProgress;
+let diagnosticTimer;
+const stageChannel = channel('portos.code-animation-stage.lifecycle');
+const renderChannel = channel('portos.composition-render.lifecycle');
+const observeStage = ({ stage, phase, elapsedMs }) => { stageProgress = { stage, phase, elapsedMs }; };
+const observeRender = ({ phase, elapsedMs, encoderPid, frame, frames }) => {
+ renderProgress = { phase, elapsedMs, encoderPid, frame, frames };
+};
+const reportPending = () => {
+ const pending = runs.find(run => !run.settled);
+ if (!pending) return;
+ const stage = stageProgress;
+ const render = renderProgress;
+ console.log(`🔬 Production fixture run ${runs.indexOf(pending) + 1} pending: stage=${stage?.stage ?? 'starting'}/${stage?.phase ?? 'starting'} stageAtMs=${stage?.elapsedMs ?? 0} render=${render?.phase ?? 'none'} renderAtMs=${render?.elapsedMs ?? 0} browserPid=${proc?.pid ?? 'none'} browserExited=${proc?.exitCode != null || proc?.signalCode != null} encoderPid=${render?.encoderPid ?? 'none'} frame=${render?.frame ?? 'none'}/${render?.frames ?? 'none'}`);
+};
+async function startTrackedRun(input, deps) {
+ stageProgress = null;
+ renderProgress = null;
+ const run = await startProductionStageRun(projectId, input, deps);
+ runs.push(run);
+ run.done.then(() => { run.settled = true; });
+ return run;
+}
const detachedChromeProcessGroupId = (child = proc) => process.platform === 'win32' ? undefined : child?.pid;
describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages with real Chrome and ffmpeg', () => {
beforeAll(async () => {
+ stageChannel.subscribe(observeStage);
+ renderChannel.subscribe(observeRender);
+ diagnosticTimer = setInterval(reportPending, 15000);
const root = lazyTempDataRoot('portos-code-animation-stages-');
try {
const ws = await _startTestChromeWithRetry({
@@ -96,9 +126,26 @@ describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages w
}, 90000);
afterAll(async () => {
- if (projectId) await query('DELETE FROM code_animation_projects WHERE id = $1', [projectId]);
- await _cleanupTestBrowser({ browser, proc, artifacts, cleanup: cleanupTempDataRoots, detachedProcessGroupId: detachedChromeProcessGroupId() });
- await close();
+ clearInterval(diagnosticTimer);
+ reportPending();
+ try {
+ for (const run of runs.filter(run => !run.settled)) {
+ try { cancelProductionStageRun(projectId, run.run.id); } catch (error) {
+ if (!['CODE_ANIMATION_RUN_COMMITTING', 'CODE_ANIMATION_RUN_NOT_ACTIVE'].includes(error.code)) throw error;
+ }
+ }
+ // A timed-out test leaves its async body alive. Drain its stage and
+ // encoder before removing the rows/files they may still be using.
+ await Promise.all(runs.map(run => run.done));
+ await _cleanupTestBrowser({ browser, proc, artifacts, detachedProcessGroupId: detachedChromeProcessGroupId(), cleanup: async () => {
+ if (projectId) await query('DELETE FROM code_animation_projects WHERE id = $1', [projectId]);
+ await cleanupTempDataRoots();
+ } });
+ await close();
+ } finally {
+ stageChannel.unsubscribe(observeStage);
+ renderChannel.unsubscribe(observeRender);
+ }
}, 25000);
it('turns a frozen original into measured findings, a repaired revision and a real MP4, with a real style frame and pilot', async () => {
@@ -107,8 +154,8 @@ describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages w
const repair = vi.fn(async ({ files, entryPath }) => ({
files: [{ path: entryPath, content: files.find(file => file.path === entryPath).content.replace('const MOVE = false;', 'const MOVE = true;') }],
}));
- const { done } = await startProductionStageRun(projectId, {}, {
- repair, render: ({ directory }) => renderComposition({ directory, jobId: 'stages-acceptance' }),
+ const { done } = await startTrackedRun({}, {
+ repair, render: ({ directory, signal }) => renderComposition({ directory, signal, jobId: 'stages-acceptance' }),
});
const status = await done;
const [{ data }] = (await getProductionHistory(projectId, { limit: 1, offset: 0 })).items;
@@ -167,8 +214,8 @@ describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages w
await writeFile(join(root, 'example.wav'), stereo);
const staged = await stageProductionSoundAsset(projectId, { revisionId: data.currentRevisionId, source, filename: 'example.wav' });
await writeFile(join(root, 'example.wav'), 'replaced');
- const next = await startProductionStageRun(projectId, { revisionId: staged.revision.id }, {
- render: ({ directory }) => renderComposition({ directory, jobId: `sound-${source}-acceptance` }),
+ const next = await startTrackedRun({ revisionId: staged.revision.id }, {
+ render: ({ directory, signal }) => renderComposition({ directory, signal, jobId: `sound-${source}-acceptance` }),
});
expect(await next.done).toBe('completed');
const saved = (await getProductionHistory(projectId, { limit: 1, offset: 0 })).items[0].data;
diff --git a/server/services/htmlComposition/encode.js b/server/services/htmlComposition/encode.js
index e19d8668ed..236f92d8c2 100644
--- a/server/services/htmlComposition/encode.js
+++ b/server/services/htmlComposition/encode.js
@@ -55,7 +55,8 @@ const MAX_FRAME_RETRIES = 2;
// recoverable browser failure; the frame is redrawn there and the encoder
// keeps running, so frames already piped are kept. `onRetry({ frame, attempt,
// error })` reports each restart, `frame` counted on song time.
-export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, reopenPage = null, onRetry = null, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {
+export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, onLifecycle, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, reopenPage = null, onRetry = null, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {
+ onLifecycle?.({ phase: 'encoder-preflight' });
const ffmpeg = await locateFfmpeg();
if (!ffmpeg) throw new Error('ffmpeg not found on PATH');
const tag = await tagFilter();
@@ -92,6 +93,11 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
else if (musicPath) args.push('-map', '1:a', '-af', `${bedTrim}${mastering ? `,${mastering.filter}` : ''},afade=t=out:st=${durationSec - 0.5}:d=0.5`, ...AAC_ENCODE_ARGS);
args.push('-frames:v', String(numFrames), '-t', String(durationSec), '-movflags', '+faststart', '-y', outputPath);
const proc = spawnProcess(ffmpeg, args, safeChildProcessOptions({ stdio: ['pipe', 'ignore', 'pipe'] }));
+ let currentFrame = null;
+ const report = (phase, frame = currentFrame) => {
+ currentFrame = frame;
+ onLifecycle?.({ phase, encoderPid: proc.pid ?? null, frame, frames: numFrames });
+ };
let stderr = '';
let exited = false;
let stopping = false;
@@ -131,6 +137,8 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
signal?.addEventListener('abort', abort, { once: true });
if (!Number.isFinite(offsetSec) || offsetSec < 0) throw new Error('offsetSec must be a non-negative number');
const captureOnce = async t => {
+ signal?.throwIfAborted();
+ report('seek', Math.round(t * fps));
page.check();
// awaitPromise in evaluate is essential: each seek owns its paint.
// Unrounded: pages pick their frame with floor(t × fps + 1e-6), and a time
@@ -139,6 +147,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
const at = offsetSec + t;
await seekComposition(page, at, Math.round(t * fps));
page.check();
+ report('capture', Math.round(t * fps));
const captureStarted = performance.now();
const { data } = await page.send('Page.captureScreenshot', SCREENSHOT).catch(error => {
throw new Error(`Composition capture failed at frame ${Math.round(t * fps)} (song ${+at.toFixed(6)}s) after ${Math.round(performance.now() - captureStarted)}ms: ${error.message}`);
@@ -165,10 +174,13 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
}
}
};
- const write = bytes => Promise.race([
- new Promise((resolve, reject) => proc.stdin.write(bytes, error => error ? reject(error) : resolve())),
- encoderStopped,
- ]);
+ const write = bytes => {
+ report('encoder-write');
+ return Promise.race([
+ new Promise((resolve, reject) => proc.stdin.write(bytes, error => error ? reject(error) : resolve())),
+ encoderStopped,
+ ]);
+ };
const toRgb = png => sharp(png).removeAlpha().raw().toBuffer();
// Output frames per sub-frame count: where a shutter render spent its time.
const histogram = {};
@@ -199,6 +211,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
onProgress?.((n + 1) / numFrames, { frame: n + 1, frames: numFrames });
}
proc.stdin.end();
+ report('encoder-finish');
await finished;
page.check();
const loudness = mastering ? await reportMasteredLoudness({ bin: ffmpeg, outputPath, before: mastering.before, signal, run: runFfmpeg }) : null;
@@ -214,6 +227,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
throw error;
} finally {
signal?.removeEventListener('abort', abort);
+ report('encoder-close');
stop('cleanup requested');
// Wait for close, not just the first error, before deleting partial output.
if (!exited) await new Promise(resolve => proc.once('close', resolve));
diff --git a/server/services/htmlComposition/index.js b/server/services/htmlComposition/index.js
index b0fb4ec479..ef2f0b3e60 100644
--- a/server/services/htmlComposition/index.js
+++ b/server/services/htmlComposition/index.js
@@ -1,3 +1,4 @@
+import { channel } from 'node:diagnostics_channel';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
@@ -16,6 +17,7 @@ import { encodeComposition, encodeContactSheet, proofTimes, synthesizeCompositio
import { validateLaunchVideoAssets } from '../../lib/launchVideoValidation.js';
const active = new Map();
+const lifecycle = channel('portos.composition-render.lifecycle');
// A music-video owner may run as long as its song, and no longer than this.
const MUSIC_VIDEO_DURATION_CAP_SEC = 900;
const MUSIC_VIDEO_OWNER = 'music-video';
@@ -139,10 +141,21 @@ export async function renderComposition(options) {
return maintenance.run('composition', options.jobId, () => renderCompositionAdmitted(options), { continuation: true });
}
-async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec, song, ...input }) {
+async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec, song, signal: parentSignal, ...input }) {
const job = { controller: new AbortController(), committing: false };
active.set(jobId, job);
+ // Relay cancellation through the same commit guard as cancel(jobId). The
+ // returned promise still waits for browser/encoder cleanup before settling.
+ const abort = () => { if (!job.committing) job.controller.abort(parentSignal.reason); };
+ parentSignal?.addEventListener('abort', abort, { once: true });
+ if (parentSignal?.aborted) abort();
const { signal } = job.controller;
+ const began = performance.now();
+ const report = (phase, { encoderPid, frame, frames, failed } = {}) => lifecycle.publish({
+ jobId, phase, elapsedMs: Math.round(performance.now() - began),
+ ...(encoderPid !== undefined ? { encoderPid, frame, frames } : {}),
+ ...(failed !== undefined ? { failed } : {}),
+ });
let page;
let audioDirectory;
let scratchRoot;
@@ -159,6 +172,8 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
try { await write(handle); } finally { await handle.close(); }
};
try {
+ report('preflight');
+ signal.throwIfAborted();
const musicVideo = await resolveMusicVideoOwner({ owner, audio, maxDurationSec });
const parsedInput = validateRequest(htmlCompositionRenderSchema, input);
let { directory } = parsedInput;
@@ -191,9 +206,11 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
scratchRoot = staged.scratchRoot;
signal.throwIfAborted();
}
+ report('browser-open');
page = await openComposition(directory, { signal, validateAssets: needsLaunchGate
? assets => { launchPlan = validateLaunchVideoAssets(assets, launchVideo); launchAssets = assets; }
: undefined });
+ report('contract');
const metadata = await page.evaluate(`(() => {
const c = globalThis.portosComposition;
if (!c || typeof c.seek !== 'function') throw new Error('portosComposition.seek is required');
@@ -223,6 +240,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
ownedPaths.push(proofPath);
const { columns } = await encodeContactSheet(page, target.contract, proofPath, { times, signal });
page.check();
+ report('browser-close');
await page.close({ verify: true });
page = null;
signal.throwIfAborted();
@@ -271,6 +289,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
master: masterLoudness !== false,
audio: musicVideo?.audio,
signal,
+ onLifecycle: detail => report(detail.phase, detail),
onProgress: (fraction, detail) => {
const done = framesDone + (detail?.frame ?? Math.round(fraction * targetFrames));
const elapsed = Date.now() - startedAt;
@@ -292,12 +311,15 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
rendered.push({ ...target, filename, outputPath, sampleHistogram, loudness });
}
// End script execution before post-processing and publishing the result.
+ report('browser-close');
await page.close({ verify: true });
page = null;
signal.throwIfAborted();
+ report('publication-admission');
await withBackupAssetPublication(async () => {
try {
for (const video of rendered) {
+ report('thumbnail');
video.thumbnail = await generateThumbnail(video.outputPath, video.id, launchPlan ? { atSec: launchPlan.posterSec } : undefined);
if (!video.thumbnail) throw new Error('Composition thumbnail generation failed');
signal.throwIfAborted();
@@ -336,6 +358,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
...(video.loudness ? { loudness: video.loudness } : {}),
filename: video.filename, thumbnail: video.thumbnail, createdAt,
}));
+ report('history');
await mutateVideoHistory(history => { history.unshift(...metas); return history; });
success = true;
// `id`/`filename` name a real history entry (the first format); `generationId`
@@ -357,6 +380,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
if (job.committing) maintenance.markCurrentUnsettled();
failure = error;
} finally {
+ report('cleanup');
if (page) await page.close().catch(error => {
maintenance.markCurrentUnsettled();
failure ||= error;
@@ -372,7 +396,9 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,
if (!success) {
for (const path of ownedPaths) await unlinkGuarded(path).catch(error => { if (error.code !== 'ENOENT') maintenance.markCurrentUnsettled(); });
}
+ parentSignal?.removeEventListener('abort', abort);
active.delete(jobId);
+ report('settled', { failed: Boolean(failure) });
}
// Cleanup precedes the terminal event, so the queue cannot advance early.
if (failure) throw failure;
diff --git a/server/services/htmlComposition/renderComposition.preflight.test.js b/server/services/htmlComposition/renderComposition.preflight.test.js
index 7dc0354a6c..ad90e342e2 100644
--- a/server/services/htmlComposition/renderComposition.preflight.test.js
+++ b/server/services/htmlComposition/renderComposition.preflight.test.js
@@ -6,6 +6,7 @@
import { describe, expect, it, vi, afterAll } from 'vitest';
import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
+import { channel } from 'node:diagnostics_channel';
import { randomUUID } from 'node:crypto';
import { lazyTempDataRoot, makePathsProxy, cleanupTempDataRoots, sweepStrayTempRoots } from '../../lib/mockPathsDataRoot.js';
@@ -37,6 +38,52 @@ const { PATHS } = await import('../../lib/fileUtils.js');
const { renderComposition } = await import('./index.js');
describe('renderComposition delivery preflight', () => {
+ // Regression: the production-stage signal used to be ignored, allowing
+ // fixture teardown to delete source/output while its renderer still ran.
+ it.each([false, true])('drains an aborted renderer before settlement (cleanup refusal: %s)', async cleanupFails => {
+ const { maintenance } = await import('../../lib/maintenanceAdmission.js');
+ const jobId = randomUUID();
+ const controller = new AbortController();
+ const encoding = Promise.withResolvers();
+ const closing = Promise.withResolvers();
+ const releaseClose = Promise.withResolvers();
+ const events = [];
+ const diagnostics = channel('portos.composition-render.lifecycle');
+ const observe = event => { if (event.jobId === jobId) events.push(event); };
+ diagnostics.subscribe(observe);
+ encodeComposition.mockImplementationOnce(async (_page, _contract, _output, { signal }) => {
+ encoding.resolve();
+ await new Promise((_, reject) => signal.addEventListener('abort', () => reject(signal.reason), { once: true }));
+ });
+ closeComposition.mockImplementationOnce(async () => {
+ closing.resolve();
+ await releaseClose.promise;
+ if (cleanupFails) throw new Error('Synthetic cleanup refusal');
+ });
+ let settled = false;
+ const done = renderComposition({ directory: 'compositions/example', jobId, signal: controller.signal });
+ const outcome = done.then(value => { settled = true; return value; }, error => { settled = true; return error; });
+ try {
+ await encoding.promise;
+ controller.abort(new Error('Synthetic stage cancellation'));
+ await closing.promise;
+ expect(settled).toBe(false);
+ expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ resource: jobId }));
+ releaseClose.resolve();
+ expect(await outcome).toMatchObject({ message: 'Synthetic stage cancellation' });
+ const blocker = maintenance.status().blockers.find(item => item.resource === jobId);
+ if (cleanupFails) expect(blocker).toMatchObject({ unsettled: true });
+ else expect(blocker).toBeUndefined();
+ expect(events.at(-1)).toMatchObject({ phase: 'settled', failed: true });
+ expect(events.every(event => Object.keys(event).every(key => ['jobId', 'phase', 'elapsedMs', 'failed'].includes(key)))).toBe(true);
+ } finally {
+ releaseClose.resolve();
+ await outcome;
+ diagnostics.unsubscribe(observe);
+ encodeComposition.mockClear();
+ }
+ });
+
it('releases ownership after an ordinary render failure finishes cleanup', async () => {
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const jobId = randomUUID();