Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: drain canceled production renders and expose bounded timeout phases (#13278)
[._.]/ Adam Eivy36aec939292b
diff --git a/server/services/codeAnimation/stages.js b/server/services/codeAnimation/stages.jsindex fd88b9a32f..dfaa57ea36 100644--- a/server/services/codeAnimation/stages.js+++ b/server/services/codeAnimation/stages.js@@ -11,6 +11,7 @@* status that keeps accepted artifacts. Nothing runs at boot and a restart* marks a stranded run interrupted without calling a provider.*/+import { channel } from 'node:diagnostics_channel';import { randomUUID } from 'crypto';import { join } from 'path';import { createCodeAnimationPackage } from '../../lib/codeAnimationPackage.js';@@ -39,6 +40,8 @@ const RESUMABLE = new Set(['interrupted', 'canceled', 'exhausted', 'failed']);const BUDGET_CODE = 'CODE_ANIMATION_BUDGET_EXHAUSTED';const CANCEL_CODE = 'CODE_ANIMATION_RUN_CANCELED';+const lifecycle = channel('portos.code-animation-stage.lifecycle');+const activeRuns = trackedRenderMap('code-animation', { noun: 'Code Animation production run', link: () => '/code-animation/production' });export const activeStageRunIds = () => [...activeRuns.keys()];@@ -108,7 +111,10 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {checkBudget(ctx);const entry = { key, rendererBinding: ctx.state.renderer?.binding ?? null, stageRunId: randomUUID(), revisionId: revision.id, sourceHash: revision.sourceHash, packageHash: revision.packageHash, status: 'running', startedAt: iso() };ctx.state.stages.push(entry);+ const report = phase => lifecycle.publish({ runId: ctx.runId, stage: key, phase, elapsedMs: Date.now() - ctx.startedAt });+ report('persist-start');await persist(ctx);+ report('execute');const { ms, dimension } = remainingMs(ctx);const deadline = new AbortController();const timer = setTimeout(() => deadline.abort(exhausted(dimension)), Math.max(1, Math.ceil(ms)));@@ -130,9 +136,11 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {throw error;} finally {clearTimeout(timer);+ report('persist-end');if (renders) ctx.state.spent.renderMs += Date.now() - began;await persist(ctx).catch(error => console.error(`❌ Code Animation stage ${key} status could not be saved: ${error.message}`));}+ report('settled');return entry;}diff --git a/server/services/codeAnimation/stages.realBrowser.db.test.js b/server/services/codeAnimation/stages.realBrowser.db.test.jsindex 5158808386..babcd0c342 100644--- a/server/services/codeAnimation/stages.realBrowser.db.test.js+++ b/server/services/codeAnimation/stages.realBrowser.db.test.js@@ -28,7 +28,7 @@ const { checkHealth, ensureSchema, query, close } = await import('../../lib/db.jconst { requireDbOrSkip } = await import('../../lib/dbTestGate.js');const { createCodeAnimationPackage } = await import('../../lib/codeAnimationPackage.js');const { createProductionProject, getProductionHistory, importProductionPackage } = await import('./projects.js');-const { startProductionStageRun } = await import('./stages.js');+const { cancelProductionStageRun, startProductionStageRun } = await import('./stages.js');const { renderComposition } = await import('../htmlComposition/index.js');const { PATHS } = await import('../../lib/fileUtils.js');@@ -66,10 +66,40 @@ let browser;let artifacts;let projectId;let startup;+// Keep only bounded phase/counter evidence; never stringify diagnostic payloads,+// which may later acquire paths or other private context.+const runs = [];+let stageProgress;+let renderProgress;+let diagnosticTimer;+const stageChannel = channel('portos.code-animation-stage.lifecycle');+const renderChannel = channel('portos.composition-render.lifecycle');+const observeStage = ({ stage, phase, elapsedMs }) => { stageProgress = { stage, phase, elapsedMs }; };+const observeRender = ({ phase, elapsedMs, encoderPid, frame, frames }) => {+ renderProgress = { phase, elapsedMs, encoderPid, frame, frames };+};+const reportPending = () => {+ const pending = runs.find(run => !run.settled);+ if (!pending) return;+ const stage = stageProgress;+ const render = renderProgress;+ console.log(`🔬 Production fixture run ${runs.indexOf(pending) + 1} pending: stage=${stage?.stage ?? 'starting'}/${stage?.phase ?? 'starting'} stageAtMs=${stage?.elapsedMs ?? 0} render=${render?.phase ?? 'none'} renderAtMs=${render?.elapsedMs ?? 0} browserPid=${proc?.pid ?? 'none'} browserExited=${proc?.exitCode != null || proc?.signalCode != null} encoderPid=${render?.encoderPid ?? 'none'} frame=${render?.frame ?? 'none'}/${render?.frames ?? 'none'}`);+};+async function startTrackedRun(input, deps) {+ stageProgress = null;+ renderProgress = null;+ const run = await startProductionStageRun(projectId, input, deps);+ runs.push(run);+ run.done.then(() => { run.settled = true; });+ return run;+}const detachedChromeProcessGroupId = (child = proc) => process.platform === 'win32' ? undefined : child?.pid;describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages with real Chrome and ffmpeg', () => {beforeAll(async () => {+ stageChannel.subscribe(observeStage);+ renderChannel.subscribe(observeRender);+ diagnosticTimer = setInterval(reportPending, 15000);const root = lazyTempDataRoot('portos-code-animation-stages-');try {const ws = await _startTestChromeWithRetry({@@ -96,9 +126,26 @@ describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages w}, 90000);afterAll(async () => {- if (projectId) await query('DELETE FROM code_animation_projects WHERE id = $1', [projectId]);- await _cleanupTestBrowser({ browser, proc, artifacts, cleanup: cleanupTempDataRoots, detachedProcessGroupId: detachedChromeProcessGroupId() });- await close();+ clearInterval(diagnosticTimer);+ reportPending();+ try {+ for (const run of runs.filter(run => !run.settled)) {+ try { cancelProductionStageRun(projectId, run.run.id); } catch (error) {+ if (!['CODE_ANIMATION_RUN_COMMITTING', 'CODE_ANIMATION_RUN_NOT_ACTIVE'].includes(error.code)) throw error;+ }+ }+ // A timed-out test leaves its async body alive. Drain its stage and+ // encoder before removing the rows/files they may still be using.+ await Promise.all(runs.map(run => run.done));+ await _cleanupTestBrowser({ browser, proc, artifacts, detachedProcessGroupId: detachedChromeProcessGroupId(), cleanup: async () => {+ if (projectId) await query('DELETE FROM code_animation_projects WHERE id = $1', [projectId]);+ await cleanupTempDataRoots();+ } });+ await close();+ } finally {+ stageChannel.unsubscribe(observeStage);+ renderChannel.unsubscribe(observeRender);+ }}, 25000);it('turns a frozen original into measured findings, a repaired revision and a real MP4, with a real style frame and pilot', async () => {@@ -107,8 +154,8 @@ describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages wconst repair = vi.fn(async ({ files, entryPath }) => ({files: [{ path: entryPath, content: files.find(file => file.path === entryPath).content.replace('const MOVE = false;', 'const MOVE = true;') }],}));- const { done } = await startProductionStageRun(projectId, {}, {- repair, render: ({ directory }) => renderComposition({ directory, jobId: 'stages-acceptance' }),+ const { done } = await startTrackedRun({}, {+ repair, render: ({ directory, signal }) => renderComposition({ directory, signal, jobId: 'stages-acceptance' }),});const status = await done;const [{ data }] = (await getProductionHistory(projectId, { limit: 1, offset: 0 })).items;@@ -167,8 +214,8 @@ describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages wawait writeFile(join(root, 'example.wav'), stereo);const staged = await stageProductionSoundAsset(projectId, { revisionId: data.currentRevisionId, source, filename: 'example.wav' });await writeFile(join(root, 'example.wav'), 'replaced');- const next = await startProductionStageRun(projectId, { revisionId: staged.revision.id }, {- render: ({ directory }) => renderComposition({ directory, jobId: `sound-${source}-acceptance` }),+ const next = await startTrackedRun({ revisionId: staged.revision.id }, {+ render: ({ directory, signal }) => renderComposition({ directory, signal, jobId: `sound-${source}-acceptance` }),});expect(await next.done).toBe('completed');const saved = (await getProductionHistory(projectId, { limit: 1, offset: 0 })).items[0].data;diff --git a/server/services/htmlComposition/encode.js b/server/services/htmlComposition/encode.jsindex e19d8668ed..236f92d8c2 100644--- a/server/services/htmlComposition/encode.js+++ b/server/services/htmlComposition/encode.js@@ -55,7 +55,8 @@ const MAX_FRAME_RETRIES = 2;// recoverable browser failure; the frame is redrawn there and the encoder// keeps running, so frames already piped are kept. `onRetry({ frame, attempt,// error })` reports each restart, `frame` counted on song time.-export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, reopenPage = null, onRetry = null, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {+export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, onLifecycle, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, reopenPage = null, onRetry = null, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {+ onLifecycle?.({ phase: 'encoder-preflight' });const ffmpeg = await locateFfmpeg();if (!ffmpeg) throw new Error('ffmpeg not found on PATH');const tag = await tagFilter();@@ -92,6 +93,11 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,else if (musicPath) args.push('-map', '1:a', '-af', `${bedTrim}${mastering ? `,${mastering.filter}` : ''},afade=t=out:st=${durationSec - 0.5}:d=0.5`, ...AAC_ENCODE_ARGS);args.push('-frames:v', String(numFrames), '-t', String(durationSec), '-movflags', '+faststart', '-y', outputPath);const proc = spawnProcess(ffmpeg, args, safeChildProcessOptions({ stdio: ['pipe', 'ignore', 'pipe'] }));+ let currentFrame = null;+ const report = (phase, frame = currentFrame) => {+ currentFrame = frame;+ onLifecycle?.({ phase, encoderPid: proc.pid ?? null, frame, frames: numFrames });+ };let stderr = '';let exited = false;let stopping = false;@@ -131,6 +137,8 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,signal?.addEventListener('abort', abort, { once: true });if (!Number.isFinite(offsetSec) || offsetSec < 0) throw new Error('offsetSec must be a non-negative number');const captureOnce = async t => {+ signal?.throwIfAborted();+ report('seek', Math.round(t * fps));page.check();// awaitPromise in evaluate is essential: each seek owns its paint.// Unrounded: pages pick their frame with floor(t × fps + 1e-6), and a time@@ -139,6 +147,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,const at = offsetSec + t;await seekComposition(page, at, Math.round(t * fps));page.check();+ report('capture', Math.round(t * fps));const captureStarted = performance.now();const { data } = await page.send('Page.captureScreenshot', SCREENSHOT).catch(error => {throw new Error(`Composition capture failed at frame ${Math.round(t * fps)} (song ${+at.toFixed(6)}s) after ${Math.round(performance.now() - captureStarted)}ms: ${error.message}`);@@ -165,10 +174,13 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,}}};- const write = bytes => Promise.race([- new Promise((resolve, reject) => proc.stdin.write(bytes, error => error ? reject(error) : resolve())),- encoderStopped,- ]);+ const write = bytes => {+ report('encoder-write');+ return Promise.race([+ new Promise((resolve, reject) => proc.stdin.write(bytes, error => error ? reject(error) : resolve())),+ encoderStopped,+ ]);+ };const toRgb = png => sharp(png).removeAlpha().raw().toBuffer();// Output frames per sub-frame count: where a shutter render spent its time.const histogram = {};@@ -199,6 +211,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,onProgress?.((n + 1) / numFrames, { frame: n + 1, frames: numFrames });}proc.stdin.end();+ report('encoder-finish');await finished;page.check();const loudness = mastering ? await reportMasteredLoudness({ bin: ffmpeg, outputPath, before: mastering.before, signal, run: runFfmpeg }) : null;@@ -214,6 +227,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,throw error;} finally {signal?.removeEventListener('abort', abort);+ report('encoder-close');stop('cleanup requested');// Wait for close, not just the first error, before deleting partial output.if (!exited) await new Promise(resolve => proc.once('close', resolve));diff --git a/server/services/htmlComposition/index.js b/server/services/htmlComposition/index.jsindex b0fb4ec479..ef2f0b3e60 100644--- a/server/services/htmlComposition/index.js+++ b/server/services/htmlComposition/index.js@@ -1,3 +1,4 @@+import { channel } from 'node:diagnostics_channel';import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';import { maintenance } from '../../lib/maintenanceAdmission.js';import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';@@ -16,6 +17,7 @@ import { encodeComposition, encodeContactSheet, proofTimes, synthesizeCompositioimport { validateLaunchVideoAssets } from '../../lib/launchVideoValidation.js';const active = new Map();+const lifecycle = channel('portos.composition-render.lifecycle');// A music-video owner may run as long as its song, and no longer than this.const MUSIC_VIDEO_DURATION_CAP_SEC = 900;const MUSIC_VIDEO_OWNER = 'music-video';@@ -139,10 +141,21 @@ export async function renderComposition(options) {return maintenance.run('composition', options.jobId, () => renderCompositionAdmitted(options), { continuation: true });}-async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec, song, ...input }) {+async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec, song, signal: parentSignal, ...input }) {const job = { controller: new AbortController(), committing: false };active.set(jobId, job);+ // Relay cancellation through the same commit guard as cancel(jobId). The+ // returned promise still waits for browser/encoder cleanup before settling.+ const abort = () => { if (!job.committing) job.controller.abort(parentSignal.reason); };+ parentSignal?.addEventListener('abort', abort, { once: true });+ if (parentSignal?.aborted) abort();const { signal } = job.controller;+ const began = performance.now();+ const report = (phase, { encoderPid, frame, frames, failed } = {}) => lifecycle.publish({+ jobId, phase, elapsedMs: Math.round(performance.now() - began),+ ...(encoderPid !== undefined ? { encoderPid, frame, frames } : {}),+ ...(failed !== undefined ? { failed } : {}),+ });let page;let audioDirectory;let scratchRoot;@@ -159,6 +172,8 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,try { await write(handle); } finally { await handle.close(); }};try {+ report('preflight');+ signal.throwIfAborted();const musicVideo = await resolveMusicVideoOwner({ owner, audio, maxDurationSec });const parsedInput = validateRequest(htmlCompositionRenderSchema, input);let { directory } = parsedInput;@@ -191,9 +206,11 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,scratchRoot = staged.scratchRoot;signal.throwIfAborted();}+ report('browser-open');page = await openComposition(directory, { signal, validateAssets: needsLaunchGate? assets => { launchPlan = validateLaunchVideoAssets(assets, launchVideo); launchAssets = assets; }: undefined });+ report('contract');const metadata = await page.evaluate(`(() => {const c = globalThis.portosComposition;if (!c || typeof c.seek !== 'function') throw new Error('portosComposition.seek is required');@@ -223,6 +240,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,ownedPaths.push(proofPath);const { columns } = await encodeContactSheet(page, target.contract, proofPath, { times, signal });page.check();+ report('browser-close');await page.close({ verify: true });page = null;signal.throwIfAborted();@@ -271,6 +289,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,master: masterLoudness !== false,audio: musicVideo?.audio,signal,+ onLifecycle: detail => report(detail.phase, detail),onProgress: (fraction, detail) => {const done = framesDone + (detail?.frame ?? Math.round(fraction * targetFrames));const elapsed = Date.now() - startedAt;@@ -292,12 +311,15 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,rendered.push({ ...target, filename, outputPath, sampleHistogram, loudness });}// End script execution before post-processing and publishing the result.+ report('browser-close');await page.close({ verify: true });page = null;signal.throwIfAborted();+ report('publication-admission');await withBackupAssetPublication(async () => {try {for (const video of rendered) {+ report('thumbnail');video.thumbnail = await generateThumbnail(video.outputPath, video.id, launchPlan ? { atSec: launchPlan.posterSec } : undefined);if (!video.thumbnail) throw new Error('Composition thumbnail generation failed');signal.throwIfAborted();@@ -336,6 +358,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,...(video.loudness ? { loudness: video.loudness } : {}),filename: video.filename, thumbnail: video.thumbnail, createdAt,}));+ report('history');await mutateVideoHistory(history => { history.unshift(...metas); return history; });success = true;// `id`/`filename` name a real history entry (the first format); `generationId`@@ -357,6 +380,7 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,if (job.committing) maintenance.markCurrentUnsettled();failure = error;} finally {+ report('cleanup');if (page) await page.close().catch(error => {maintenance.markCurrentUnsettled();failure ||= error;@@ -372,7 +396,9 @@ async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec,if (!success) {for (const path of ownedPaths) await unlinkGuarded(path).catch(error => { if (error.code !== 'ENOENT') maintenance.markCurrentUnsettled(); });}+ parentSignal?.removeEventListener('abort', abort);active.delete(jobId);+ report('settled', { failed: Boolean(failure) });}// Cleanup precedes the terminal event, so the queue cannot advance early.if (failure) throw failure;diff --git a/server/services/htmlComposition/renderComposition.preflight.test.js b/server/services/htmlComposition/renderComposition.preflight.test.jsindex 7dc0354a6c..ad90e342e2 100644--- a/server/services/htmlComposition/renderComposition.preflight.test.js+++ b/server/services/htmlComposition/renderComposition.preflight.test.js@@ -6,6 +6,7 @@import { describe, expect, it, vi, afterAll } from 'vitest';import { mkdir, writeFile } from 'node:fs/promises';import { join } from 'node:path';+import { channel } from 'node:diagnostics_channel';import { randomUUID } from 'node:crypto';import { lazyTempDataRoot, makePathsProxy, cleanupTempDataRoots, sweepStrayTempRoots } from '../../lib/mockPathsDataRoot.js';@@ -37,6 +38,52 @@ const { PATHS } = await import('../../lib/fileUtils.js');const { renderComposition } = await import('./index.js');describe('renderComposition delivery preflight', () => {+ // Regression: the production-stage signal used to be ignored, allowing+ // fixture teardown to delete source/output while its renderer still ran.+ it.each([false, true])('drains an aborted renderer before settlement (cleanup refusal: %s)', async cleanupFails => {+ const { maintenance } = await import('../../lib/maintenanceAdmission.js');+ const jobId = randomUUID();+ const controller = new AbortController();+ const encoding = Promise.withResolvers();+ const closing = Promise.withResolvers();+ const releaseClose = Promise.withResolvers();+ const events = [];+ const diagnostics = channel('portos.composition-render.lifecycle');+ const observe = event => { if (event.jobId === jobId) events.push(event); };+ diagnostics.subscribe(observe);+ encodeComposition.mockImplementationOnce(async (_page, _contract, _output, { signal }) => {+ encoding.resolve();+ await new Promise((_, reject) => signal.addEventListener('abort', () => reject(signal.reason), { once: true }));+ });+ closeComposition.mockImplementationOnce(async () => {+ closing.resolve();+ await releaseClose.promise;+ if (cleanupFails) throw new Error('Synthetic cleanup refusal');+ });+ let settled = false;+ const done = renderComposition({ directory: 'compositions/example', jobId, signal: controller.signal });+ const outcome = done.then(value => { settled = true; return value; }, error => { settled = true; return error; });+ try {+ await encoding.promise;+ controller.abort(new Error('Synthetic stage cancellation'));+ await closing.promise;+ expect(settled).toBe(false);+ expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ resource: jobId }));+ releaseClose.resolve();+ expect(await outcome).toMatchObject({ message: 'Synthetic stage cancellation' });+ const blocker = maintenance.status().blockers.find(item => item.resource === jobId);+ if (cleanupFails) expect(blocker).toMatchObject({ unsettled: true });+ else expect(blocker).toBeUndefined();+ expect(events.at(-1)).toMatchObject({ phase: 'settled', failed: true });+ expect(events.every(event => Object.keys(event).every(key => ['jobId', 'phase', 'elapsedMs', 'failed'].includes(key)))).toBe(true);+ } finally {+ releaseClose.resolve();+ await outcome;+ diagnostics.unsubscribe(observe);+ encodeComposition.mockClear();+ }+ });+it('releases ownership after an ordinary render failure finishes cleanup', async () => {const { maintenance } = await import('../../lib/maintenanceAdmission.js');const jobId = randomUUID();