Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop Code Animation repairs on cancel or deadline (#10659)

Adam Eivyf65435763f4a
diff --git a/server/services/codeAnimation/stages.db.test.js b/server/services/codeAnimation/stages.db.test.js
index 24bd77101d..7955f3e03f 100644
--- a/server/services/codeAnimation/stages.db.test.js
+++ b/server/services/codeAnimation/stages.db.test.js
@@ -1,5 +1,5 @@
/** Bounded production stages → PostgreSQL → managed files; synthetic renders, no provider. */
-import { afterAll, describe, expect, it, vi } from 'vitest';
+import { afterAll, afterEach, describe, expect, it, vi } from 'vitest';
import { access, readFile } from 'fs/promises';
import { join } from 'path';
import { createHash } from 'crypto';
@@ -7,7 +7,7 @@ import { lazyTempDataRoot, makePathsProxy, cleanupTempDataRoots } from '../../li
vi.mock('../../lib/paths.js', async importOriginal =>
makePathsProxy(await importOriginal(), { dataRoot: () => lazyTempDataRoot('code-animation-stages-') }));
// Inject faults only into render-source writes/removal; revision storage stays real.
-const stagingFaults = vi.hoisted(() => ({ write: false, cleanup: false }));
+const stagingFaults = vi.hoisted(() => ({ write: false, cleanup: false, afterReserve: null }));
vi.mock('fs/promises', async importOriginal => {
const real = await importOriginal();
return {
@@ -33,33 +33,44 @@ vi.mock('fs/promises', async importOriginal => {
},
};
});
+vi.mock('./projectStore.js', async importOriginal => {
+ const real = await importOriginal();
+ return {
+ ...real,
+ reserveRunBytes: async (...args) => {
+ const reserved = await real.reserveRunBytes(...args);
+ stagingFaults.afterReserve?.();
+ return reserved;
+ },
+ };
+});
vi.mock('../socket.js', () => ({ emitCodeAnimationChanged: vi.fn() }));
-// Exercise the default review adapter with only its external provider boundary doubled.
-vi.mock('./preflight.js', async importOriginal => ({
- ...await importOriginal(),
- preflightProductionProject: vi.fn(async () => ({
- resolved: { providerId: 'example-provider', model: 'example-model' }, problems: [],
- capabilities: { imageInputAccepted: true }, allowFallback: false,
- })),
+// Default adapters and real preflight, with only external provider boundaries doubled.
+vi.mock('../providers.js', () => {
+ const provider = { id: 'example-provider', type: 'api', enabled: true, models: ['example-model'], defaultModel: 'example-model' };
+ return { getProviderById: vi.fn(async () => provider), getSelectableProviders: vi.fn(async () => ({ providers: [provider] })) };
+});
+vi.mock('../promptRunner.js', () => ({
+ runPromptThroughProvider: vi.fn(), resolveEffectiveModel: (provider, model) => model || provider.defaultModel,
}));
-vi.mock('../providers.js', () => ({ getProviderById: vi.fn(async () => ({ id: 'example-provider' })) }));
-vi.mock('../promptRunner.js', () => ({ runPromptThroughProvider: vi.fn() }));
vi.mock('../runner.js', () => ({ stopRun: vi.fn() }));
import { runPromptThroughProvider } from '../promptRunner.js';
+import { stopRun } from '../runner.js';
import { checkHealth, ensureSchema, query, close } from '../../lib/db.js';
import { requireDbOrSkip } from '../../lib/dbTestGate.js';
import { createCodeAnimationPackage } from '../../lib/codeAnimationPackage.js';
import { PATHS } from '../../lib/paths.js';
import {
- createProductionProject, exportProductionPackage, getProductionHistory, getProductionProject, importProductionPackage,
+ acceptProductionSource, createProductionProject, exportProductionPackage, getProductionHistory, getProductionProject, importProductionPackage,
patchProductionProject,
} from './projects.js';
-import { cancelProductionStageRun, startProductionStageRun } from './stages.js';
+import { activeStageRunIds, cancelProductionStageRun, startProductionStageRun } from './stages.js';
const health = await checkHealth().catch(error => ({ connected: false, error: error.message }));
const ready = requireDbOrSkip('codeAnimation/stages.db.test', health.connected, health.error);
if (ready) await ensureSchema();
const ids = [];
+afterEach(() => { vi.useRealTimers(); stopRun.mockReset(); runPromptThroughProvider.mockReset(); stagingFaults.afterReserve = null; });
afterAll(async () => {
if (ready && ids.length) await query('DELETE FROM code_animation_projects WHERE id = ANY($1::text[])', [ids]);
await close(); cleanupTempDataRoots('code-animation-stages-');
@@ -74,7 +85,7 @@ const manifest = (audio = { kind: 'silence' }) => ({
});
const pkg = (source, audio) => createCodeAnimationPackage(manifest(audio), [{ path: 'src/index.html', content: source }]);
const project = async (budgets = {}, audio) => {
- const created = await createProductionProject({ manifest: manifest(audio), budgets });
+ const created = await createProductionProject({ manifest: manifest(audio), budgets, localSettings: { providerId: 'example-provider', model: 'example-model' } });
ids.push(created.id);
const { revision } = await importProductionPackage(created.id, pkg('<html><canvas></canvas>FROZEN</html>', audio));
return { id: created.id, revision };
@@ -100,6 +111,115 @@ const render = vi.fn(async () => ({ jobId: 'media-job', id: 'video-1', filename:
const exists = path => access(path).then(() => true, () => false);
describe.skipIf(!ready)('Production stage runs', () => {
+ // These pin the real default repair adapter at its provider boundary: a
+ // canceled/deadline response must not create a candidate or reserve bytes.
+ it.each(['cancel', 'late-ack', 'deadline', 'failed-stop'])('stops default repair without publishing a late result (%s)', async mode => {
+ const { id, revision } = await project({ timeSeconds: 60 });
+ await acceptProductionSource(id, revision.id);
+ const started = Promise.withResolvers();
+ const response = Promise.withResolvers();
+ const stopped = Promise.withResolvers();
+ let providerArgs;
+ runPromptThroughProvider.mockClear();
+ runPromptThroughProvider.mockImplementation(async args => {
+ providerArgs = args;
+ if (mode !== 'late-ack') {
+ args.onRunCreated('owned-repair');
+ await args.beforeExecute();
+ }
+ started.resolve();
+ const result = await response.promise;
+ args.onRunSettled('owned-repair');
+ return result;
+ });
+ stopRun.mockImplementation(async runId => {
+ stopped.resolve(runId);
+ if (mode === 'failed-stop') throw new Error('Synthetic stop failure');
+ response.resolve({ text: '```html\n<html>MOVING</html>\n```' });
+ });
+ if (mode === 'deadline') vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] });
+ const first = await startProductionStageRun(id, {}, { sample, render });
+ await started.promise;
+ const before = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];
+ expect(providerArgs).toMatchObject({
+ source: 'code-animation-repair', timeout: expect.any(Number), absoluteTimeoutMs: providerArgs.timeout,
+ onRunCreated: expect.any(Function), onRunSettled: expect.any(Function), beforeExecute: expect.any(Function),
+ });
+ expect(providerArgs.timeout).toBeGreaterThan(0);
+ expect(providerArgs.timeout).toBeLessThanOrEqual(60000);
+ if (mode === 'deadline') await vi.advanceTimersByTimeAsync(60000);
+ else cancelProductionStageRun(id, first.run.id);
+ if (mode === 'late-ack') {
+ expect(stopRun).not.toHaveBeenCalled();
+ providerArgs.onRunCreated('owned-repair');
+ expect(() => providerArgs.beforeExecute()).toThrow();
+ }
+ expect(await stopped.promise).toBe('owned-repair');
+ if (mode === 'failed-stop') response.resolve({ text: '```html\n<html>MOVING</html>\n```' });
+ expect(await first.done).toBe(mode === 'deadline' ? 'exhausted' : 'canceled');
+ vi.useRealTimers();
+ expect(activeStageRunIds()).not.toContain(first.run.id);
+ expect(stopRun).toHaveBeenCalledTimes(1);
+ const saved = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];
+ expect(saved.data).toMatchObject({
+ resumable: true, stopReason: mode === 'deadline' ? 'time' : 'canceled',
+ repairs: [], output: null, currentRevisionId: revision.id, reservedBytes: before.data.reservedBytes,
+ spent: { tokens: 0, iterations: 0, diskBytes: before.data.spent.diskBytes },
+ });
+ expect(saved.data.stages.find(stage => stage.key === 'repair').status).toBe(mode === 'deadline' ? 'exhausted' : 'canceled');
+ expect(await getProductionProject(id)).toMatchObject({ candidateRevisionId: null, acceptedRevisionId: revision.id });
+ expect((await exportProductionPackage(id, revision.id)).files[0].content).toContain('FROZEN');
+ // A stopped repair releases the project's live slot for another run.
+ const retry = await startProductionStageRun(id, {}, { sample, repair: unfreeze, render });
+ expect(await retry.done).toBe('completed');
+ });
+
+ it.each([false, true])('publishes a normal default repair with route substitution allowed=%s', async allowFallback => {
+ const { id, revision } = await project();
+ if (allowFallback) await patchProductionProject(id, { localSettings: { providerId: 'example-provider', model: 'example-model', substitution: 'allowed' } });
+ runPromptThroughProvider.mockImplementation(async args => {
+ args.onRunCreated('owned-repair');
+ await args.beforeExecute();
+ args.onRunSettled('owned-repair');
+ return { text: '```html\n<html>MOVING</html>\n```', provider: { id: allowFallback ? 'fallback-provider' : 'example-provider' }, model: 'example-model', usedFallback: allowFallback };
+ });
+ const first = await startProductionStageRun(id, {}, { sample, render });
+ expect(await first.done).toBe('completed');
+ expect(stopRun).not.toHaveBeenCalled();
+ expect((await getProductionProject(id)).candidateRevisionId).not.toBe(revision.id);
+ const saved = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];
+ expect(saved.data.effective).toMatchObject({ substituted: allowFallback, effective: { providerId: allowFallback ? 'fallback-provider' : 'example-provider' } });
+ expect(saved.data.stages.find(stage => stage.key === 'repair').status).toBe('completed');
+ });
+
+ it('releases a repair reservation when cancellation occurs before any source tree is staged', async () => {
+ const { id, revision } = await project();
+ const started = Promise.withResolvers();
+ const response = Promise.withResolvers();
+ runPromptThroughProvider.mockImplementation(async args => {
+ args.onRunCreated('owned-repair');
+ await args.beforeExecute();
+ started.resolve();
+ const result = await response.promise;
+ args.onRunSettled('owned-repair');
+ return result;
+ });
+ const first = await startProductionStageRun(id, {}, { sample, render });
+ await started.promise;
+ const before = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];
+ stagingFaults.afterReserve = () => {
+ stagingFaults.afterReserve = null;
+ cancelProductionStageRun(id, first.run.id);
+ };
+ response.resolve({ text: '```html\n<html>MOVING</html>\n```' });
+ expect(await first.done).toBe('canceled');
+ const saved = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];
+ expect(saved.data).toMatchObject({ reservedBytes: before.data.reservedBytes, repairs: [], currentRevisionId: revision.id });
+ expect(saved.data.spent.diskBytes).toBe(before.data.spent.diskBytes);
+ expect((await getProductionProject(id)).candidateRevisionId).toBe(revision.id);
+ expect(stopRun).not.toHaveBeenCalled(); // Provider already settled before reservation.
+ });
+
it.each([true, false])('accounts for failed render-source staging when cleanup refusal is %s', async refuseCleanup => {
const { id, revision } = await project();
await patchProductionProject(id, { budgets: { diskBytes: revision.totalBytes * 2 } });
@@ -130,7 +250,8 @@ describe.skipIf(!ready)('Production stage runs', () => {
expect((await getProductionProject(id)).candidateRevisionId).toBe(revision.id);
// Budget admission distinguishes retained bytes from confirmed cleanup.
const retrySample = vi.fn().mockRejectedValue(Object.assign(new Error('Synthetic sample failure'), { code: 'SAMPLE_FAILED' }));
- const retry = await startProductionStageRun(id, {}, { sample: retrySample });
+ const retryRepair = vi.fn().mockRejectedValue(new Error('Synthetic repair failure'));
+ const retry = await startProductionStageRun(id, {}, { sample: retrySample, repair: retryRepair });
expect(await retry.done).toBe(refuseCleanup ? 'exhausted' : 'failed');
expect(retrySample).toHaveBeenCalledTimes(refuseCleanup ? 0 : 1);
});
diff --git a/server/services/codeAnimation/stages.js b/server/services/codeAnimation/stages.js
index f7ac0c6f9c..a5e5f8f460 100644
--- a/server/services/codeAnimation/stages.js
+++ b/server/services/codeAnimation/stages.js
@@ -109,10 +109,17 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {
ctx.state.stages.push(entry);
await persist(ctx);
const { ms, dimension } = remainingMs(ctx);
- const signal = AbortSignal.any([ctx.controller.signal, AbortSignal.timeout(ms)]);
+ const deadline = new AbortController();
+ const timer = setTimeout(() => deadline.abort(exhausted(dimension)), Math.max(1, Math.ceil(ms)));
+ if (ms <= 0) deadline.abort(exhausted(dimension));
+ const signal = AbortSignal.any([ctx.controller.signal, deadline.signal]);
const began = Date.now();
try {
- Object.assign(entry, await body({ signal, entry }), { status: 'completed', completedAt: iso() });
+ signal.throwIfAborted();
+ const result = await body({ signal, entry });
+ // A published final render has crossed its atomic commit point already.
+ if (!activeRuns.get(ctx.runId)?.committing) signal.throwIfAborted();
+ Object.assign(entry, result, { status: 'completed', completedAt: iso() });
} catch (error) {
const canceled = ctx.controller.signal.aborted;
const outOfBudget = !canceled && signal.aborted;
@@ -121,6 +128,7 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {
if (outOfBudget) throw exhausted(dimension);
throw error;
} finally {
+ clearTimeout(timer);
if (renders) ctx.state.spent.renderMs += Date.now() - began;
await persist(ctx).catch(error => console.error(`❌ Code Animation stage ${key} status could not be saved: ${error.message}`));
}
@@ -301,7 +309,9 @@ const repairPrompt = ({ manifest, files, entryPath, findings }) => [
* settings, run one prompt, and reject a substituted route unless the user
* allowed it. Only reached from a user-started run.
*/
-async function repairViaAuthoringRoute({ project, manifest, files, entryPath, findings }) {
+async function repairViaAuthoringRoute({ project, manifest, files, entryPath, findings, signal, timeoutMs }) {
+ signal.throwIfAborted();
+ const deadline = Date.now() + timeoutMs;
const [{ preflightProductionProject, _recordEffectiveRoute }, { runPromptThroughProvider }, { getProviderById }, { extractAnimationHtml }] = await Promise.all([
import('./preflight.js'), import('../promptRunner.js'), import('../providers.js'), import('./prompt.js'),
]);
@@ -311,10 +321,35 @@ async function repairViaAuthoringRoute({ project, manifest, files, entryPath, fi
}
const provider = await getProviderById(preflight.resolved.providerId);
const prompt = repairPrompt({ manifest, files, entryPath, findings });
- const result = await runPromptThroughProvider({
- provider, model: preflight.resolved.model, effort: preflight.resolved.effort || undefined, prompt,
- source: 'code-animation-repair', cwd: PATHS.data, allowFallback: preflight.allowFallback,
- });
+ const { stopRun } = await import('../runner.js');
+ const ownedRuns = new Set();
+ const stopping = new Map();
+ const stopProvider = id => {
+ if (stopping.has(id)) return;
+ const stopped = stopRun(id).catch(error => {
+ console.error(`❌ Code Animation repair provider cancellation failed: ${error.message}`);
+ });
+ stopping.set(id, stopped);
+ };
+ const stopOwned = () => { for (const id of ownedRuns) stopProvider(id); };
+ signal.addEventListener('abort', stopOwned, { once: true });
+ let result;
+ try {
+ signal.throwIfAborted();
+ const timeout = Math.max(1, deadline - Date.now());
+ result = await runPromptThroughProvider({
+ provider, model: preflight.resolved.model, effort: preflight.resolved.effort || undefined, prompt,
+ timeout, absoluteTimeoutMs: timeout,
+ beforeExecute: () => signal.throwIfAborted(),
+ onRunCreated: id => { ownedRuns.add(id); if (signal.aborted) stopProvider(id); },
+ onRunSettled: id => ownedRuns.delete(id),
+ source: 'code-animation-repair', cwd: PATHS.data, allowFallback: preflight.allowFallback,
+ });
+ signal.throwIfAborted();
+ } finally {
+ signal.removeEventListener('abort', stopOwned);
+ await Promise.all(stopping.values());
+ }
const route = _recordEffectiveRoute(preflight.resolved, project.localSettings, result);
const source = manifest.renderer.kind === 'blender' ? result.text.match(/```(?:python|py)\s*\n([\s\S]*?)```/)?.[1] : extractAnimationHtml(result.text);
if (!source) throw new ServerError('The model response did not contain the required source document', { status: 422, code: 'CODE_ANIMATION_REPAIR_INVALID' });
@@ -328,7 +363,9 @@ async function repairStage(ctx, revision, findings) {
let created = null;
await runStage(ctx, 'repair', revision, async ({ signal, entry }) => {
const intent = { findingKinds: [...new Set(errors.map(finding => finding.kind))], fromRevisionId: revision.id };
- const out = await ctx.deps.repair({ project: ctx.project, manifest: revision.manifest, files: revision.files, entryPath: revision.entryPath, findings: errors, signal });
+ signal.throwIfAborted();
+ const out = await ctx.deps.repair({ project: ctx.project, manifest: revision.manifest, files: revision.files, entryPath: revision.entryPath, findings: errors, signal, timeoutMs: remainingMs(ctx).ms });
+ signal.throwIfAborted();
spent.tokens += out.tokens || 0;
spent.iterations += 1;
if (out.effective) ctx.state.effective = out.effective;
@@ -343,16 +380,28 @@ async function repairStage(ctx, revision, findings) {
const sourceHash = sourceHashOf(pkg.files);
if (sourceHash === revision.sourceHash) throw new ServerError('The repair did not change the source', { status: 422, code: 'CODE_ANIMATION_REPAIR_NOOP' });
const totalBytes = pkg.files.reduce((sum, file) => sum + Buffer.byteLength(file.content, file.encoding === 'base64' ? 'base64' : 'utf8'), 0);
+ signal.throwIfAborted();
await reserve(ctx, totalBytes);
+ if (signal.aborted) {
+ // No repair tree exists yet; the stage's final persist releases this
+ // reservation. Once staging starts, uncertain storage stays charged.
+ ctx.state.reservedBytes -= totalBytes;
+ spent.diskBytes -= totalBytes;
+ signal.throwIfAborted();
+ }
const id = randomUUID();
const storage = await stageProjectFiles(ctx.projectId, id, pkg.files);
+ signal.throwIfAborted();
created = {
id, packageHash: pkg.revisionHash, sourceHash, totalBytes, schemaVersion: pkg.schemaVersion, manifest: pkg.manifest,
files: pkg.files.map(({ content: _content, ...file }) => file), storage, createdAt: iso(),
parentRevisionId: revision.id, repair: { runId: ctx.runId, stageRunId: entry.stageRunId, intent },
};
// The repaired revision row first names the tree staged above (#9982).
- await withBackupAssetPublication(() => store.commitRepairRevision(ctx.projectId, created));
+ await withBackupAssetPublication(() => {
+ signal.throwIfAborted();
+ return store.commitRepairRevision(ctx.projectId, created);
+ });
ctx.state.repairs.push({ revisionId: id, fromRevisionId: revision.id, sourceHash, findingKinds: intent.findingKinds, tokens: out.tokens || 0 });
ctx.state.currentRevisionId = id;
return { intent, toRevisionId: id, toSourceHash: sourceHash };