Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop Code Animation repairs on cancel or deadline (#10659)
Adam Eivyf65435763f4a
diff --git a/server/services/codeAnimation/stages.db.test.js b/server/services/codeAnimation/stages.db.test.jsindex 24bd77101d..7955f3e03f 100644--- a/server/services/codeAnimation/stages.db.test.js+++ b/server/services/codeAnimation/stages.db.test.js@@ -1,5 +1,5 @@/** Bounded production stages → PostgreSQL → managed files; synthetic renders, no provider. */-import { afterAll, describe, expect, it, vi } from 'vitest';+import { afterAll, afterEach, describe, expect, it, vi } from 'vitest';import { access, readFile } from 'fs/promises';import { join } from 'path';import { createHash } from 'crypto';@@ -7,7 +7,7 @@ import { lazyTempDataRoot, makePathsProxy, cleanupTempDataRoots } from '../../livi.mock('../../lib/paths.js', async importOriginal =>makePathsProxy(await importOriginal(), { dataRoot: () => lazyTempDataRoot('code-animation-stages-') }));// Inject faults only into render-source writes/removal; revision storage stays real.-const stagingFaults = vi.hoisted(() => ({ write: false, cleanup: false }));+const stagingFaults = vi.hoisted(() => ({ write: false, cleanup: false, afterReserve: null }));vi.mock('fs/promises', async importOriginal => {const real = await importOriginal();return {@@ -33,33 +33,44 @@ vi.mock('fs/promises', async importOriginal => {},};});+vi.mock('./projectStore.js', async importOriginal => {+ const real = await importOriginal();+ return {+ ...real,+ reserveRunBytes: async (...args) => {+ const reserved = await real.reserveRunBytes(...args);+ stagingFaults.afterReserve?.();+ return reserved;+ },+ };+});vi.mock('../socket.js', () => ({ emitCodeAnimationChanged: vi.fn() }));-// Exercise the default review adapter with only its external provider boundary doubled.-vi.mock('./preflight.js', async importOriginal => ({- ...await importOriginal(),- preflightProductionProject: vi.fn(async () => ({- resolved: { providerId: 'example-provider', model: 'example-model' }, problems: [],- capabilities: { imageInputAccepted: true }, allowFallback: false,- })),+// Default adapters and real preflight, with only external provider boundaries doubled.+vi.mock('../providers.js', () => {+ const provider = { id: 'example-provider', type: 'api', enabled: true, models: ['example-model'], defaultModel: 'example-model' };+ return { getProviderById: vi.fn(async () => provider), getSelectableProviders: vi.fn(async () => ({ providers: [provider] })) };+});+vi.mock('../promptRunner.js', () => ({+ runPromptThroughProvider: vi.fn(), resolveEffectiveModel: (provider, model) => model || provider.defaultModel,}));-vi.mock('../providers.js', () => ({ getProviderById: vi.fn(async () => ({ id: 'example-provider' })) }));-vi.mock('../promptRunner.js', () => ({ runPromptThroughProvider: vi.fn() }));vi.mock('../runner.js', () => ({ stopRun: vi.fn() }));import { runPromptThroughProvider } from '../promptRunner.js';+import { stopRun } from '../runner.js';import { checkHealth, ensureSchema, query, close } from '../../lib/db.js';import { requireDbOrSkip } from '../../lib/dbTestGate.js';import { createCodeAnimationPackage } from '../../lib/codeAnimationPackage.js';import { PATHS } from '../../lib/paths.js';import {- createProductionProject, exportProductionPackage, getProductionHistory, getProductionProject, importProductionPackage,+ acceptProductionSource, createProductionProject, exportProductionPackage, getProductionHistory, getProductionProject, importProductionPackage,patchProductionProject,} from './projects.js';-import { cancelProductionStageRun, startProductionStageRun } from './stages.js';+import { activeStageRunIds, cancelProductionStageRun, startProductionStageRun } from './stages.js';const health = await checkHealth().catch(error => ({ connected: false, error: error.message }));const ready = requireDbOrSkip('codeAnimation/stages.db.test', health.connected, health.error);if (ready) await ensureSchema();const ids = [];+afterEach(() => { vi.useRealTimers(); stopRun.mockReset(); runPromptThroughProvider.mockReset(); stagingFaults.afterReserve = null; });afterAll(async () => {if (ready && ids.length) await query('DELETE FROM code_animation_projects WHERE id = ANY($1::text[])', [ids]);await close(); cleanupTempDataRoots('code-animation-stages-');@@ -74,7 +85,7 @@ const manifest = (audio = { kind: 'silence' }) => ({});const pkg = (source, audio) => createCodeAnimationPackage(manifest(audio), [{ path: 'src/index.html', content: source }]);const project = async (budgets = {}, audio) => {- const created = await createProductionProject({ manifest: manifest(audio), budgets });+ const created = await createProductionProject({ manifest: manifest(audio), budgets, localSettings: { providerId: 'example-provider', model: 'example-model' } });ids.push(created.id);const { revision } = await importProductionPackage(created.id, pkg('<html><canvas></canvas>FROZEN</html>', audio));return { id: created.id, revision };@@ -100,6 +111,115 @@ const render = vi.fn(async () => ({ jobId: 'media-job', id: 'video-1', filename:const exists = path => access(path).then(() => true, () => false);describe.skipIf(!ready)('Production stage runs', () => {+ // These pin the real default repair adapter at its provider boundary: a+ // canceled/deadline response must not create a candidate or reserve bytes.+ it.each(['cancel', 'late-ack', 'deadline', 'failed-stop'])('stops default repair without publishing a late result (%s)', async mode => {+ const { id, revision } = await project({ timeSeconds: 60 });+ await acceptProductionSource(id, revision.id);+ const started = Promise.withResolvers();+ const response = Promise.withResolvers();+ const stopped = Promise.withResolvers();+ let providerArgs;+ runPromptThroughProvider.mockClear();+ runPromptThroughProvider.mockImplementation(async args => {+ providerArgs = args;+ if (mode !== 'late-ack') {+ args.onRunCreated('owned-repair');+ await args.beforeExecute();+ }+ started.resolve();+ const result = await response.promise;+ args.onRunSettled('owned-repair');+ return result;+ });+ stopRun.mockImplementation(async runId => {+ stopped.resolve(runId);+ if (mode === 'failed-stop') throw new Error('Synthetic stop failure');+ response.resolve({ text: '```html\n<html>MOVING</html>\n```' });+ });+ if (mode === 'deadline') vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] });+ const first = await startProductionStageRun(id, {}, { sample, render });+ await started.promise;+ const before = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];+ expect(providerArgs).toMatchObject({+ source: 'code-animation-repair', timeout: expect.any(Number), absoluteTimeoutMs: providerArgs.timeout,+ onRunCreated: expect.any(Function), onRunSettled: expect.any(Function), beforeExecute: expect.any(Function),+ });+ expect(providerArgs.timeout).toBeGreaterThan(0);+ expect(providerArgs.timeout).toBeLessThanOrEqual(60000);+ if (mode === 'deadline') await vi.advanceTimersByTimeAsync(60000);+ else cancelProductionStageRun(id, first.run.id);+ if (mode === 'late-ack') {+ expect(stopRun).not.toHaveBeenCalled();+ providerArgs.onRunCreated('owned-repair');+ expect(() => providerArgs.beforeExecute()).toThrow();+ }+ expect(await stopped.promise).toBe('owned-repair');+ if (mode === 'failed-stop') response.resolve({ text: '```html\n<html>MOVING</html>\n```' });+ expect(await first.done).toBe(mode === 'deadline' ? 'exhausted' : 'canceled');+ vi.useRealTimers();+ expect(activeStageRunIds()).not.toContain(first.run.id);+ expect(stopRun).toHaveBeenCalledTimes(1);+ const saved = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];+ expect(saved.data).toMatchObject({+ resumable: true, stopReason: mode === 'deadline' ? 'time' : 'canceled',+ repairs: [], output: null, currentRevisionId: revision.id, reservedBytes: before.data.reservedBytes,+ spent: { tokens: 0, iterations: 0, diskBytes: before.data.spent.diskBytes },+ });+ expect(saved.data.stages.find(stage => stage.key === 'repair').status).toBe(mode === 'deadline' ? 'exhausted' : 'canceled');+ expect(await getProductionProject(id)).toMatchObject({ candidateRevisionId: null, acceptedRevisionId: revision.id });+ expect((await exportProductionPackage(id, revision.id)).files[0].content).toContain('FROZEN');+ // A stopped repair releases the project's live slot for another run.+ const retry = await startProductionStageRun(id, {}, { sample, repair: unfreeze, render });+ expect(await retry.done).toBe('completed');+ });++ it.each([false, true])('publishes a normal default repair with route substitution allowed=%s', async allowFallback => {+ const { id, revision } = await project();+ if (allowFallback) await patchProductionProject(id, { localSettings: { providerId: 'example-provider', model: 'example-model', substitution: 'allowed' } });+ runPromptThroughProvider.mockImplementation(async args => {+ args.onRunCreated('owned-repair');+ await args.beforeExecute();+ args.onRunSettled('owned-repair');+ return { text: '```html\n<html>MOVING</html>\n```', provider: { id: allowFallback ? 'fallback-provider' : 'example-provider' }, model: 'example-model', usedFallback: allowFallback };+ });+ const first = await startProductionStageRun(id, {}, { sample, render });+ expect(await first.done).toBe('completed');+ expect(stopRun).not.toHaveBeenCalled();+ expect((await getProductionProject(id)).candidateRevisionId).not.toBe(revision.id);+ const saved = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];+ expect(saved.data.effective).toMatchObject({ substituted: allowFallback, effective: { providerId: allowFallback ? 'fallback-provider' : 'example-provider' } });+ expect(saved.data.stages.find(stage => stage.key === 'repair').status).toBe('completed');+ });++ it('releases a repair reservation when cancellation occurs before any source tree is staged', async () => {+ const { id, revision } = await project();+ const started = Promise.withResolvers();+ const response = Promise.withResolvers();+ runPromptThroughProvider.mockImplementation(async args => {+ args.onRunCreated('owned-repair');+ await args.beforeExecute();+ started.resolve();+ const result = await response.promise;+ args.onRunSettled('owned-repair');+ return result;+ });+ const first = await startProductionStageRun(id, {}, { sample, render });+ await started.promise;+ const before = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];+ stagingFaults.afterReserve = () => {+ stagingFaults.afterReserve = null;+ cancelProductionStageRun(id, first.run.id);+ };+ response.resolve({ text: '```html\n<html>MOVING</html>\n```' });+ expect(await first.done).toBe('canceled');+ const saved = (await getProductionHistory(id, { limit: 1, offset: 0 })).items[0];+ expect(saved.data).toMatchObject({ reservedBytes: before.data.reservedBytes, repairs: [], currentRevisionId: revision.id });+ expect(saved.data.spent.diskBytes).toBe(before.data.spent.diskBytes);+ expect((await getProductionProject(id)).candidateRevisionId).toBe(revision.id);+ expect(stopRun).not.toHaveBeenCalled(); // Provider already settled before reservation.+ });+it.each([true, false])('accounts for failed render-source staging when cleanup refusal is %s', async refuseCleanup => {const { id, revision } = await project();await patchProductionProject(id, { budgets: { diskBytes: revision.totalBytes * 2 } });@@ -130,7 +250,8 @@ describe.skipIf(!ready)('Production stage runs', () => {expect((await getProductionProject(id)).candidateRevisionId).toBe(revision.id);// Budget admission distinguishes retained bytes from confirmed cleanup.const retrySample = vi.fn().mockRejectedValue(Object.assign(new Error('Synthetic sample failure'), { code: 'SAMPLE_FAILED' }));- const retry = await startProductionStageRun(id, {}, { sample: retrySample });+ const retryRepair = vi.fn().mockRejectedValue(new Error('Synthetic repair failure'));+ const retry = await startProductionStageRun(id, {}, { sample: retrySample, repair: retryRepair });expect(await retry.done).toBe(refuseCleanup ? 'exhausted' : 'failed');expect(retrySample).toHaveBeenCalledTimes(refuseCleanup ? 0 : 1);});diff --git a/server/services/codeAnimation/stages.js b/server/services/codeAnimation/stages.jsindex f7ac0c6f9c..a5e5f8f460 100644--- a/server/services/codeAnimation/stages.js+++ b/server/services/codeAnimation/stages.js@@ -109,10 +109,17 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {ctx.state.stages.push(entry);await persist(ctx);const { ms, dimension } = remainingMs(ctx);- const signal = AbortSignal.any([ctx.controller.signal, AbortSignal.timeout(ms)]);+ const deadline = new AbortController();+ const timer = setTimeout(() => deadline.abort(exhausted(dimension)), Math.max(1, Math.ceil(ms)));+ if (ms <= 0) deadline.abort(exhausted(dimension));+ const signal = AbortSignal.any([ctx.controller.signal, deadline.signal]);const began = Date.now();try {- Object.assign(entry, await body({ signal, entry }), { status: 'completed', completedAt: iso() });+ signal.throwIfAborted();+ const result = await body({ signal, entry });+ // A published final render has crossed its atomic commit point already.+ if (!activeRuns.get(ctx.runId)?.committing) signal.throwIfAborted();+ Object.assign(entry, result, { status: 'completed', completedAt: iso() });} catch (error) {const canceled = ctx.controller.signal.aborted;const outOfBudget = !canceled && signal.aborted;@@ -121,6 +128,7 @@ async function runStage(ctx, key, revision, body, { renders = false } = {}) {if (outOfBudget) throw exhausted(dimension);throw error;} finally {+ clearTimeout(timer);if (renders) ctx.state.spent.renderMs += Date.now() - began;await persist(ctx).catch(error => console.error(`❌ Code Animation stage ${key} status could not be saved: ${error.message}`));}@@ -301,7 +309,9 @@ const repairPrompt = ({ manifest, files, entryPath, findings }) => [* settings, run one prompt, and reject a substituted route unless the user* allowed it. Only reached from a user-started run.*/-async function repairViaAuthoringRoute({ project, manifest, files, entryPath, findings }) {+async function repairViaAuthoringRoute({ project, manifest, files, entryPath, findings, signal, timeoutMs }) {+ signal.throwIfAborted();+ const deadline = Date.now() + timeoutMs;const [{ preflightProductionProject, _recordEffectiveRoute }, { runPromptThroughProvider }, { getProviderById }, { extractAnimationHtml }] = await Promise.all([import('./preflight.js'), import('../promptRunner.js'), import('../providers.js'), import('./prompt.js'),]);@@ -311,10 +321,35 @@ async function repairViaAuthoringRoute({ project, manifest, files, entryPath, fi}const provider = await getProviderById(preflight.resolved.providerId);const prompt = repairPrompt({ manifest, files, entryPath, findings });- const result = await runPromptThroughProvider({- provider, model: preflight.resolved.model, effort: preflight.resolved.effort || undefined, prompt,- source: 'code-animation-repair', cwd: PATHS.data, allowFallback: preflight.allowFallback,- });+ const { stopRun } = await import('../runner.js');+ const ownedRuns = new Set();+ const stopping = new Map();+ const stopProvider = id => {+ if (stopping.has(id)) return;+ const stopped = stopRun(id).catch(error => {+ console.error(`❌ Code Animation repair provider cancellation failed: ${error.message}`);+ });+ stopping.set(id, stopped);+ };+ const stopOwned = () => { for (const id of ownedRuns) stopProvider(id); };+ signal.addEventListener('abort', stopOwned, { once: true });+ let result;+ try {+ signal.throwIfAborted();+ const timeout = Math.max(1, deadline - Date.now());+ result = await runPromptThroughProvider({+ provider, model: preflight.resolved.model, effort: preflight.resolved.effort || undefined, prompt,+ timeout, absoluteTimeoutMs: timeout,+ beforeExecute: () => signal.throwIfAborted(),+ onRunCreated: id => { ownedRuns.add(id); if (signal.aborted) stopProvider(id); },+ onRunSettled: id => ownedRuns.delete(id),+ source: 'code-animation-repair', cwd: PATHS.data, allowFallback: preflight.allowFallback,+ });+ signal.throwIfAborted();+ } finally {+ signal.removeEventListener('abort', stopOwned);+ await Promise.all(stopping.values());+ }const route = _recordEffectiveRoute(preflight.resolved, project.localSettings, result);const source = manifest.renderer.kind === 'blender' ? result.text.match(/```(?:python|py)\s*\n([\s\S]*?)```/)?.[1] : extractAnimationHtml(result.text);if (!source) throw new ServerError('The model response did not contain the required source document', { status: 422, code: 'CODE_ANIMATION_REPAIR_INVALID' });@@ -328,7 +363,9 @@ async function repairStage(ctx, revision, findings) {let created = null;await runStage(ctx, 'repair', revision, async ({ signal, entry }) => {const intent = { findingKinds: [...new Set(errors.map(finding => finding.kind))], fromRevisionId: revision.id };- const out = await ctx.deps.repair({ project: ctx.project, manifest: revision.manifest, files: revision.files, entryPath: revision.entryPath, findings: errors, signal });+ signal.throwIfAborted();+ const out = await ctx.deps.repair({ project: ctx.project, manifest: revision.manifest, files: revision.files, entryPath: revision.entryPath, findings: errors, signal, timeoutMs: remainingMs(ctx).ms });+ signal.throwIfAborted();spent.tokens += out.tokens || 0;spent.iterations += 1;if (out.effective) ctx.state.effective = out.effective;@@ -343,16 +380,28 @@ async function repairStage(ctx, revision, findings) {const sourceHash = sourceHashOf(pkg.files);if (sourceHash === revision.sourceHash) throw new ServerError('The repair did not change the source', { status: 422, code: 'CODE_ANIMATION_REPAIR_NOOP' });const totalBytes = pkg.files.reduce((sum, file) => sum + Buffer.byteLength(file.content, file.encoding === 'base64' ? 'base64' : 'utf8'), 0);+ signal.throwIfAborted();await reserve(ctx, totalBytes);+ if (signal.aborted) {+ // No repair tree exists yet; the stage's final persist releases this+ // reservation. Once staging starts, uncertain storage stays charged.+ ctx.state.reservedBytes -= totalBytes;+ spent.diskBytes -= totalBytes;+ signal.throwIfAborted();+ }const id = randomUUID();const storage = await stageProjectFiles(ctx.projectId, id, pkg.files);+ signal.throwIfAborted();created = {id, packageHash: pkg.revisionHash, sourceHash, totalBytes, schemaVersion: pkg.schemaVersion, manifest: pkg.manifest,files: pkg.files.map(({ content: _content, ...file }) => file), storage, createdAt: iso(),parentRevisionId: revision.id, repair: { runId: ctx.runId, stageRunId: entry.stageRunId, intent },};// The repaired revision row first names the tree staged above (#9982).- await withBackupAssetPublication(() => store.commitRepairRevision(ctx.projectId, created));+ await withBackupAssetPublication(() => {+ signal.throwIfAborted();+ return store.commitRepairRevision(ctx.projectId, created);+ });ctx.state.repairs.push({ revisionId: id, fromRevisionId: revision.id, sourceHash, findingKinds: intent.findingKinds, tokens: out.tokens || 0 });ctx.state.currentRevisionId = id;return { intent, toRevisionId: id, toSourceHash: sourceHash };