Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
[issue-11668] Recover interrupted pipeline text stages safely (#11703)
[._.]/8bfff490e47f
diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.jsindex 655bcce7b9..8b0d18c9ce 100644--- a/server/lib/schemaVersions.js+++ b/server/lib/schemaVersions.js@@ -95,7 +95,9 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({// ahead-version transfer; when its older snapshot is later received by a v3// peer, the merge preserves an existing climax role instead of letting the// older sanitizer's omission LWW-strip or overwrite it.- pipelineIssues: 3,+ // v4 = text-stage generation identity for owner-scoped restart recovery.+ // Older sanitizers strip it; gate transfers to keep execution ownership intact.+ pipelineIssues: 4,// v2 = `series.arc.readerMap` added (Unified Story Builder). Additive +// gracefully-degrading, but version-gated so a not-yet-upgraded peer can't// round-trip a series through its readerMap-unaware sanitizer and LWW-stripdiff --git a/server/lib/schemaVersions.test.js b/server/lib/schemaVersions.test.jsindex d5046b67a7..00a973a186 100644--- a/server/lib/schemaVersions.test.js+++ b/server/lib/schemaVersions.test.js@@ -29,8 +29,8 @@ describe('PORTOS_SCHEMA_VERSIONS', () => {});it('declares pipeline collection layout versions', () => {- // pipelineIssues v3 adds the independently persisted climax arc role.- expect(PORTOS_SCHEMA_VERSIONS.pipelineIssues).toBe(3);+ // pipelineIssues v4 preserves text-stage generation ownership.+ expect(PORTOS_SCHEMA_VERSIONS.pipelineIssues).toBe(4);// pipelineSeries bumped to 14 for authored arc.seriesDesign; v13 added characterArcs[].evolution// (#6440) (v2 = readerMap, v3 = tickingClock, v4 = styleGuide,// v5 = coverImage, v6 = characterArcs, v7 = factCritical + factReference,@@ -92,7 +92,7 @@ describe('buildPortosMeta', () => {const meta = await buildPortosMeta();expect(meta.portosVersion).toMatch(/^\d+\.\d+\.\d+/);expect(meta.schemaVersions.universes).toBe(12);- expect(meta.schemaVersions.pipelineIssues).toBe(3);+ expect(meta.schemaVersions.pipelineIssues).toBe(4);expect(meta.schemaVersions.pipelineSeries).toBe(14);});diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex e6aa7b337f..32816a5c6b 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -126,6 +126,7 @@ import { registerPostReminderSchedule } from './meatspacePostReminder.js';import { registerAutobiographyReminderSchedule } from './autobiographyReminder.js';import { recoverInterruptedRepoClones, recoverStuckClassifications } from './brain.js';import { recoverStuckAnalyses } from './writersRoom/evaluator.js';+import { recoverStuckGeneratingStages } from './pipeline/textStages.js';import { recoverStuckAutoRuns } from './pipeline/autoRunner.js';import { recoverStuckAutopilots } from './pipeline/seriesAutopilot.js';import { recoverInFlightProjects } from './creativeDirector/recovery.js';@@ -463,6 +464,7 @@ const startBackgroundServices = ({ spawnerReady, io, httpsEnabled, port }) => {// after the log's currentSeq is loaded, or they'd write low/duplicate sequence// numbers and corrupt peer cursors.recoverStuckAnalyses().catch(err => logBootstrapFailure('❌ Writers Room recovery failed', err));+ recoverStuckGeneratingStages().catch(err => logBootstrapFailure('❌ Pipeline text-stage recovery failed', err));recoverStuckAutoRuns().catch(err => logBootstrapFailure('❌ Pipeline auto-run recovery failed', err));recoverStuckAutopilots().catch(err => logBootstrapFailure('❌ Pipeline autopilot recovery failed', err));recoverStuckMusicVideoRenders().catch(err => logBootstrapFailure('❌ Music Video recovery failed', err));diff --git a/server/services/googleAuth.workflow.test.js b/server/services/googleAuth.workflow.test.jsindex ee20ce64ca..26c5ba7f55 100644--- a/server/services/googleAuth.workflow.test.js+++ b/server/services/googleAuth.workflow.test.js@@ -1,4 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { join } from 'node:path';const fixture = vi.hoisted(() => ({ files: new Map(), account: { id: '11111111-1111-4111-8111-111111111111', type: 'gmail', enabled: true } }));vi.mock('../lib/fileUtils.js', async importOriginal => ({@@ -25,14 +26,14 @@ import { syncAccount } from './messageSync.js';import { sendGmail } from './messageGmailSync.js';const json = body => new Response(JSON.stringify(body), { headers: { 'Content-Type': 'application/json' } });-const tokensPath = '/mock/calendar/google-auth/tokens.json';+const tokensPath = join('/mock/calendar', 'google-auth', 'tokens.json');let ownedPromises;beforeEach(async () => {ownedPromises = [];fixture.files.clear();await clearAuth();- fixture.files.set('/mock/calendar/google-auth/credentials.json', JSON.stringify({ clientId: 'example-client', clientSecret: 'example-secret' }));+ fixture.files.set(join('/mock/calendar', 'google-auth', 'credentials.json'), JSON.stringify({ clientId: 'example-client', clientSecret: 'example-secret' }));fixture.files.set(tokensPath, JSON.stringify({ access_token: 'expired-access', refresh_token: 'durable-refresh', expiry_date: 1 }));vi.spyOn(console, 'log').mockImplementation(() => {});vi.spyOn(console, 'warn').mockImplementation(() => {});diff --git a/server/services/pipeline/issueSync.js b/server/services/pipeline/issueSync.jsindex 20abe92c19..2b0406d490 100644--- a/server/services/pipeline/issueSync.js+++ b/server/services/pipeline/issueSync.js@@ -13,7 +13,7 @@ import {maybeJournalBeforeOverwrite, setSyncBaseHash, contentHashForRecord,flushBaseHashes, deleteSyncBaseHash, withBaseHashFlushBatch,} from '../../lib/conflictJournal.js';-import { store, readState, saveIssuesNow, renumberInline, sanitizeIssue } from './issuesShared.js';+import { store, readState, saveIssuesNow, renumberInline, sanitizeIssue, TEXT_STAGE_IDS } from './issuesShared.js';import { isStr } from '../../lib/textUtils.js';/**@@ -105,6 +105,20 @@ export async function mergeIssuesFromSync(remoteIssues, {if (senderCannotRepresentClimax && local.arcRole === 'climax' && sanitized.arcRole !== 'climax') {sanitized = { ...sanitized, arcRole: 'climax' };}+ // Legacy peers may edit the issue while round-tripping an unchanged+ // stage through a sanitizer that drops generation ownership. Retain+ // the marker only for the exact same stage revision. An actual stage+ // edit has unknown execution ownership, never the old local owner's.+ if (senderSchemaVersions === null || (Number(senderSchemaVersions.pipelineIssues) || 0) < 4) {+ for (const stageId of TEXT_STAGE_IDS) {+ const prior = local.stages[stageId];+ const incoming = sanitized.stages[stageId];+ if (prior.generation && !incoming.generation+ && JSON.stringify({ ...prior, generation: null }) === JSON.stringify({ ...incoming, generation: null })) {+ sanitized.stages[stageId] = { ...incoming, generation: prior.generation };+ }+ }+ }// Non-blocking conflict journal — archive the losing local issue on// a true 3-way divergence; always advances the base hash. Never// throws into the merge (convergence wins).diff --git a/server/services/pipeline/issues.test.js b/server/services/pipeline/issues.test.jsindex 72a446dee0..7f6605ec09 100644--- a/server/services/pipeline/issues.test.js+++ b/server/services/pipeline/issues.test.js@@ -1024,6 +1024,27 @@ describe('pipeline issues service', () => {});});+ it.each([null, { pipelineIssues: 3 }])('retains generation ownership only for unchanged stage revisions from legacy schema %j', async (senderSchemaVersions) => {+ const created = await svc.createIssue({ seriesId: 'ser-1', title: 'Local work' });+ const generation = { id: 'generation-one', instanceId: 'local-instance' };+ const { issue: local } = await svc.updateStage(created.id, 'prose', { status: 'generating', generation });+ const { generation: omitted, ...legacyStage } = local.stages.prose;+ expect(omitted).toEqual(generation);+ await svc.mergeIssuesFromSync([{ ...local, title: 'Peer title edit', updatedAt: '2099-01-01T00:00:00.000Z',+ stages: { ...local.stages, prose: legacyStage },+ }], { senderSchemaVersions });+ const retained = await svc.getIssue(local.id);+ expect(retained.title).toBe('Peer title edit');+ expect(retained.stages.prose.generation).toEqual(generation);++ // The peer starts different work without an owner-aware schema. Keeping+ // the previous marker here would falsely authorize local boot recovery.+ await svc.mergeIssuesFromSync([{ ...retained, updatedAt: '2099-01-02T00:00:00.000Z',+ stages: { ...retained.stages, prose: { ...legacyStage, updatedAt: '2099-01-02T00:00:00.000Z' } },+ }], { senderSchemaVersions });+ expect((await svc.getIssue(local.id)).stages.prose).toMatchObject({ status: 'generating', generation: null });+ });+it('allows a v3 sender to intentionally replace climax', async () => {const local = await svc.createIssue({ seriesId: 'ser-1', title: 'Decisive Choice' });const withClimax = await svc.updateIssue(local.id, { arcRole: 'climax' });diff --git a/server/services/pipeline/issuesShared.js b/server/services/pipeline/issuesShared.jsindex 7d7b1539c2..fdd1abe73b 100644--- a/server/services/pipeline/issuesShared.js+++ b/server/services/pipeline/issuesShared.js@@ -277,6 +277,11 @@ const sanitizeDraftGate = (raw) => {const sanitizeTextStage = (raw) => ({...sanitizeStage(raw),+ // Execution ownership survives federation; missing legacy metadata stays unknown.+ generation: isStr(raw?.generation?.id) && raw.generation.id.length <= 64+ && isStr(raw.generation.instanceId) && raw.generation.instanceId.length <= 64+ && raw.generation.id && raw.generation.instanceId+ ? { id: raw.generation.id, instanceId: raw.generation.instanceId } : null,canonExtraction: sanitizeCanonExtraction(raw?.canonExtraction),descGaps: sanitizeDescGaps(raw?.descGaps),draftGate: sanitizeDraftGate(raw?.draftGate),diff --git a/server/services/pipeline/seriesAutopilot.test.js b/server/services/pipeline/seriesAutopilot.test.jsindex 81e4aae157..9ab4f3ce9b 100644--- a/server/services/pipeline/seriesAutopilot.test.js+++ b/server/services/pipeline/seriesAutopilot.test.js@@ -20,6 +20,7 @@ vi.mock('crypto', async () => {return { ...actual, randomUUID: () => `uuid-${++uuidCounter}` };});+vi.mock('../instanceIdentity.js', () => ({ getInstanceId: vi.fn(async () => 'test-instance'), UNKNOWN_INSTANCE_ID: 'unknown' }));vi.mock('../instances.js', () => mockNoPeers());vi.mock('../sharing/peerSync.js', () => mockNoPeerSync());diff --git a/server/services/pipeline/textStages.js b/server/services/pipeline/textStages.jsindex b0ee93ec1a..6adc47c597 100644--- a/server/services/pipeline/textStages.js+++ b/server/services/pipeline/textStages.js@@ -10,8 +10,7 @@* styleNotes) plus every *prior* stage's output, so downstream stages can* reference upstream content with `{{stages.idea.output}}` etc.*- * Errors bubble (per project convention — no try/catch) except at the SSE- * boundary in autoRunner.js, which routes failures through a finalizer.+ * Generation owns a persist-then-rethrow boundary so every failure is retryable.** Progress (#3393): every phase of a run is broadcast through* `textStageProgress.js` so the UI shows attempt/judge progress instead of a@@ -19,12 +18,14 @@* emit is a no-op and generation is byte-for-byte the un-streamed path.*/+import { randomUUID } from 'node:crypto';+import { getInstanceId, UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';import { runStagedLLM, resolveStageContext } from '../stageRunner.js';import { getStage } from '../promptService.js';import { beginStageProgress, emitStageProgress, finishStageProgress } from './textStageProgress.js';import { getSeries } from './series.js';import { extractCanonFromProse, summarizeCanonExtraction } from '../universeCanon.js';-import { getIssue, listIssues, updateStage, assertStageUnlocked, TEXT_STAGE_IDS } from './issues.js';+import { getIssue, listIssues, updateStage, updateStageWithLatest, assertStageUnlocked, TEXT_STAGE_IDS } from './issues.js';import { pickCanon } from './seriesCanon.js';import { getUniverse } from '../universeBuilder.js';import { compareIssuesByPosition, NO_LINKED_UNIVERSE_PLACEHOLDER } from './arcPlanner.js';@@ -36,6 +37,9 @@ import { filterCanonForIssue } from '../../lib/storyBible.js';import { usableInputTokens, trimContextToBudget, CHARS_PER_TOKEN } from '../../lib/contextBudget.js';import { escapeRegExp } from '../../lib/textUtils.js';+// Separate from the advisory progress channel: overlapping runs each stay live.+const liveGenerations = new Set();+const STAGE_TO_TEMPLATE = Object.freeze({idea: 'pipeline-idea-expansion',prose: 'pipeline-prose',@@ -657,43 +661,32 @@ async function buildGenerationContext({ series, canon, world, issue, stageId, op}// Run the stage LLM once and persist the response as the active stage state.-// The try/catch is the mandatory persist-then-rethrow boundary: without it an-// LLM throw would leave the stage stuck in `generating` forever. Returns+// The enclosing generation owns failure recovery. Returns// { issue, stage, runId, output }. Each call with a new runId snapshots the// prior active state into runHistory (see snapshotRunHistory) — which is how the// draft gate's rejected attempts stay recoverable for inspection/restore.-async function runStageLLMOnce(issueId, stageId, template, ctx, options) {- let result;- try {- result = await runStagedLLM(template, ctx, {- providerOverride: options.providerId,- // Soft run-level default (Series Autopilot, #1514): unlike providerId it- // loses to a per-stage pin and soft-falls-through to active when- // unavailable. Route callers pass providerId (hard); autopilot passes- // providerIdDefault (soft).- providerDefault: options.providerIdDefault,- modelOverride: options.model,- // Soft run-level model default (Series Autopilot, #1558): loses to a- // per-stage `model` pin, mirroring providerIdDefault. Route callers pass- // model (hard); autopilot passes modelIdDefault (soft).- modelDefault: options.modelIdDefault,- // Soft run-level reasoning effort (Series Autopilot, #3641): loses to a- // per-stage `effort` pin, and the runner clamps/drops it per provider.- effortDefault: options.effortIdDefault,- source: 'pipeline-text-stage',- onRunCreated: options.onRunCreated,- onRunSettled: options.onRunSettled,- });- } catch (err) {- await updateStage(issueId, stageId, {- status: 'error',- errorMessage: (err?.message || String(err)).slice(0, 4000),- });- throw err;- }-+async function runStageLLMOnce(issueId, stageId, template, ctx, options, persist) {+ const result = await runStagedLLM(template, ctx, {+ providerOverride: options.providerId,+ // Soft run-level default (Series Autopilot, #1514): unlike providerId it+ // loses to a per-stage pin and soft-falls-through to active when+ // unavailable. Route callers pass providerId (hard); autopilot passes+ // providerIdDefault (soft).+ providerDefault: options.providerIdDefault,+ modelOverride: options.model,+ // Soft run-level model default (Series Autopilot, #1558): loses to a+ // per-stage `model` pin, mirroring providerIdDefault. Route callers pass+ // model (hard); autopilot passes modelIdDefault (soft).+ modelDefault: options.modelIdDefault,+ // Soft run-level reasoning effort (Series Autopilot, #3641): loses to a+ // per-stage `effort` pin, and the runner clamps/drops it per provider.+ effortDefault: options.effortIdDefault,+ source: 'pipeline-text-stage',+ onRunCreated: options.onRunCreated,+ onRunSettled: options.onRunSettled,+ });const output = (result.content || '').trim();- const { issue, stage } = await updateStage(issueId, stageId, {+ const { issue, stage } = await persist({status: output ? 'ready' : 'error',output,lastRunId: result.runId,@@ -716,7 +709,7 @@ async function runStageLLMOnce(issueId, stageId, template, ctx, options) {* budget exhausted) and keep the best attempt so far. Route callers pass no* chargeAction, so a manual regenerate is never cos-billed.*/-async function runDraftGate({ issueId, stageId, template, ctx, options, gate, emit = () => {} }) {+async function runDraftGate({ issueId, stageId, template, ctx, options, gate, persist, emit = () => {} }) {// Dynamic import breaks the textStages ↔ pipelineJudge require cycle (the judge// imports scopeCharactersForIssue/stageContentOf from here) and keeps the judge// out of the module-eval graph for callers that never gate.@@ -738,7 +731,9 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, em}const started = Date.now();emit({ type: 'phase', phase: 'generate', label: `Drafting attempt ${i + 1} of ${gate.attempts}`, attempt: i + 1, attempts: gate.attempts });- const res = await runStageLLMOnce(issueId, stageId, template, ctx, options);+ const res = await runStageLLMOnce(issueId, stageId, template, ctx, options, persist);+ // A newer generation owns the record; don't judge or re-roll its draft.+ if (res.stage.lastRunId !== res.runId) return res;last = res;let qualityScore = null;let overall = null;@@ -786,7 +781,7 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, em// 'ready' status. The displaced last attempt is snapshotted back into history.if (best && last && best.runId !== last.runId) {emit({ type: 'phase', phase: 'restore', label: 'Restoring the best-scoring draft', attempt: attempts.length, attempts: gate.attempts });- const restored = await updateStage(issueId, stageId, {+ const restored = await persist({status: best.output ? 'ready' : 'error',input: best.input,output: best.output,@@ -815,7 +810,7 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, emrejected: a.runId !== winnerRunId,})),};- const stamped = await updateStage(issueId, stageId, { draftGate }).catch((err) => {+ const stamped = await persist({ draftGate }).catch((err) => {console.warn(`⚠️ Failed to record draft-gate scorecard for issue ${issueId.slice(0, 8)}: ${err.message}`);return null;});@@ -839,7 +834,7 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, em// extract usefully. Non-fatal — prose succeeded, and a noisy extract shouldn't// roll back the user's accepted draft. An orphan series (no universeId) skips// extraction entirely. Returns the stamped { issue, stage } or null (unchanged).-async function maybeExtractCanon(issueId, stageId, output, series, options, emit = () => {}) {+async function maybeExtractCanon(issueId, stageId, output, series, options, persist, emit = () => {}) {if (stageId !== 'prose' || !output || !series.universeId) return null;emit({ type: 'phase', phase: 'canon', label: 'Extracting canon from the draft' });// Canon extraction follows whichever provider/model drove this prose stage —@@ -864,7 +859,7 @@ async function maybeExtractCanon(issueId, stageId, output, series, options, emitreturn summarizeCanonExtraction({ error: err, provider, model });},);- return updateStage(issueId, 'prose', { canonExtraction: marker }).catch((err) => {+ return persist({ canonExtraction: marker }).catch((err) => {console.warn(`⚠️ Failed to record canon-extraction status for issue ${issueId.slice(0, 8)}: ${err.message}`);return null;});@@ -890,6 +885,8 @@ export async function generateStage(issueId, stageId, options = {}) {throw new Error(`generateStage: unsupported stageId "${stageId}"`);}// Generation may overlap, but only the first run owns this stream.+ const generation = { id: randomUUID(), instanceId: await getInstanceId() };+ liveGenerations.add(generation.id);const ownsProgress = beginStageProgress(issueId, stageId);const emit = (payload) => {if (ownsProgress) emitStageProgress(issueId, stageId, payload);@@ -897,8 +894,9 @@ export async function generateStage(issueId, stageId, options = {}) {emit({ type: 'start', issueId, stageId, at: new Date().toISOString() });// Settle to an outcome object rather than try/catch so the terminal frame// ships on BOTH paths before the error resumes bubbling to the caller.- const outcome = await runTextStage(issueId, stageId, options, emit)+ const outcome = await runTextStage(issueId, stageId, options, emit, generation).then((result) => ({ result }), (error) => ({ error }));+ liveGenerations.delete(generation.id);if (outcome.error) {if (ownsProgress) finishStageProgress(issueId, stageId, {type: 'error',@@ -922,8 +920,8 @@ export async function generateStage(issueId, stageId, options = {}) {// The generation itself — everything between the `start` and terminal progress// frames. Split out of generateStage so the frames wrap the whole run (context-// build included) without a try/catch around it.-async function runTextStage(issueId, stageId, options, emit) {+// build included). Persistence is guarded by the generation identity.+async function runTextStage(issueId, stageId, options, emit, generation) {const template = STAGE_TO_TEMPLATE[stageId];const issue = await getIssue(issueId);const series = await getSeries(issue.seriesId);@@ -941,29 +939,65 @@ async function runTextStage(issueId, stageId, options, emit) {// three rejects.assertStageUnlocked(issue, stageId);- await updateStage(issueId, stageId, { status: 'generating', errorMessage: '' });-- emit({ type: 'phase', phase: 'context', label: 'Building the prompt context' });- const ctx = await buildGenerationContext({ series, canon, world, issue, stageId, options });- const gate = resolveDraftGate(stageId, template, options);-- // Draft gate (opt-in, default off) vs the single-shot path. The single-shot- // path is byte-for-byte the pre-#2169 behavior so a stage with draftAttempts=1- // (the default) is unchanged.- let result;- if (gate.attempts > 1) {- result = await runDraftGate({ issueId, stageId, template, ctx, options, gate, emit });- } else {- emit({ type: 'phase', phase: 'generate', label: 'Drafting', attempt: 1, attempts: 1 });- result = await runStageLLMOnce(issueId, stageId, template, ctx, options);- }+ await updateStage(issueId, stageId, { status: 'generating', errorMessage: '', generation });+ const persist = (patch) => updateStageWithLatest(issueId, stageId, (current) =>+ current.generation?.id === generation.id+ && current.generation?.instanceId === generation.instanceId ? patch : {});++ try {+ emit({ type: 'phase', phase: 'context', label: 'Building the prompt context' });+ const ctx = await buildGenerationContext({ series, canon, world, issue, stageId, options });+ const gate = resolveDraftGate(stageId, template, options);++ // Draft gate (opt-in, default off) vs the single-shot path. The single-shot+ // path is byte-for-byte the pre-#2169 behavior so a stage with draftAttempts=1+ // (the default) is unchanged.+ let result;+ if (gate.attempts > 1) {+ result = await runDraftGate({ issueId, stageId, template, ctx, options, gate, persist, emit });+ } else {+ emit({ type: 'phase', phase: 'generate', label: 'Drafting', attempt: 1, attempts: 1 });+ result = await runStageLLMOnce(issueId, stageId, template, ctx, options, persist);+ }++ let { issue: updatedIssue, stage } = result;+ // Prose canon extraction on the FINAL (winning) output — once, not per attempt.+ const current = await getIssue(issueId);+ if (current.stages[stageId].generation?.id !== generation.id) return { ...result, issue: current, stage: current.stages[stageId] };+ const stamped = await maybeExtractCanon(issueId, stageId, result.output, series, options, persist, emit);+ if (stamped) ({ issue: updatedIssue, stage } = stamped);- let { issue: updatedIssue, stage } = result;- // Prose canon extraction on the FINAL (winning) output — once, not per attempt.- const stamped = await maybeExtractCanon(issueId, stageId, result.output, series, options, emit);- if (stamped) ({ issue: updatedIssue, stage } = stamped);+ return { issue: updatedIssue, stage, runId: result.runId };+ } catch (error) {+ await persist({+ status: 'error',+ errorMessage: (error?.message || String(error)).slice(0, 4000),+ }).catch((persistError) => {+ console.error(`❌ Pipeline stage error recovery failed: ${persistError.message}`);+ });+ throw error;+ }+}- return { issue: updatedIssue, stage, runId: result.runId };+/** Only this install's abandoned generations may be demoted; legacy ownership is unknown. */+export async function recoverStuckGeneratingStages() {+ const instanceId = await getInstanceId();+ if (!instanceId || instanceId === UNKNOWN_INSTANCE_ID) return 0;+ let recovered = 0;+ for (const issue of await listIssues({ withHistory: false })) {+ for (const stageId of TEXT_STAGE_IDS) {+ const snapshot = issue.stages[stageId];+ if (snapshot.status !== 'generating' || snapshot.generation?.instanceId !== instanceId) continue;+ await updateStageWithLatest(issue.id, stageId, (current) => {+ if (current.status !== 'generating' || current.generation?.instanceId !== instanceId+ || current.generation?.id !== snapshot.generation.id+ || current.updatedAt !== snapshot.updatedAt || liveGenerations.has(current.generation.id)) return {};+ recovered += 1;+ return { status: 'error', errorMessage: 'Interrupted by a server restart — generate again.' };+ });+ }+ }+ return recovered;}// Export internals for tests.diff --git a/server/services/pipeline/textStages.test.js b/server/services/pipeline/textStages.test.jsindex dde4d575b7..a6dd826491 100644--- a/server/services/pipeline/textStages.test.js+++ b/server/services/pipeline/textStages.test.js@@ -18,6 +18,7 @@ tryReadFile: vi.fn().mockResolvedValue(null),ensureDir: vi.fn().mockResolvedValue(undefined),atomicWrite: vi.fn(async (path, data) => { fileStore.set(path, data); }),readJSONFile: vi.fn(async (path, fallback) => (fileStore.has(path) ? fileStore.get(path) : fallback)),+ readJSONFileStrict: vi.fn(async (path, fallback) => ({ ok: true, value: fileStore.has(path) ? fileStore.get(path) : fallback })),}));let uuidCounter = 0;@@ -26,6 +27,7 @@ vi.mock('crypto', async () => {return { ...actual, randomUUID: () => `uuid-${++uuidCounter}` };});+vi.mock('../instanceIdentity.js', () => ({ getInstanceId: vi.fn(async () => 'test-instance'), UNKNOWN_INSTANCE_ID: 'unknown' }));vi.mock('../instances.js', () => mockNoPeers());vi.mock('../sharing/peerSync.js', () => mockNoPeerSync());@@ -193,6 +195,54 @@ describe('pipeline text stage generator', () => {expect(after.stages.idea.errorMessage).toContain('simulated provider 500');});+ it('recovers context failures after the generating write and preserves the prior draft', async () => {+ const { issue } = await seed();+ await issuesSvc.updateStage(issue.id, 'idea', { output: 'Prior draft', lastRunId: 'prior-run' });+ const error = new Error('Context could not be built');+ const options = { get seedInput() { throw error; } };+ await expect(textStages.generateStage(issue.id, 'idea', options)).rejects.toBe(error);+ expect((await issuesSvc.getIssue(issue.id)).stages.idea).toMatchObject({+ status: 'error', errorMessage: error.message, output: 'Prior draft', lastRunId: 'prior-run', runHistory: [],+ });+ });++ it('rethrows the original context error when persisting recovery also fails', async () => {+ const { issue } = await seed();+ const { atomicWrite } = await import('../../lib/fileUtils.js');+ const error = new Error('Original context failure');+ const options = { get seedInput() {+ atomicWrite.mockRejectedValueOnce(new Error('Recovery storage failure'));+ throw error;+ } };+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});+ await expect(textStages.generateStage(issue.id, 'idea', options)).rejects.toBe(error);+ expect(log).toHaveBeenCalledWith(expect.stringContaining('Recovery storage failure'));+ log.mockRestore();+ });++ it('recovers only abandoned local generations, preserving foreign and legacy work across sync', async () => {+ const { series, issue } = await seed();+ const foreign = await issuesSvc.createIssue({ seriesId: series.id, title: 'Peer work' });+ const legacy = await issuesSvc.createIssue({ seriesId: series.id, title: 'Unknown work' });+ await issuesSvc.updateStage(issue.id, 'idea', {+ status: 'generating', output: 'Prior draft', generation: { id: 'interrupted', instanceId: 'test-instance' },+ });+ await issuesSvc.mergeIssuesFromSync([{ ...foreign, updatedAt: '2099-01-01T00:00:00.000Z',+ stages: { ...foreign.stages, prose: { status: 'generating', generation: { id: 'peer-run', instanceId: 'peer-instance' } } },+ }], { senderSchemaVersions: { pipelineIssues: 4 } });+ await issuesSvc.updateStage(legacy.id, 'prose', { status: 'generating' });+ expect(await textStages.recoverStuckGeneratingStages()).toBe(1);+ expect((await issuesSvc.getIssue(issue.id)).stages.idea).toMatchObject({+ status: 'error', output: 'Prior draft', errorMessage: 'Interrupted by a server restart — generate again.',+ });+ expect((await issuesSvc.getIssue(foreign.id)).stages.prose).toMatchObject({+ status: 'generating', generation: { id: 'peer-run', instanceId: 'peer-instance' },+ });+ expect((await issuesSvc.getIssue(legacy.id)).stages.prose.status).toBe('generating');+ expect(await textStages.recoverStuckGeneratingStages()).toBe(0);+ expect((await textStages.generateStage(issue.id, 'idea')).stage.status).toBe('ready');+ });+it('rejects unsupported stage ids', async () => {const { issue } = await seed();await expect(textStages.generateStage(issue.id, 'comicPages')).rejects.toThrow(/unsupported stageId/);@@ -1337,6 +1387,7 @@ describe('text-stage generation progress frames (#3393)', () => {await firstAtProvider;const second = textStages.generateStage(issue.id, 'prose', {});await secondAtProvider;+ expect(await textStages.recoverStuckGeneratingStages()).toBe(0);finishSecond();await second;expect(framesOf(res).some((frame) => frame.type === 'complete')).toBe(false);@@ -1345,9 +1396,44 @@ describe('text-stage generation progress frames (#3393)', () => {finishFirst();await first;+ expect((await issuesSvc.getIssue(issue.id)).stages.prose.output).toBe('Second draft');expect(framesOf(res).filter((frame) => frame.type === 'complete')).toHaveLength(1);});+ it('a late provider failure cannot demote a newer live generation', async () => {+ const { issue } = await seedProse();+ let startedFirst;+ let startedSecond;+ let failFirst;+ let finishSecond;+ const firstAtProvider = new Promise((resolve) => { startedFirst = resolve; });+ const secondAtProvider = new Promise((resolve) => { startedSecond = resolve; });+ const firstCanFail = new Promise((resolve) => { failFirst = resolve; });+ const secondCanFinish = new Promise((resolve) => { finishSecond = resolve; });+ runnerSvc.executeApiRun.mockImplementationOnce(async ({ onComplete }) => {+ startedFirst();+ await firstCanFail;+ onComplete({ success: false, error: 'Old run failed' });+ }).mockImplementationOnce(async ({ onData, onComplete }) => {+ startedSecond();+ await secondCanFinish;+ onData('New draft');+ onComplete({ success: true });+ });+ const first = textStages.generateStage(issue.id, 'prose');+ const firstFailed = expect(first).rejects.toThrow('Old run failed');+ await firstAtProvider;+ const second = textStages.generateStage(issue.id, 'prose');+ await secondAtProvider;+ failFirst();+ await firstFailed;+ expect((await issuesSvc.getIssue(issue.id)).stages.prose.status).toBe('generating');+ expect(await textStages.recoverStuckGeneratingStages()).toBe(0);+ finishSecond();+ await second;+ expect((await issuesSvc.getIssue(issue.id)).stages.prose).toMatchObject({ status: 'ready', output: 'New draft' });+ });+it('streams start → context → generate → complete on the single-shot path', async () => {const { issue } = await seedProse();const res = fakeRes();diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.jsindex 987b049d0b..5529f7660e 100644--- a/server/services/sharing/peerSync.test.js+++ b/server/services/sharing/peerSync.test.js@@ -4152,7 +4152,7 @@ describe('peerSync', () => {record: { id: 'ser-1', name: 'Foo' },assetManifest: [],sourceInstanceId: 'peer-a',- portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 4 } },+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 5 } },});expect(mergeSeriesFromSync).toHaveBeenCalled();expect(mergeIssuesFromSync).not.toHaveBeenCalled();@@ -4167,10 +4167,10 @@ describe('peerSync', () => {issues: [{ id: 'i1', seriesId: 'ser-1', deleted: false, deletedAt: null }],assetManifest: [],sourceInstanceId: 'peer-a',- portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 4 } },+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 5 } },}).catch((err) => err);expect(rejection.code).toBe('PEER_SYNC_SCHEMA_VERSION_AHEAD');- expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 4, receiverV: 3 }]);+ expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 5, receiverV: 4 }]);expect(mergeSeriesFromSync).not.toHaveBeenCalled();expect(mergeIssuesFromSync).not.toHaveBeenCalled();});@@ -4265,10 +4265,10 @@ describe('peerSync', () => {issues: [{ id: 'i1', seriesId: 'ser-1', deleted: false, deletedAt: null }],assetManifest: [],sourceInstanceId: 'peer-a',- portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 4 } },+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 5 } },}).catch((err) => err);expect(rejection.code).toBe('PEER_SYNC_SCHEMA_VERSION_AHEAD');- expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 4, receiverV: 3 }]);+ expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 5, receiverV: 4 }]);expect(mergeSeriesFromSync).not.toHaveBeenCalled();expect(mergeIssuesFromSync).not.toHaveBeenCalled();});