Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

[issue-11668] Recover interrupted pipeline text stages safely (#11703)

[._.]/8bfff490e47f
diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.js
index 655bcce7b9..8b0d18c9ce 100644
--- a/server/lib/schemaVersions.js
+++ b/server/lib/schemaVersions.js
@@ -95,7 +95,9 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({
// ahead-version transfer; when its older snapshot is later received by a v3
// peer, the merge preserves an existing climax role instead of letting the
// older sanitizer's omission LWW-strip or overwrite it.
- pipelineIssues: 3,
+ // v4 = text-stage generation identity for owner-scoped restart recovery.
+ // Older sanitizers strip it; gate transfers to keep execution ownership intact.
+ pipelineIssues: 4,
// v2 = `series.arc.readerMap` added (Unified Story Builder). Additive +
// gracefully-degrading, but version-gated so a not-yet-upgraded peer can't
// round-trip a series through its readerMap-unaware sanitizer and LWW-strip
diff --git a/server/lib/schemaVersions.test.js b/server/lib/schemaVersions.test.js
index d5046b67a7..00a973a186 100644
--- a/server/lib/schemaVersions.test.js
+++ b/server/lib/schemaVersions.test.js
@@ -29,8 +29,8 @@ describe('PORTOS_SCHEMA_VERSIONS', () => {
});
it('declares pipeline collection layout versions', () => {
- // pipelineIssues v3 adds the independently persisted climax arc role.
- expect(PORTOS_SCHEMA_VERSIONS.pipelineIssues).toBe(3);
+ // pipelineIssues v4 preserves text-stage generation ownership.
+ expect(PORTOS_SCHEMA_VERSIONS.pipelineIssues).toBe(4);
// pipelineSeries bumped to 14 for authored arc.seriesDesign; v13 added characterArcs[].evolution
// (#6440) (v2 = readerMap, v3 = tickingClock, v4 = styleGuide,
// v5 = coverImage, v6 = characterArcs, v7 = factCritical + factReference,
@@ -92,7 +92,7 @@ describe('buildPortosMeta', () => {
const meta = await buildPortosMeta();
expect(meta.portosVersion).toMatch(/^\d+\.\d+\.\d+/);
expect(meta.schemaVersions.universes).toBe(12);
- expect(meta.schemaVersions.pipelineIssues).toBe(3);
+ expect(meta.schemaVersions.pipelineIssues).toBe(4);
expect(meta.schemaVersions.pipelineSeries).toBe(14);
});
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index e6aa7b337f..32816a5c6b 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -126,6 +126,7 @@ import { registerPostReminderSchedule } from './meatspacePostReminder.js';
import { registerAutobiographyReminderSchedule } from './autobiographyReminder.js';
import { recoverInterruptedRepoClones, recoverStuckClassifications } from './brain.js';
import { recoverStuckAnalyses } from './writersRoom/evaluator.js';
+import { recoverStuckGeneratingStages } from './pipeline/textStages.js';
import { recoverStuckAutoRuns } from './pipeline/autoRunner.js';
import { recoverStuckAutopilots } from './pipeline/seriesAutopilot.js';
import { recoverInFlightProjects } from './creativeDirector/recovery.js';
@@ -463,6 +464,7 @@ const startBackgroundServices = ({ spawnerReady, io, httpsEnabled, port }) => {
// after the log's currentSeq is loaded, or they'd write low/duplicate sequence
// numbers and corrupt peer cursors.
recoverStuckAnalyses().catch(err => logBootstrapFailure('❌ Writers Room recovery failed', err));
+ recoverStuckGeneratingStages().catch(err => logBootstrapFailure('❌ Pipeline text-stage recovery failed', err));
recoverStuckAutoRuns().catch(err => logBootstrapFailure('❌ Pipeline auto-run recovery failed', err));
recoverStuckAutopilots().catch(err => logBootstrapFailure('❌ Pipeline autopilot recovery failed', err));
recoverStuckMusicVideoRenders().catch(err => logBootstrapFailure('❌ Music Video recovery failed', err));
diff --git a/server/services/googleAuth.workflow.test.js b/server/services/googleAuth.workflow.test.js
index ee20ce64ca..26c5ba7f55 100644
--- a/server/services/googleAuth.workflow.test.js
+++ b/server/services/googleAuth.workflow.test.js
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { join } from 'node:path';
const fixture = vi.hoisted(() => ({ files: new Map(), account: { id: '11111111-1111-4111-8111-111111111111', type: 'gmail', enabled: true } }));
vi.mock('../lib/fileUtils.js', async importOriginal => ({
@@ -25,14 +26,14 @@ import { syncAccount } from './messageSync.js';
import { sendGmail } from './messageGmailSync.js';
const json = body => new Response(JSON.stringify(body), { headers: { 'Content-Type': 'application/json' } });
-const tokensPath = '/mock/calendar/google-auth/tokens.json';
+const tokensPath = join('/mock/calendar', 'google-auth', 'tokens.json');
let ownedPromises;
beforeEach(async () => {
ownedPromises = [];
fixture.files.clear();
await clearAuth();
- fixture.files.set('/mock/calendar/google-auth/credentials.json', JSON.stringify({ clientId: 'example-client', clientSecret: 'example-secret' }));
+ fixture.files.set(join('/mock/calendar', 'google-auth', 'credentials.json'), JSON.stringify({ clientId: 'example-client', clientSecret: 'example-secret' }));
fixture.files.set(tokensPath, JSON.stringify({ access_token: 'expired-access', refresh_token: 'durable-refresh', expiry_date: 1 }));
vi.spyOn(console, 'log').mockImplementation(() => {});
vi.spyOn(console, 'warn').mockImplementation(() => {});
diff --git a/server/services/pipeline/issueSync.js b/server/services/pipeline/issueSync.js
index 20abe92c19..2b0406d490 100644
--- a/server/services/pipeline/issueSync.js
+++ b/server/services/pipeline/issueSync.js
@@ -13,7 +13,7 @@ import {
maybeJournalBeforeOverwrite, setSyncBaseHash, contentHashForRecord,
flushBaseHashes, deleteSyncBaseHash, withBaseHashFlushBatch,
} from '../../lib/conflictJournal.js';
-import { store, readState, saveIssuesNow, renumberInline, sanitizeIssue } from './issuesShared.js';
+import { store, readState, saveIssuesNow, renumberInline, sanitizeIssue, TEXT_STAGE_IDS } from './issuesShared.js';
import { isStr } from '../../lib/textUtils.js';
/**
@@ -105,6 +105,20 @@ export async function mergeIssuesFromSync(remoteIssues, {
if (senderCannotRepresentClimax && local.arcRole === 'climax' && sanitized.arcRole !== 'climax') {
sanitized = { ...sanitized, arcRole: 'climax' };
}
+ // Legacy peers may edit the issue while round-tripping an unchanged
+ // stage through a sanitizer that drops generation ownership. Retain
+ // the marker only for the exact same stage revision. An actual stage
+ // edit has unknown execution ownership, never the old local owner's.
+ if (senderSchemaVersions === null || (Number(senderSchemaVersions.pipelineIssues) || 0) < 4) {
+ for (const stageId of TEXT_STAGE_IDS) {
+ const prior = local.stages[stageId];
+ const incoming = sanitized.stages[stageId];
+ if (prior.generation && !incoming.generation
+ && JSON.stringify({ ...prior, generation: null }) === JSON.stringify({ ...incoming, generation: null })) {
+ sanitized.stages[stageId] = { ...incoming, generation: prior.generation };
+ }
+ }
+ }
// Non-blocking conflict journal — archive the losing local issue on
// a true 3-way divergence; always advances the base hash. Never
// throws into the merge (convergence wins).
diff --git a/server/services/pipeline/issues.test.js b/server/services/pipeline/issues.test.js
index 72a446dee0..7f6605ec09 100644
--- a/server/services/pipeline/issues.test.js
+++ b/server/services/pipeline/issues.test.js
@@ -1024,6 +1024,27 @@ describe('pipeline issues service', () => {
});
});
+ it.each([null, { pipelineIssues: 3 }])('retains generation ownership only for unchanged stage revisions from legacy schema %j', async (senderSchemaVersions) => {
+ const created = await svc.createIssue({ seriesId: 'ser-1', title: 'Local work' });
+ const generation = { id: 'generation-one', instanceId: 'local-instance' };
+ const { issue: local } = await svc.updateStage(created.id, 'prose', { status: 'generating', generation });
+ const { generation: omitted, ...legacyStage } = local.stages.prose;
+ expect(omitted).toEqual(generation);
+ await svc.mergeIssuesFromSync([{ ...local, title: 'Peer title edit', updatedAt: '2099-01-01T00:00:00.000Z',
+ stages: { ...local.stages, prose: legacyStage },
+ }], { senderSchemaVersions });
+ const retained = await svc.getIssue(local.id);
+ expect(retained.title).toBe('Peer title edit');
+ expect(retained.stages.prose.generation).toEqual(generation);
+
+ // The peer starts different work without an owner-aware schema. Keeping
+ // the previous marker here would falsely authorize local boot recovery.
+ await svc.mergeIssuesFromSync([{ ...retained, updatedAt: '2099-01-02T00:00:00.000Z',
+ stages: { ...retained.stages, prose: { ...legacyStage, updatedAt: '2099-01-02T00:00:00.000Z' } },
+ }], { senderSchemaVersions });
+ expect((await svc.getIssue(local.id)).stages.prose).toMatchObject({ status: 'generating', generation: null });
+ });
+
it('allows a v3 sender to intentionally replace climax', async () => {
const local = await svc.createIssue({ seriesId: 'ser-1', title: 'Decisive Choice' });
const withClimax = await svc.updateIssue(local.id, { arcRole: 'climax' });
diff --git a/server/services/pipeline/issuesShared.js b/server/services/pipeline/issuesShared.js
index 7d7b1539c2..fdd1abe73b 100644
--- a/server/services/pipeline/issuesShared.js
+++ b/server/services/pipeline/issuesShared.js
@@ -277,6 +277,11 @@ const sanitizeDraftGate = (raw) => {
const sanitizeTextStage = (raw) => ({
...sanitizeStage(raw),
+ // Execution ownership survives federation; missing legacy metadata stays unknown.
+ generation: isStr(raw?.generation?.id) && raw.generation.id.length <= 64
+ && isStr(raw.generation.instanceId) && raw.generation.instanceId.length <= 64
+ && raw.generation.id && raw.generation.instanceId
+ ? { id: raw.generation.id, instanceId: raw.generation.instanceId } : null,
canonExtraction: sanitizeCanonExtraction(raw?.canonExtraction),
descGaps: sanitizeDescGaps(raw?.descGaps),
draftGate: sanitizeDraftGate(raw?.draftGate),
diff --git a/server/services/pipeline/seriesAutopilot.test.js b/server/services/pipeline/seriesAutopilot.test.js
index 81e4aae157..9ab4f3ce9b 100644
--- a/server/services/pipeline/seriesAutopilot.test.js
+++ b/server/services/pipeline/seriesAutopilot.test.js
@@ -20,6 +20,7 @@ vi.mock('crypto', async () => {
return { ...actual, randomUUID: () => `uuid-${++uuidCounter}` };
});
+vi.mock('../instanceIdentity.js', () => ({ getInstanceId: vi.fn(async () => 'test-instance'), UNKNOWN_INSTANCE_ID: 'unknown' }));
vi.mock('../instances.js', () => mockNoPeers());
vi.mock('../sharing/peerSync.js', () => mockNoPeerSync());
diff --git a/server/services/pipeline/textStages.js b/server/services/pipeline/textStages.js
index b0ee93ec1a..6adc47c597 100644
--- a/server/services/pipeline/textStages.js
+++ b/server/services/pipeline/textStages.js
@@ -10,8 +10,7 @@
* styleNotes) plus every *prior* stage's output, so downstream stages can
* reference upstream content with `{{stages.idea.output}}` etc.
*
- * Errors bubble (per project convention — no try/catch) except at the SSE
- * boundary in autoRunner.js, which routes failures through a finalizer.
+ * Generation owns a persist-then-rethrow boundary so every failure is retryable.
*
* Progress (#3393): every phase of a run is broadcast through
* `textStageProgress.js` so the UI shows attempt/judge progress instead of a
@@ -19,12 +18,14 @@
* emit is a no-op and generation is byte-for-byte the un-streamed path.
*/
+import { randomUUID } from 'node:crypto';
+import { getInstanceId, UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';
import { runStagedLLM, resolveStageContext } from '../stageRunner.js';
import { getStage } from '../promptService.js';
import { beginStageProgress, emitStageProgress, finishStageProgress } from './textStageProgress.js';
import { getSeries } from './series.js';
import { extractCanonFromProse, summarizeCanonExtraction } from '../universeCanon.js';
-import { getIssue, listIssues, updateStage, assertStageUnlocked, TEXT_STAGE_IDS } from './issues.js';
+import { getIssue, listIssues, updateStage, updateStageWithLatest, assertStageUnlocked, TEXT_STAGE_IDS } from './issues.js';
import { pickCanon } from './seriesCanon.js';
import { getUniverse } from '../universeBuilder.js';
import { compareIssuesByPosition, NO_LINKED_UNIVERSE_PLACEHOLDER } from './arcPlanner.js';
@@ -36,6 +37,9 @@ import { filterCanonForIssue } from '../../lib/storyBible.js';
import { usableInputTokens, trimContextToBudget, CHARS_PER_TOKEN } from '../../lib/contextBudget.js';
import { escapeRegExp } from '../../lib/textUtils.js';
+// Separate from the advisory progress channel: overlapping runs each stay live.
+const liveGenerations = new Set();
+
const STAGE_TO_TEMPLATE = Object.freeze({
idea: 'pipeline-idea-expansion',
prose: 'pipeline-prose',
@@ -657,43 +661,32 @@ async function buildGenerationContext({ series, canon, world, issue, stageId, op
}
// Run the stage LLM once and persist the response as the active stage state.
-// The try/catch is the mandatory persist-then-rethrow boundary: without it an
-// LLM throw would leave the stage stuck in `generating` forever. Returns
+// The enclosing generation owns failure recovery. Returns
// { issue, stage, runId, output }. Each call with a new runId snapshots the
// prior active state into runHistory (see snapshotRunHistory) — which is how the
// draft gate's rejected attempts stay recoverable for inspection/restore.
-async function runStageLLMOnce(issueId, stageId, template, ctx, options) {
- let result;
- try {
- result = await runStagedLLM(template, ctx, {
- providerOverride: options.providerId,
- // Soft run-level default (Series Autopilot, #1514): unlike providerId it
- // loses to a per-stage pin and soft-falls-through to active when
- // unavailable. Route callers pass providerId (hard); autopilot passes
- // providerIdDefault (soft).
- providerDefault: options.providerIdDefault,
- modelOverride: options.model,
- // Soft run-level model default (Series Autopilot, #1558): loses to a
- // per-stage `model` pin, mirroring providerIdDefault. Route callers pass
- // model (hard); autopilot passes modelIdDefault (soft).
- modelDefault: options.modelIdDefault,
- // Soft run-level reasoning effort (Series Autopilot, #3641): loses to a
- // per-stage `effort` pin, and the runner clamps/drops it per provider.
- effortDefault: options.effortIdDefault,
- source: 'pipeline-text-stage',
- onRunCreated: options.onRunCreated,
- onRunSettled: options.onRunSettled,
- });
- } catch (err) {
- await updateStage(issueId, stageId, {
- status: 'error',
- errorMessage: (err?.message || String(err)).slice(0, 4000),
- });
- throw err;
- }
-
+async function runStageLLMOnce(issueId, stageId, template, ctx, options, persist) {
+ const result = await runStagedLLM(template, ctx, {
+ providerOverride: options.providerId,
+ // Soft run-level default (Series Autopilot, #1514): unlike providerId it
+ // loses to a per-stage pin and soft-falls-through to active when
+ // unavailable. Route callers pass providerId (hard); autopilot passes
+ // providerIdDefault (soft).
+ providerDefault: options.providerIdDefault,
+ modelOverride: options.model,
+ // Soft run-level model default (Series Autopilot, #1558): loses to a
+ // per-stage `model` pin, mirroring providerIdDefault. Route callers pass
+ // model (hard); autopilot passes modelIdDefault (soft).
+ modelDefault: options.modelIdDefault,
+ // Soft run-level reasoning effort (Series Autopilot, #3641): loses to a
+ // per-stage `effort` pin, and the runner clamps/drops it per provider.
+ effortDefault: options.effortIdDefault,
+ source: 'pipeline-text-stage',
+ onRunCreated: options.onRunCreated,
+ onRunSettled: options.onRunSettled,
+ });
const output = (result.content || '').trim();
- const { issue, stage } = await updateStage(issueId, stageId, {
+ const { issue, stage } = await persist({
status: output ? 'ready' : 'error',
output,
lastRunId: result.runId,
@@ -716,7 +709,7 @@ async function runStageLLMOnce(issueId, stageId, template, ctx, options) {
* budget exhausted) and keep the best attempt so far. Route callers pass no
* chargeAction, so a manual regenerate is never cos-billed.
*/
-async function runDraftGate({ issueId, stageId, template, ctx, options, gate, emit = () => {} }) {
+async function runDraftGate({ issueId, stageId, template, ctx, options, gate, persist, emit = () => {} }) {
// Dynamic import breaks the textStages ↔ pipelineJudge require cycle (the judge
// imports scopeCharactersForIssue/stageContentOf from here) and keeps the judge
// out of the module-eval graph for callers that never gate.
@@ -738,7 +731,9 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, em
}
const started = Date.now();
emit({ type: 'phase', phase: 'generate', label: `Drafting attempt ${i + 1} of ${gate.attempts}`, attempt: i + 1, attempts: gate.attempts });
- const res = await runStageLLMOnce(issueId, stageId, template, ctx, options);
+ const res = await runStageLLMOnce(issueId, stageId, template, ctx, options, persist);
+ // A newer generation owns the record; don't judge or re-roll its draft.
+ if (res.stage.lastRunId !== res.runId) return res;
last = res;
let qualityScore = null;
let overall = null;
@@ -786,7 +781,7 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, em
// 'ready' status. The displaced last attempt is snapshotted back into history.
if (best && last && best.runId !== last.runId) {
emit({ type: 'phase', phase: 'restore', label: 'Restoring the best-scoring draft', attempt: attempts.length, attempts: gate.attempts });
- const restored = await updateStage(issueId, stageId, {
+ const restored = await persist({
status: best.output ? 'ready' : 'error',
input: best.input,
output: best.output,
@@ -815,7 +810,7 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, em
rejected: a.runId !== winnerRunId,
})),
};
- const stamped = await updateStage(issueId, stageId, { draftGate }).catch((err) => {
+ const stamped = await persist({ draftGate }).catch((err) => {
console.warn(`⚠️ Failed to record draft-gate scorecard for issue ${issueId.slice(0, 8)}: ${err.message}`);
return null;
});
@@ -839,7 +834,7 @@ async function runDraftGate({ issueId, stageId, template, ctx, options, gate, em
// extract usefully. Non-fatal — prose succeeded, and a noisy extract shouldn't
// roll back the user's accepted draft. An orphan series (no universeId) skips
// extraction entirely. Returns the stamped { issue, stage } or null (unchanged).
-async function maybeExtractCanon(issueId, stageId, output, series, options, emit = () => {}) {
+async function maybeExtractCanon(issueId, stageId, output, series, options, persist, emit = () => {}) {
if (stageId !== 'prose' || !output || !series.universeId) return null;
emit({ type: 'phase', phase: 'canon', label: 'Extracting canon from the draft' });
// Canon extraction follows whichever provider/model drove this prose stage —
@@ -864,7 +859,7 @@ async function maybeExtractCanon(issueId, stageId, output, series, options, emit
return summarizeCanonExtraction({ error: err, provider, model });
},
);
- return updateStage(issueId, 'prose', { canonExtraction: marker }).catch((err) => {
+ return persist({ canonExtraction: marker }).catch((err) => {
console.warn(`⚠️ Failed to record canon-extraction status for issue ${issueId.slice(0, 8)}: ${err.message}`);
return null;
});
@@ -890,6 +885,8 @@ export async function generateStage(issueId, stageId, options = {}) {
throw new Error(`generateStage: unsupported stageId "${stageId}"`);
}
// Generation may overlap, but only the first run owns this stream.
+ const generation = { id: randomUUID(), instanceId: await getInstanceId() };
+ liveGenerations.add(generation.id);
const ownsProgress = beginStageProgress(issueId, stageId);
const emit = (payload) => {
if (ownsProgress) emitStageProgress(issueId, stageId, payload);
@@ -897,8 +894,9 @@ export async function generateStage(issueId, stageId, options = {}) {
emit({ type: 'start', issueId, stageId, at: new Date().toISOString() });
// Settle to an outcome object rather than try/catch so the terminal frame
// ships on BOTH paths before the error resumes bubbling to the caller.
- const outcome = await runTextStage(issueId, stageId, options, emit)
+ const outcome = await runTextStage(issueId, stageId, options, emit, generation)
.then((result) => ({ result }), (error) => ({ error }));
+ liveGenerations.delete(generation.id);
if (outcome.error) {
if (ownsProgress) finishStageProgress(issueId, stageId, {
type: 'error',
@@ -922,8 +920,8 @@ export async function generateStage(issueId, stageId, options = {}) {
// The generation itself — everything between the `start` and terminal progress
// frames. Split out of generateStage so the frames wrap the whole run (context
-// build included) without a try/catch around it.
-async function runTextStage(issueId, stageId, options, emit) {
+// build included). Persistence is guarded by the generation identity.
+async function runTextStage(issueId, stageId, options, emit, generation) {
const template = STAGE_TO_TEMPLATE[stageId];
const issue = await getIssue(issueId);
const series = await getSeries(issue.seriesId);
@@ -941,29 +939,65 @@ async function runTextStage(issueId, stageId, options, emit) {
// three rejects.
assertStageUnlocked(issue, stageId);
- await updateStage(issueId, stageId, { status: 'generating', errorMessage: '' });
-
- emit({ type: 'phase', phase: 'context', label: 'Building the prompt context' });
- const ctx = await buildGenerationContext({ series, canon, world, issue, stageId, options });
- const gate = resolveDraftGate(stageId, template, options);
-
- // Draft gate (opt-in, default off) vs the single-shot path. The single-shot
- // path is byte-for-byte the pre-#2169 behavior so a stage with draftAttempts=1
- // (the default) is unchanged.
- let result;
- if (gate.attempts > 1) {
- result = await runDraftGate({ issueId, stageId, template, ctx, options, gate, emit });
- } else {
- emit({ type: 'phase', phase: 'generate', label: 'Drafting', attempt: 1, attempts: 1 });
- result = await runStageLLMOnce(issueId, stageId, template, ctx, options);
- }
+ await updateStage(issueId, stageId, { status: 'generating', errorMessage: '', generation });
+ const persist = (patch) => updateStageWithLatest(issueId, stageId, (current) =>
+ current.generation?.id === generation.id
+ && current.generation?.instanceId === generation.instanceId ? patch : {});
+
+ try {
+ emit({ type: 'phase', phase: 'context', label: 'Building the prompt context' });
+ const ctx = await buildGenerationContext({ series, canon, world, issue, stageId, options });
+ const gate = resolveDraftGate(stageId, template, options);
+
+ // Draft gate (opt-in, default off) vs the single-shot path. The single-shot
+ // path is byte-for-byte the pre-#2169 behavior so a stage with draftAttempts=1
+ // (the default) is unchanged.
+ let result;
+ if (gate.attempts > 1) {
+ result = await runDraftGate({ issueId, stageId, template, ctx, options, gate, persist, emit });
+ } else {
+ emit({ type: 'phase', phase: 'generate', label: 'Drafting', attempt: 1, attempts: 1 });
+ result = await runStageLLMOnce(issueId, stageId, template, ctx, options, persist);
+ }
+
+ let { issue: updatedIssue, stage } = result;
+ // Prose canon extraction on the FINAL (winning) output — once, not per attempt.
+ const current = await getIssue(issueId);
+ if (current.stages[stageId].generation?.id !== generation.id) return { ...result, issue: current, stage: current.stages[stageId] };
+ const stamped = await maybeExtractCanon(issueId, stageId, result.output, series, options, persist, emit);
+ if (stamped) ({ issue: updatedIssue, stage } = stamped);
- let { issue: updatedIssue, stage } = result;
- // Prose canon extraction on the FINAL (winning) output — once, not per attempt.
- const stamped = await maybeExtractCanon(issueId, stageId, result.output, series, options, emit);
- if (stamped) ({ issue: updatedIssue, stage } = stamped);
+ return { issue: updatedIssue, stage, runId: result.runId };
+ } catch (error) {
+ await persist({
+ status: 'error',
+ errorMessage: (error?.message || String(error)).slice(0, 4000),
+ }).catch((persistError) => {
+ console.error(`❌ Pipeline stage error recovery failed: ${persistError.message}`);
+ });
+ throw error;
+ }
+}
- return { issue: updatedIssue, stage, runId: result.runId };
+/** Only this install's abandoned generations may be demoted; legacy ownership is unknown. */
+export async function recoverStuckGeneratingStages() {
+ const instanceId = await getInstanceId();
+ if (!instanceId || instanceId === UNKNOWN_INSTANCE_ID) return 0;
+ let recovered = 0;
+ for (const issue of await listIssues({ withHistory: false })) {
+ for (const stageId of TEXT_STAGE_IDS) {
+ const snapshot = issue.stages[stageId];
+ if (snapshot.status !== 'generating' || snapshot.generation?.instanceId !== instanceId) continue;
+ await updateStageWithLatest(issue.id, stageId, (current) => {
+ if (current.status !== 'generating' || current.generation?.instanceId !== instanceId
+ || current.generation?.id !== snapshot.generation.id
+ || current.updatedAt !== snapshot.updatedAt || liveGenerations.has(current.generation.id)) return {};
+ recovered += 1;
+ return { status: 'error', errorMessage: 'Interrupted by a server restart — generate again.' };
+ });
+ }
+ }
+ return recovered;
}
// Export internals for tests.
diff --git a/server/services/pipeline/textStages.test.js b/server/services/pipeline/textStages.test.js
index dde4d575b7..a6dd826491 100644
--- a/server/services/pipeline/textStages.test.js
+++ b/server/services/pipeline/textStages.test.js
@@ -18,6 +18,7 @@ tryReadFile: vi.fn().mockResolvedValue(null),
ensureDir: vi.fn().mockResolvedValue(undefined),
atomicWrite: vi.fn(async (path, data) => { fileStore.set(path, data); }),
readJSONFile: vi.fn(async (path, fallback) => (fileStore.has(path) ? fileStore.get(path) : fallback)),
+ readJSONFileStrict: vi.fn(async (path, fallback) => ({ ok: true, value: fileStore.has(path) ? fileStore.get(path) : fallback })),
}));
let uuidCounter = 0;
@@ -26,6 +27,7 @@ vi.mock('crypto', async () => {
return { ...actual, randomUUID: () => `uuid-${++uuidCounter}` };
});
+vi.mock('../instanceIdentity.js', () => ({ getInstanceId: vi.fn(async () => 'test-instance'), UNKNOWN_INSTANCE_ID: 'unknown' }));
vi.mock('../instances.js', () => mockNoPeers());
vi.mock('../sharing/peerSync.js', () => mockNoPeerSync());
@@ -193,6 +195,54 @@ describe('pipeline text stage generator', () => {
expect(after.stages.idea.errorMessage).toContain('simulated provider 500');
});
+ it('recovers context failures after the generating write and preserves the prior draft', async () => {
+ const { issue } = await seed();
+ await issuesSvc.updateStage(issue.id, 'idea', { output: 'Prior draft', lastRunId: 'prior-run' });
+ const error = new Error('Context could not be built');
+ const options = { get seedInput() { throw error; } };
+ await expect(textStages.generateStage(issue.id, 'idea', options)).rejects.toBe(error);
+ expect((await issuesSvc.getIssue(issue.id)).stages.idea).toMatchObject({
+ status: 'error', errorMessage: error.message, output: 'Prior draft', lastRunId: 'prior-run', runHistory: [],
+ });
+ });
+
+ it('rethrows the original context error when persisting recovery also fails', async () => {
+ const { issue } = await seed();
+ const { atomicWrite } = await import('../../lib/fileUtils.js');
+ const error = new Error('Original context failure');
+ const options = { get seedInput() {
+ atomicWrite.mockRejectedValueOnce(new Error('Recovery storage failure'));
+ throw error;
+ } };
+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});
+ await expect(textStages.generateStage(issue.id, 'idea', options)).rejects.toBe(error);
+ expect(log).toHaveBeenCalledWith(expect.stringContaining('Recovery storage failure'));
+ log.mockRestore();
+ });
+
+ it('recovers only abandoned local generations, preserving foreign and legacy work across sync', async () => {
+ const { series, issue } = await seed();
+ const foreign = await issuesSvc.createIssue({ seriesId: series.id, title: 'Peer work' });
+ const legacy = await issuesSvc.createIssue({ seriesId: series.id, title: 'Unknown work' });
+ await issuesSvc.updateStage(issue.id, 'idea', {
+ status: 'generating', output: 'Prior draft', generation: { id: 'interrupted', instanceId: 'test-instance' },
+ });
+ await issuesSvc.mergeIssuesFromSync([{ ...foreign, updatedAt: '2099-01-01T00:00:00.000Z',
+ stages: { ...foreign.stages, prose: { status: 'generating', generation: { id: 'peer-run', instanceId: 'peer-instance' } } },
+ }], { senderSchemaVersions: { pipelineIssues: 4 } });
+ await issuesSvc.updateStage(legacy.id, 'prose', { status: 'generating' });
+ expect(await textStages.recoverStuckGeneratingStages()).toBe(1);
+ expect((await issuesSvc.getIssue(issue.id)).stages.idea).toMatchObject({
+ status: 'error', output: 'Prior draft', errorMessage: 'Interrupted by a server restart — generate again.',
+ });
+ expect((await issuesSvc.getIssue(foreign.id)).stages.prose).toMatchObject({
+ status: 'generating', generation: { id: 'peer-run', instanceId: 'peer-instance' },
+ });
+ expect((await issuesSvc.getIssue(legacy.id)).stages.prose.status).toBe('generating');
+ expect(await textStages.recoverStuckGeneratingStages()).toBe(0);
+ expect((await textStages.generateStage(issue.id, 'idea')).stage.status).toBe('ready');
+ });
+
it('rejects unsupported stage ids', async () => {
const { issue } = await seed();
await expect(textStages.generateStage(issue.id, 'comicPages')).rejects.toThrow(/unsupported stageId/);
@@ -1337,6 +1387,7 @@ describe('text-stage generation progress frames (#3393)', () => {
await firstAtProvider;
const second = textStages.generateStage(issue.id, 'prose', {});
await secondAtProvider;
+ expect(await textStages.recoverStuckGeneratingStages()).toBe(0);
finishSecond();
await second;
expect(framesOf(res).some((frame) => frame.type === 'complete')).toBe(false);
@@ -1345,9 +1396,44 @@ describe('text-stage generation progress frames (#3393)', () => {
finishFirst();
await first;
+ expect((await issuesSvc.getIssue(issue.id)).stages.prose.output).toBe('Second draft');
expect(framesOf(res).filter((frame) => frame.type === 'complete')).toHaveLength(1);
});
+ it('a late provider failure cannot demote a newer live generation', async () => {
+ const { issue } = await seedProse();
+ let startedFirst;
+ let startedSecond;
+ let failFirst;
+ let finishSecond;
+ const firstAtProvider = new Promise((resolve) => { startedFirst = resolve; });
+ const secondAtProvider = new Promise((resolve) => { startedSecond = resolve; });
+ const firstCanFail = new Promise((resolve) => { failFirst = resolve; });
+ const secondCanFinish = new Promise((resolve) => { finishSecond = resolve; });
+ runnerSvc.executeApiRun.mockImplementationOnce(async ({ onComplete }) => {
+ startedFirst();
+ await firstCanFail;
+ onComplete({ success: false, error: 'Old run failed' });
+ }).mockImplementationOnce(async ({ onData, onComplete }) => {
+ startedSecond();
+ await secondCanFinish;
+ onData('New draft');
+ onComplete({ success: true });
+ });
+ const first = textStages.generateStage(issue.id, 'prose');
+ const firstFailed = expect(first).rejects.toThrow('Old run failed');
+ await firstAtProvider;
+ const second = textStages.generateStage(issue.id, 'prose');
+ await secondAtProvider;
+ failFirst();
+ await firstFailed;
+ expect((await issuesSvc.getIssue(issue.id)).stages.prose.status).toBe('generating');
+ expect(await textStages.recoverStuckGeneratingStages()).toBe(0);
+ finishSecond();
+ await second;
+ expect((await issuesSvc.getIssue(issue.id)).stages.prose).toMatchObject({ status: 'ready', output: 'New draft' });
+ });
+
it('streams start → context → generate → complete on the single-shot path', async () => {
const { issue } = await seedProse();
const res = fakeRes();
diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.js
index 987b049d0b..5529f7660e 100644
--- a/server/services/sharing/peerSync.test.js
+++ b/server/services/sharing/peerSync.test.js
@@ -4152,7 +4152,7 @@ describe('peerSync', () => {
record: { id: 'ser-1', name: 'Foo' },
assetManifest: [],
sourceInstanceId: 'peer-a',
- portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 4 } },
+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 5 } },
});
expect(mergeSeriesFromSync).toHaveBeenCalled();
expect(mergeIssuesFromSync).not.toHaveBeenCalled();
@@ -4167,10 +4167,10 @@ describe('peerSync', () => {
issues: [{ id: 'i1', seriesId: 'ser-1', deleted: false, deletedAt: null }],
assetManifest: [],
sourceInstanceId: 'peer-a',
- portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 4 } },
+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 5 } },
}).catch((err) => err);
expect(rejection.code).toBe('PEER_SYNC_SCHEMA_VERSION_AHEAD');
- expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 4, receiverV: 3 }]);
+ expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 5, receiverV: 4 }]);
expect(mergeSeriesFromSync).not.toHaveBeenCalled();
expect(mergeIssuesFromSync).not.toHaveBeenCalled();
});
@@ -4265,10 +4265,10 @@ describe('peerSync', () => {
issues: [{ id: 'i1', seriesId: 'ser-1', deleted: false, deletedAt: null }],
assetManifest: [],
sourceInstanceId: 'peer-a',
- portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 4 } },
+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, pipelineSeries: 1, pipelineIssues: 5 } },
}).catch((err) => err);
expect(rejection.code).toBe('PEER_SYNC_SCHEMA_VERSION_AHEAD');
- expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 4, receiverV: 3 }]);
+ expect(rejection.details.ahead).toEqual([{ category: 'pipelineIssues', senderV: 5, receiverV: 4 }]);
expect(mergeSeriesFromSync).not.toHaveBeenCalled();
expect(mergeIssuesFromSync).not.toHaveBeenCalled();
});