Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Recover legacy pipeline text stages explicitly (#11730)

[._.]/513cc6c1c74d
diff --git a/client/src/components/pipeline/stages/TextStagePanel.jsx b/client/src/components/pipeline/stages/TextStagePanel.jsx
index fa66db77b5..edd41061e1 100644
--- a/client/src/components/pipeline/stages/TextStagePanel.jsx
+++ b/client/src/components/pipeline/stages/TextStagePanel.jsx
@@ -11,7 +11,7 @@ import toast from '../../ui/Toast';
import InlineDiff from '../../ui/InlineDiff';
import ProseEditor from '../../ui/ProseEditor';
import {
- generatePipelineStage, updatePipelineIssue,
+ generatePipelineStage, updatePipelineIssue, getPipelineStageRecovery, recoverPipelineStage,
pipelineStageProgressUrl,
PIPELINE_STAGE_LABELS,
PIPELINE_TEXT_STAGES,
@@ -108,6 +108,37 @@ export default function TextStagePanel({
const activeKeyRef = useRef(progressKey);
useEffect(() => { activeKeyRef.current = progressKey; }, [progressKey]);
+ const recoveryTargetRef = useRef(null);
+ useEffect(() => {
+ recoveryTargetRef.current = { key: progressKey, stage };
+ return () => { recoveryTargetRef.current = null; };
+ }, [progressKey, stage]);
+ const [recovery, setRecovery] = useState(null);
+ useEffect(() => {
+ let active = true;
+ if (stage.status === 'generating') {
+ getPipelineStageRecovery(issue.id, stageId, { silent: true }).then((data) => {
+ if (active) setRecovery({ key: progressKey, stage, data });
+ }).catch(() => { if (active) setRecovery(null); });
+ }
+ return () => { active = false; };
+ }, [issue.id, stageId, stage, progressKey]);
+ const recoveryInfo = recovery?.key === progressKey && recovery.stage === stage ? recovery.data : null;
+ const [runRecover, recovering] = useAsyncAction(
+ (requestTarget) => recoverPipelineStage(issue.id, stageId, recoveryInfo.revision, { silent: true }).catch((error) => {
+ if (recoveryTargetRef.current !== requestTarget) return null;
+ throw error;
+ }),
+ { errorMessage: 'Recovery failed' },
+ );
+ const handleRecover = async () => {
+ const requestTarget = recoveryTargetRef.current;
+ const result = await runRecover(requestTarget);
+ if (!result || recoveryTargetRef.current !== requestTarget) return;
+ onStageUpdate?.(stageId, result.stage, result.issue);
+ toast.success('Stage recovered. You can generate again; editorial locks still apply.');
+ };
+
// Other text stages that currently have content — the candidate source
// material for this generation. Excludes the target stage itself. Lets you
// generate any stage FROM any other populated stage (backport), e.g. prose
@@ -288,7 +319,7 @@ export default function TextStagePanel({
<button
type="button"
onClick={handleSave}
- disabled={!dirty || saving || !!activeDiff}
+ disabled={!dirty || saving || recovering || !!activeDiff}
title={activeDiff ? 'Resolve the new-version comparison first' : undefined}
className="inline-flex items-center gap-2 px-3 py-1.5 rounded-lg bg-port-card border border-port-border text-white text-sm hover:border-port-accent/50 disabled:opacity-40"
>
@@ -300,7 +331,7 @@ export default function TextStagePanel({
onClick={outputDirty ? () => setConfirmRegenerate(true) : handleGenerate}
// `saving` too: an in-flight save PATCH writes the same stage record
// this run will overwrite when it lands.
- disabled={generating || actionsGated || saving || !!activeDiff}
+ disabled={generating || actionsGated || saving || recovering || !!activeDiff}
title={actionsGated
? 'Saving settings…'
: (outputDirty ? 'You have unsaved edits — you’ll compare them against the new version first' : undefined)}
@@ -312,6 +343,29 @@ export default function TextStagePanel({
</div>
</div>
+ {stage.status === 'generating' && recoveryInfo ? (
+ <div className="border border-port-warning/40 rounded p-3 space-y-2 text-sm">
+ <p>{recoveryInfo.live
+ ? 'A generation is active on this machine.'
+ : recoveryInfo.ownership === 'foreign'
+ ? 'This generation belongs to another machine. Its current progress is not known here.'
+ : recoveryInfo.ownership === 'unknown'
+ ? 'Execution ownership is unknown. This work may still be running on another machine. Taking over allows a fresh local generation and preserves your draft and history.'
+ : 'This generation belongs to this machine.'}</p>
+ {recoveryInfo.recoverable && !localGenerating ? (
+ <button
+ type="button"
+ onClick={handleRecover}
+ disabled={recovering || saving || dirty || actionsGated}
+ title={dirty ? 'Save your edits before taking over.' : undefined}
+ className="px-3 py-1.5 rounded-lg bg-port-card border border-port-border disabled:opacity-50"
+ >
+ {recovering ? 'Taking over…' : 'Take over stage'}
+ </button>
+ ) : null}
+ </div>
+ ) : null}
+
{confirmRegenerate && outputDirty ? (
<div className="flex items-center justify-between gap-3 flex-wrap border border-port-warning/40 rounded p-3 bg-port-warning/5">
<span className="text-xs text-gray-300">
@@ -323,7 +377,7 @@ export default function TextStagePanel({
onClick={handleGenerate}
// Same gates as the Generate button it stands in for — the confirm
// row can sit open while a save or a settings write starts.
- disabled={generating || actionsGated || saving}
+ disabled={generating || actionsGated || saving || recovering}
className="inline-flex items-center gap-2 px-3 py-1.5 rounded-lg bg-port-accent text-white text-sm font-medium disabled:opacity-50"
>
<GitCompare size={14} />
@@ -453,6 +507,7 @@ export default function TextStagePanel({
<label className="block">
<span className="block text-xs uppercase tracking-wider text-gray-500 mb-1">Seed idea</span>
<textarea
+ disabled={recovering}
value={draftInput}
onChange={(e) => setDraftInput(e.target.value)}
placeholder={seedPlaceholder}
@@ -466,6 +521,7 @@ export default function TextStagePanel({
<div className="block">
<span className="block text-xs uppercase tracking-wider text-gray-500 mb-1">Output</span>
<ProseEditor
+ disabled={recovering}
value={draftOutput}
onChange={(e) => setDraftOutput(e.target.value)}
placeholder={outputPlaceholder}
@@ -479,6 +535,7 @@ export default function TextStagePanel({
<span className="block text-xs uppercase tracking-wider text-gray-500 mb-1">Output</span>
<textarea
id={`pipeline-stage-${stageId}-output`}
+ disabled={recovering}
value={draftOutput}
onChange={(e) => setDraftOutput(e.target.value)}
placeholder={outputPlaceholder}
diff --git a/client/src/components/pipeline/stages/TextStagePanel.test.jsx b/client/src/components/pipeline/stages/TextStagePanel.test.jsx
index 7d4f1e9cd2..5b414d7a6c 100644
--- a/client/src/components/pipeline/stages/TextStagePanel.test.jsx
+++ b/client/src/components/pipeline/stages/TextStagePanel.test.jsx
@@ -5,6 +5,8 @@ import { MockEventSource, lastEventSource } from '../../../test/mockEventSource'
vi.mock('../../../services/api', () => ({
generatePipelineStage: vi.fn(),
+ getPipelineStageRecovery: vi.fn(async () => ({ recoverable: false, ownership: 'local', live: true })),
+ recoverPipelineStage: vi.fn(),
updatePipelineIssue: vi.fn(),
restorePipelineStageVersion: vi.fn(),
pipelineStageProgressUrl: (issueId, stageId) => `/api/pipeline/issues/${issueId}/stages/${stageId}/generate/progress`,
@@ -21,7 +23,7 @@ vi.mock('../../ui/Toast', () => ({
import TextStagePanel from './TextStagePanel';
import toast from '../../ui/Toast';
-import { generatePipelineStage, updatePipelineIssue } from '../../../services/api';
+import { generatePipelineStage, updatePipelineIssue, getPipelineStageRecovery, recoverPipelineStage } from '../../../services/api';
const makeIssue = (ideaOverrides = {}) => ({
id: 'iss-1',
@@ -46,6 +48,137 @@ describe('TextStagePanel', () => {
vi.clearAllMocks();
});
+ it('offers explicit legacy takeover, preserves content and unlocks generation after success', async () => {
+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'legacy-revision', ownership: 'unknown', live: false, recoverable: true });
+ const issue = makeIssue({ status: 'generating', output: 'Preserved draft' });
+ const stage = { ...issue.stages.idea, status: 'error' };
+ recoverPipelineStage.mockResolvedValueOnce({ issue: { ...issue, stages: { idea: stage } }, stage });
+ const onStageUpdate = vi.fn();
+ const { rerender } = renderPanel(issue, { onStageUpdate });
+ expect(screen.getByRole('button', { name: 'Generate' })).toBeDisabled();
+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));
+ expect(screen.getByText(/may still be running on another machine/)).toBeInTheDocument();
+ expect(recoverPipelineStage).toHaveBeenCalledWith('iss-1', 'idea', 'legacy-revision', { silent: true });
+ expect(onStageUpdate).toHaveBeenCalledWith('idea', stage, expect.any(Object));
+ rerender(<TextStagePanel issue={{ ...issue, stages: { idea: stage } }} stageId="idea" />);
+ expect(screen.getByRole('button', { name: 'Generate' })).toBeEnabled();
+ expect(screen.getByLabelText('Output')).toHaveValue('Preserved draft');
+ });
+
+ it.each(['success', 'failure'])('ignores a pending takeover %s after switching issue and stage', async (outcome) => {
+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'revision-a', ownership: 'unknown', live: false, recoverable: true });
+ let resolveRecovery;
+ let rejectRecovery;
+ recoverPipelineStage.mockReturnValueOnce(new Promise((resolve, reject) => {
+ resolveRecovery = resolve;
+ rejectRecovery = reject;
+ }));
+ const issueA = makeIssue({ status: 'generating', output: 'Draft A' });
+ const onStageUpdate = vi.fn();
+ const { rerender } = renderPanel(issueA, { onStageUpdate });
+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));
+ const issueB = { id: 'iss-2', stages: { prose: { status: 'ready', input: '', output: 'Draft B', runHistory: [] } } };
+ rerender(<TextStagePanel issue={issueB} stageId="prose" onStageUpdate={onStageUpdate} />);
+ expect(screen.getByLabelText('Output')).toHaveValue('Draft B');
+ await act(async () => {
+ if (outcome === 'success') {
+ const stage = { ...issueA.stages.idea, status: 'error' };
+ resolveRecovery({ issue: { ...issueA, stages: { idea: stage } }, stage });
+ } else {
+ rejectRecovery(new Error('Stage A changed'));
+ }
+ });
+ expect(onStageUpdate).not.toHaveBeenCalled();
+ expect(toast.success).not.toHaveBeenCalled();
+ expect(toast.error).not.toHaveBeenCalled();
+ expect(screen.getByLabelText('Output')).toHaveValue('Draft B');
+ expect(screen.getByRole('button', { name: 'Generate' })).toBeEnabled();
+ });
+
+ it.each(['success', 'failure'])('ignores takeover %s after leaving and returning to a replaced stage', async (outcome) => {
+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'old-revision', ownership: 'unknown', live: false, recoverable: true });
+ let resolveRecovery;
+ let rejectRecovery;
+ recoverPipelineStage.mockReturnValueOnce(new Promise((resolve, reject) => {
+ resolveRecovery = resolve;
+ rejectRecovery = reject;
+ }));
+ const issueA = makeIssue({ status: 'generating', output: 'Old draft A' });
+ const onStageUpdate = vi.fn();
+ const { rerender } = renderPanel(issueA, { onStageUpdate });
+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));
+ rerender(<TextStagePanel issue={{ id: 'iss-2', stages: { idea: { status: 'ready', output: 'Draft B' } } }} stageId="idea" onStageUpdate={onStageUpdate} />);
+ const updatedIssueA = makeIssue({ status: 'ready', output: 'New draft A from another action' });
+ rerender(<TextStagePanel issue={updatedIssueA} stageId="idea" onStageUpdate={onStageUpdate} />);
+ await act(async () => {
+ if (outcome === 'success') {
+ const stage = { ...issueA.stages.idea, status: 'error' };
+ resolveRecovery({ issue: { ...issueA, stages: { idea: stage } }, stage });
+ } else {
+ rejectRecovery(new Error('Old takeover failed'));
+ }
+ });
+ expect(onStageUpdate).not.toHaveBeenCalled();
+ expect(toast.success).not.toHaveBeenCalled();
+ expect(toast.error).not.toHaveBeenCalled();
+ expect(screen.getByLabelText('Output')).toHaveValue('New draft A from another action');
+ expect(screen.getByRole('button', { name: 'Generate' })).toBeEnabled();
+ });
+
+ it.each(['success', 'failure'])('ignores takeover %s after the panel unmounts', async (outcome) => {
+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'old-revision', ownership: 'unknown', live: false, recoverable: true });
+ let settle;
+ recoverPipelineStage.mockReturnValueOnce(new Promise((resolve, reject) => {
+ settle = () => outcome === 'success' ? resolve({ stage: { status: 'error' } }) : reject(new Error('Old takeover failed'));
+ }));
+ const onStageUpdate = vi.fn();
+ const { unmount } = renderPanel(makeIssue({ status: 'generating' }), { onStageUpdate });
+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));
+ unmount();
+ await act(async () => settle());
+ expect(onStageUpdate).not.toHaveBeenCalled();
+ expect(toast.success).not.toHaveBeenCalled();
+ expect(toast.error).not.toHaveBeenCalled();
+ });
+
+ it('disables takeover while preserving unsaved seed and output edits', async () => {
+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'legacy-revision', ownership: 'unknown', live: false, recoverable: true });
+ renderPanel(makeIssue({ status: 'generating', input: 'Saved seed', output: 'Saved draft' }));
+ const takeover = await screen.findByRole('button', { name: 'Take over stage' });
+ expect(takeover).toBeEnabled();
+ await userEvent.type(screen.getByPlaceholderText('seed…'), ' with unsaved detail');
+ await userEvent.type(screen.getByLabelText('Output'), '\nUnsaved second paragraph.');
+ expect(takeover).toBeDisabled();
+ await userEvent.click(takeover);
+ expect(recoverPipelineStage).not.toHaveBeenCalled();
+ expect(screen.getByPlaceholderText('seed…')).toHaveValue('Saved seed with unsaved detail');
+ expect(screen.getByLabelText('Output')).toHaveValue('Saved draft\nUnsaved second paragraph.');
+ });
+
+ it.each(['A local generation is active.', 'The stage changed. Refresh it before taking over.'])(
+ 'keeps generation blocked and content intact when takeover is refused: %s', async (message) => {
+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'shown-revision', ownership: 'unknown', live: false, recoverable: true });
+ recoverPipelineStage.mockRejectedValueOnce(new Error(message));
+ const onStageUpdate = vi.fn();
+ renderPanel(makeIssue({ status: 'generating', output: 'Keep this draft' }), { onStageUpdate });
+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));
+ await waitFor(() => expect(toast.error).toHaveBeenCalledWith(message));
+ expect(onStageUpdate).not.toHaveBeenCalled();
+ expect(screen.getByRole('button', { name: 'Generate' })).toBeDisabled();
+ expect(screen.getByLabelText('Output')).toHaveValue('Keep this draft');
+ },
+ );
+
+ it.each([
+ { ownership: 'foreign', live: false, text: /belongs to another machine/ },
+ { ownership: 'unknown', live: true, text: /active on this machine/ },
+ ])('does not offer recovery for $ownership ownership with live=$live', async ({ ownership, live, text }) => {
+ getPipelineStageRecovery.mockResolvedValueOnce({ ownership, live, recoverable: false });
+ renderPanel(makeIssue({ status: 'generating' }));
+ expect(await screen.findByText(text)).toBeInTheDocument();
+ expect(screen.queryByRole('button', { name: 'Take over stage' })).not.toBeInTheDocument();
+ });
+
it('Generate success: calls the generate API with the right shape and reflects the new content once the parent lifts it', async () => {
const stage = { status: 'ready', input: '', output: 'Generated idea text.', runHistory: [] };
generatePipelineStage.mockResolvedValue({ stage });
@@ -122,9 +255,10 @@ describe('TextStagePanel', () => {
expect(screen.getByRole('button', { name: /Save edits/i })).toBeDisabled();
});
- it('resume-on-mount: mounting with server-side "generating" status keeps Generate disabled without starting a fresh run', () => {
+ it('resume-on-mount: mounting with server-side "generating" status keeps Generate disabled without starting a fresh run', async () => {
const issue = makeIssue({ status: 'generating' });
renderPanel(issue);
+ await screen.findByText('A generation is active on this machine.');
expect(screen.getByRole('button', { name: 'Generate' })).toBeDisabled();
expect(generatePipelineStage).not.toHaveBeenCalled();
diff --git a/client/src/services/apiPipeline.js b/client/src/services/apiPipeline.js
index f52fba6076..dd34ad3a92 100644
--- a/client/src/services/apiPipeline.js
+++ b/client/src/services/apiPipeline.js
@@ -143,6 +143,13 @@ export const deletePipelineIssue = (id, options = {}) =>
request(`/pipeline/issues/${encodeURIComponent(id)}`, { method: 'DELETE', ...options });
// ---- Stage operations ----
+export const getPipelineStageRecovery = (issueId, stageId, options = {}) =>
+ request(`/pipeline/issues/${encodeURIComponent(issueId)}/stages/${encodeURIComponent(stageId)}/recovery`, options);
+export const recoverPipelineStage = (issueId, stageId, revision, options = {}) =>
+ request(`/pipeline/issues/${encodeURIComponent(issueId)}/stages/${encodeURIComponent(stageId)}/recover`, {
+ method: 'POST', body: JSON.stringify({ revision }), ...options,
+ });
+
export const generatePipelineStage = (issueId, stageId, opts = {}, options = {}) =>
request(`/pipeline/issues/${encodeURIComponent(issueId)}/stages/${encodeURIComponent(stageId)}/generate`, {
method: 'POST',
diff --git a/server/lib/hostControlRoutes.test.js b/server/lib/hostControlRoutes.test.js
index d1dc7ebbd8..38c5200714 100644
--- a/server/lib/hostControlRoutes.test.js
+++ b/server/lib/hostControlRoutes.test.js
@@ -520,6 +520,8 @@ describe('Pipeline authoring remainder policy (#10907)', () => {
'PATCH /api/pipeline/tts/narrate/segment',
'POST /api/pipeline/editorial/custom-checks',
'POST /api/pipeline/issues/:id/auto-run-text/cancel',
+ // Recovery only resets persisted state; generating again uses the gated route.
+ 'POST /api/pipeline/issues/:id/stages/:stageId/recover',
'POST /api/pipeline/issues/:id/stages/:stageId/restore',
'POST /api/pipeline/issues/:id/stages/audio/cues/:cueIdx/render',
'POST /api/pipeline/issues/:id/stages/audio/extract-lines',
diff --git a/server/lib/pipelineValidation.js b/server/lib/pipelineValidation.js
index 7cc2d7eeb8..c7e2336b02 100644
--- a/server/lib/pipelineValidation.js
+++ b/server/lib/pipelineValidation.js
@@ -32,6 +32,8 @@ import { CHECK_SCOPES, CHECK_SEVERITIES } from './editorial/checkInfra/taxonomy.
import { SHOT_TYPES, SCREEN_DIRECTIONS } from './shotGrammar.js';
import { characterEvolutionSchema } from './characterEvolutionValidation.js';
+export const pipelineStageRecoverySchema = z.object({ revision: z.string().regex(/^[a-f0-9]{64}$/) });
+
// =============================================================================
// WRITERS ROOM SCHEMAS
// =============================================================================
diff --git a/server/routes/pipeline/issues.js b/server/routes/pipeline/issues.js
index aafcad991e..ad33435ff6 100644
--- a/server/routes/pipeline/issues.js
+++ b/server/routes/pipeline/issues.js
@@ -51,6 +51,8 @@ import {
import { ARC_ROLES } from '../../lib/storyArc.js';
import { mapServiceError, countExtractedCanon } from './shared.js';
+import textStageRecoveryRouter from './textStageRecovery.js';
+
const router = Router();
// ---- Issue schemas ----
@@ -432,6 +434,8 @@ router.delete('/issues/:id', asyncHandler(async (req, res) => {
// Stage operations
// =====================
+router.use(textStageRecoveryRouter);
+
router.post('/issues/:id/stages/:stageId/generate', asyncHandler(async (req, res) => {
const { id, stageId } = req.params;
if (!issuesSvc.TEXT_STAGE_IDS.includes(stageId)) {
diff --git a/server/routes/pipeline/textStageRecovery.js b/server/routes/pipeline/textStageRecovery.js
new file mode 100644
index 0000000000..1acbd66b46
--- /dev/null
+++ b/server/routes/pipeline/textStageRecovery.js
@@ -0,0 +1,23 @@
+import { Router } from 'express';
+import { z } from 'zod';
+import { asyncHandler, createServiceErrorMapper } from '../../lib/errorHandler.js';
+import { validateRequest } from '../../lib/validation.js';
+import { pipelineStageRecoverySchema } from '../../lib/pipelineValidation.js';
+import { TEXT_STAGE_IDS, ERR_NOT_FOUND } from '../../services/pipeline/issuesShared.js';
+import { getStageRecovery, recoverLegacyStage } from '../../services/pipeline/textStages.js';
+
+const router = Router();
+const paramsSchema = z.object({ id: z.string().min(1), stageId: z.enum(TEXT_STAGE_IDS) });
+const mapError = createServiceErrorMapper({ [ERR_NOT_FOUND]: 404, PIPELINE_RECOVERY_CONFLICT: 409 });
+
+router.get('/issues/:id/stages/:stageId/recovery', asyncHandler(async (req, res) => {
+ const { id, stageId } = validateRequest(paramsSchema, req.params);
+ res.json(await getStageRecovery(id, stageId).catch((err) => { throw mapError(err); }));
+}));
+router.post('/issues/:id/stages/:stageId/recover', asyncHandler(async (req, res) => {
+ const { id, stageId } = validateRequest(paramsSchema, req.params);
+ const { revision } = validateRequest(pipelineStageRecoverySchema, req.body ?? {});
+ res.json(await recoverLegacyStage(id, stageId, revision).catch((err) => { throw mapError(err); }));
+}));
+
+export default router;
diff --git a/server/services/pipeline/issueStages.js b/server/services/pipeline/issueStages.js
index c172ab8cfd..1d6506f00a 100644
--- a/server/services/pipeline/issueStages.js
+++ b/server/services/pipeline/issueStages.js
@@ -8,6 +8,7 @@
* renumber/bulk write that rewrites the same issue — one mutex per series.
*/
+import { createHash } from 'node:crypto';
import { isPlainObject } from '../../lib/objects.js';
import { pickRenderedFilename } from '../../lib/renderSlot.js';
import { emitRecordUpdated } from '../sharing/recordEvents.js';
@@ -22,6 +23,11 @@ import {
import { isStr } from '../../lib/textUtils.js';
import { codedError } from '../../lib/codedError.js';
+// Hash the complete sanitized stage, including content and history: timestamps alone
+// miss same-tick edits and issue-level PATCHes that retain the stage timestamp.
+export const textStageRevision = (stage) => createHash('sha256')
+ .update(JSON.stringify(sanitizeTextStage(stage))).digest('hex');
+
/**
* Partial update to a single stage on an issue. Use this from generators so
* a stage write doesn't have to load the full issue, mutate, and re-validate.
diff --git a/server/services/pipeline/textStages.js b/server/services/pipeline/textStages.js
index 6adc47c597..0a94db2ccb 100644
--- a/server/services/pipeline/textStages.js
+++ b/server/services/pipeline/textStages.js
@@ -18,6 +18,7 @@
* emit is a no-op and generation is byte-for-byte the un-streamed path.
*/
+import { codedError } from '../../lib/codedError.js';
import { randomUUID } from 'node:crypto';
import { getInstanceId, UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';
import { runStagedLLM, resolveStageContext } from '../stageRunner.js';
@@ -25,7 +26,7 @@ import { getStage } from '../promptService.js';
import { beginStageProgress, emitStageProgress, finishStageProgress } from './textStageProgress.js';
import { getSeries } from './series.js';
import { extractCanonFromProse, summarizeCanonExtraction } from '../universeCanon.js';
-import { getIssue, listIssues, updateStage, updateStageWithLatest, assertStageUnlocked, TEXT_STAGE_IDS } from './issues.js';
+import { getIssue, listIssues, updateStage, updateStageWithLatest, assertStageUnlocked, textStageRevision, TEXT_STAGE_IDS } from './issues.js';
import { pickCanon } from './seriesCanon.js';
import { getUniverse } from '../universeBuilder.js';
import { compareIssuesByPosition, NO_LINKED_UNIVERSE_PLACEHOLDER } from './arcPlanner.js';
@@ -38,7 +39,9 @@ import { usableInputTokens, trimContextToBudget, CHARS_PER_TOKEN } from '../../l
import { escapeRegExp } from '../../lib/textUtils.js';
// Separate from the advisory progress channel: overlapping runs each stay live.
-const liveGenerations = new Set();
+const liveGenerations = new Map();
+const hasLiveGeneration = (issueId, stageId) => [...liveGenerations.values()]
+ .some((run) => run.issueId === issueId && run.stageId === stageId);
const STAGE_TO_TEMPLATE = Object.freeze({
idea: 'pipeline-idea-expansion',
@@ -885,8 +888,8 @@ export async function generateStage(issueId, stageId, options = {}) {
throw new Error(`generateStage: unsupported stageId "${stageId}"`);
}
// Generation may overlap, but only the first run owns this stream.
- const generation = { id: randomUUID(), instanceId: await getInstanceId() };
- liveGenerations.add(generation.id);
+ const generation = { id: randomUUID(), instanceId: null };
+ liveGenerations.set(generation.id, { issueId, stageId });
const ownsProgress = beginStageProgress(issueId, stageId);
const emit = (payload) => {
if (ownsProgress) emitStageProgress(issueId, stageId, payload);
@@ -894,7 +897,10 @@ export async function generateStage(issueId, stageId, options = {}) {
emit({ type: 'start', issueId, stageId, at: new Date().toISOString() });
// Settle to an outcome object rather than try/catch so the terminal frame
// ships on BOTH paths before the error resumes bubbling to the caller.
- const outcome = await runTextStage(issueId, stageId, options, emit, generation)
+ const outcome = await getInstanceId().then((instanceId) => {
+ generation.instanceId = instanceId;
+ return runTextStage(issueId, stageId, options, emit, generation);
+ })
.then((result) => ({ result }), (error) => ({ error }));
liveGenerations.delete(generation.id);
if (outcome.error) {
@@ -979,6 +985,37 @@ async function runTextStage(issueId, stageId, options, emit, generation) {
}
}
+const hasUnknownOwner = (stage) => !stage.generation?.instanceId
+ || stage.generation.instanceId === UNKNOWN_INSTANCE_ID;
+
+export async function getStageRecovery(issueId, stageId) {
+ const issue = await getIssue(issueId);
+ const stage = issue.stages[stageId];
+ const instanceId = await getInstanceId();
+ const ownership = hasUnknownOwner(stage) ? 'unknown'
+ : (stage.generation.instanceId === instanceId ? 'local' : 'foreign');
+ const live = hasLiveGeneration(issueId, stageId);
+ return { revision: textStageRevision(stage), ownership, live,
+ recoverable: stage.status === 'generating' && ownership === 'unknown' && !live };
+}
+
+export function recoverLegacyStage(issueId, stageId, revision) {
+ return updateStageWithLatest(issueId, stageId, (current) => {
+ if (hasLiveGeneration(issueId, stageId)) {
+ throw codedError('A local generation is active. Wait for it to finish.', 'PIPELINE_RECOVERY_CONFLICT');
+ }
+ if (textStageRevision(current) !== revision) {
+ throw codedError('The stage changed. Refresh it before taking over.', 'PIPELINE_RECOVERY_CONFLICT');
+ }
+ if (current.status !== 'generating' || !hasUnknownOwner(current)) {
+ throw codedError('Only generating stages with unknown execution ownership can be taken over.', 'PIPELINE_RECOVERY_CONFLICT');
+ }
+ // Keep drafts, run history and editorial locks; this does not generate or publish.
+ return { status: 'error', generation: null,
+ errorMessage: 'Operator recovered unknown execution ownership — generate again.' };
+ });
+}
+
/** Only this install's abandoned generations may be demoted; legacy ownership is unknown. */
export async function recoverStuckGeneratingStages() {
const instanceId = await getInstanceId();
diff --git a/server/services/pipeline/textStages.test.js b/server/services/pipeline/textStages.test.js
index a6dd826491..84d9a1d5e5 100644
--- a/server/services/pipeline/textStages.test.js
+++ b/server/services/pipeline/textStages.test.js
@@ -1,3 +1,8 @@
+import express from 'express';
+import { request } from '../../lib/testHelper.js';
+import { errorMiddleware } from '../../lib/errorHandler.js';
+import recoveryRouter from '../../routes/pipeline/textStageRecovery.js';
+import { getInstanceId } from '../instanceIdentity.js';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { readFileSync } from 'fs';
import { dirname, join } from 'path';
@@ -119,6 +124,61 @@ describe('pipeline text stage generator', () => {
return { series, issue };
}
+ const recoveryApp = express();
+ recoveryApp.use(express.json(), recoveryRouter, errorMiddleware);
+ const recoveryPath = (id, action) => `/issues/${id}/stages/idea/${action}`;
+
+ it('route takeover preserves drafts, history and locks, then allows a fresh generation after unlocking', async () => {
+ const { issue } = await seed();
+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', input: 'Seed', output: 'Draft',
+ locked: true, lastRunId: 'old-run', runHistory: [{ runId: 'earlier', input: 'Prior seed', output: 'Prior draft' }] });
+ const before = (await issuesSvc.getIssue(issue.id)).stages.idea;
+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));
+ expect(status.body).toMatchObject({ ownership: 'unknown', live: false, recoverable: true });
+ const result = await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision });
+ expect(result.status).toBe(200);
+ expect(result.body.stage).toMatchObject({ status: 'error', input: before.input, output: before.output,
+ runHistory: before.runHistory, lastRunId: before.lastRunId, locked: true });
+ await expect(textStages.generateStage(issue.id, 'idea')).rejects.toThrow(/locked/i);
+ await issuesSvc.updateStage(issue.id, 'idea', { locked: false });
+ expect((await textStages.generateStage(issue.id, 'idea')).stage.status).toBe('ready');
+ });
+
+ it('route refuses a run started after the action was shown, before ownership is persisted', async () => {
+ const { issue } = await seed();
+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', output: 'Preserved' });
+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));
+ let releaseIdentity;
+ vi.mocked(getInstanceId).mockImplementationOnce(() => new Promise((resolve) => { releaseIdentity = resolve; }));
+ const running = textStages.generateStage(issue.id, 'idea');
+ const refused = await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision });
+ releaseIdentity('test-instance');
+ await running;
+ expect(refused.status).toBe(409);
+ expect(refused.body.error).toMatch(/local generation is active/i);
+ });
+
+ it('route rejects a revision race even when a user edit retains the stage timestamp', async () => {
+ const { issue } = await seed();
+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', output: 'Original' });
+ const before = (await issuesSvc.getIssue(issue.id)).stages.idea;
+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));
+ await issuesSvc.updateIssue(issue.id, { stages: { idea: { output: 'New edit' } } });
+ const refused = await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision });
+ expect(refused.status).toBe(409);
+ expect((await issuesSvc.getIssue(issue.id)).stages.idea).toMatchObject({ output: 'New edit', updatedAt: before.updatedAt });
+ });
+
+ it('route identifies foreign execution and rejects takeover and malformed requests', async () => {
+ const { issue } = await seed();
+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', generation: { id: 'foreign-run', instanceId: 'other-instance' } });
+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));
+ expect(status.body).toMatchObject({ ownership: 'foreign', recoverable: false });
+ expect((await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision })).status).toBe(409);
+ expect((await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: null })).status).toBe(400);
+ expect((await request(recoveryApp).post(`/issues/${issue.id}/stages/audio/recover`).send({ revision: status.body.revision })).status).toBe(400);
+ });
+
it('generateStage moves a stage idle → ready and persists output + runId', async () => {
const { issue } = await seed();
const result = await textStages.generateStage(issue.id, 'idea', { seedInput: 'foundry mystery' });