Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Recover legacy pipeline text stages explicitly (#11730)
[._.]/513cc6c1c74d
diff --git a/client/src/components/pipeline/stages/TextStagePanel.jsx b/client/src/components/pipeline/stages/TextStagePanel.jsxindex fa66db77b5..edd41061e1 100644--- a/client/src/components/pipeline/stages/TextStagePanel.jsx+++ b/client/src/components/pipeline/stages/TextStagePanel.jsx@@ -11,7 +11,7 @@ import toast from '../../ui/Toast';import InlineDiff from '../../ui/InlineDiff';import ProseEditor from '../../ui/ProseEditor';import {- generatePipelineStage, updatePipelineIssue,+ generatePipelineStage, updatePipelineIssue, getPipelineStageRecovery, recoverPipelineStage,pipelineStageProgressUrl,PIPELINE_STAGE_LABELS,PIPELINE_TEXT_STAGES,@@ -108,6 +108,37 @@ export default function TextStagePanel({const activeKeyRef = useRef(progressKey);useEffect(() => { activeKeyRef.current = progressKey; }, [progressKey]);+ const recoveryTargetRef = useRef(null);+ useEffect(() => {+ recoveryTargetRef.current = { key: progressKey, stage };+ return () => { recoveryTargetRef.current = null; };+ }, [progressKey, stage]);+ const [recovery, setRecovery] = useState(null);+ useEffect(() => {+ let active = true;+ if (stage.status === 'generating') {+ getPipelineStageRecovery(issue.id, stageId, { silent: true }).then((data) => {+ if (active) setRecovery({ key: progressKey, stage, data });+ }).catch(() => { if (active) setRecovery(null); });+ }+ return () => { active = false; };+ }, [issue.id, stageId, stage, progressKey]);+ const recoveryInfo = recovery?.key === progressKey && recovery.stage === stage ? recovery.data : null;+ const [runRecover, recovering] = useAsyncAction(+ (requestTarget) => recoverPipelineStage(issue.id, stageId, recoveryInfo.revision, { silent: true }).catch((error) => {+ if (recoveryTargetRef.current !== requestTarget) return null;+ throw error;+ }),+ { errorMessage: 'Recovery failed' },+ );+ const handleRecover = async () => {+ const requestTarget = recoveryTargetRef.current;+ const result = await runRecover(requestTarget);+ if (!result || recoveryTargetRef.current !== requestTarget) return;+ onStageUpdate?.(stageId, result.stage, result.issue);+ toast.success('Stage recovered. You can generate again; editorial locks still apply.');+ };+// Other text stages that currently have content — the candidate source// material for this generation. Excludes the target stage itself. Lets you// generate any stage FROM any other populated stage (backport), e.g. prose@@ -288,7 +319,7 @@ export default function TextStagePanel({<buttontype="button"onClick={handleSave}- disabled={!dirty || saving || !!activeDiff}+ disabled={!dirty || saving || recovering || !!activeDiff}title={activeDiff ? 'Resolve the new-version comparison first' : undefined}className="inline-flex items-center gap-2 px-3 py-1.5 rounded-lg bg-port-card border border-port-border text-white text-sm hover:border-port-accent/50 disabled:opacity-40">@@ -300,7 +331,7 @@ export default function TextStagePanel({onClick={outputDirty ? () => setConfirmRegenerate(true) : handleGenerate}// `saving` too: an in-flight save PATCH writes the same stage record// this run will overwrite when it lands.- disabled={generating || actionsGated || saving || !!activeDiff}+ disabled={generating || actionsGated || saving || recovering || !!activeDiff}title={actionsGated? 'Saving settings…': (outputDirty ? 'You have unsaved edits — you’ll compare them against the new version first' : undefined)}@@ -312,6 +343,29 @@ export default function TextStagePanel({</div></div>+ {stage.status === 'generating' && recoveryInfo ? (+ <div className="border border-port-warning/40 rounded p-3 space-y-2 text-sm">+ <p>{recoveryInfo.live+ ? 'A generation is active on this machine.'+ : recoveryInfo.ownership === 'foreign'+ ? 'This generation belongs to another machine. Its current progress is not known here.'+ : recoveryInfo.ownership === 'unknown'+ ? 'Execution ownership is unknown. This work may still be running on another machine. Taking over allows a fresh local generation and preserves your draft and history.'+ : 'This generation belongs to this machine.'}</p>+ {recoveryInfo.recoverable && !localGenerating ? (+ <button+ type="button"+ onClick={handleRecover}+ disabled={recovering || saving || dirty || actionsGated}+ title={dirty ? 'Save your edits before taking over.' : undefined}+ className="px-3 py-1.5 rounded-lg bg-port-card border border-port-border disabled:opacity-50"+ >+ {recovering ? 'Taking over…' : 'Take over stage'}+ </button>+ ) : null}+ </div>+ ) : null}+{confirmRegenerate && outputDirty ? (<div className="flex items-center justify-between gap-3 flex-wrap border border-port-warning/40 rounded p-3 bg-port-warning/5"><span className="text-xs text-gray-300">@@ -323,7 +377,7 @@ export default function TextStagePanel({onClick={handleGenerate}// Same gates as the Generate button it stands in for — the confirm// row can sit open while a save or a settings write starts.- disabled={generating || actionsGated || saving}+ disabled={generating || actionsGated || saving || recovering}className="inline-flex items-center gap-2 px-3 py-1.5 rounded-lg bg-port-accent text-white text-sm font-medium disabled:opacity-50"><GitCompare size={14} />@@ -453,6 +507,7 @@ export default function TextStagePanel({<label className="block"><span className="block text-xs uppercase tracking-wider text-gray-500 mb-1">Seed idea</span><textarea+ disabled={recovering}value={draftInput}onChange={(e) => setDraftInput(e.target.value)}placeholder={seedPlaceholder}@@ -466,6 +521,7 @@ export default function TextStagePanel({<div className="block"><span className="block text-xs uppercase tracking-wider text-gray-500 mb-1">Output</span><ProseEditor+ disabled={recovering}value={draftOutput}onChange={(e) => setDraftOutput(e.target.value)}placeholder={outputPlaceholder}@@ -479,6 +535,7 @@ export default function TextStagePanel({<span className="block text-xs uppercase tracking-wider text-gray-500 mb-1">Output</span><textareaid={`pipeline-stage-${stageId}-output`}+ disabled={recovering}value={draftOutput}onChange={(e) => setDraftOutput(e.target.value)}placeholder={outputPlaceholder}diff --git a/client/src/components/pipeline/stages/TextStagePanel.test.jsx b/client/src/components/pipeline/stages/TextStagePanel.test.jsxindex 7d4f1e9cd2..5b414d7a6c 100644--- a/client/src/components/pipeline/stages/TextStagePanel.test.jsx+++ b/client/src/components/pipeline/stages/TextStagePanel.test.jsx@@ -5,6 +5,8 @@ import { MockEventSource, lastEventSource } from '../../../test/mockEventSource'vi.mock('../../../services/api', () => ({generatePipelineStage: vi.fn(),+ getPipelineStageRecovery: vi.fn(async () => ({ recoverable: false, ownership: 'local', live: true })),+ recoverPipelineStage: vi.fn(),updatePipelineIssue: vi.fn(),restorePipelineStageVersion: vi.fn(),pipelineStageProgressUrl: (issueId, stageId) => `/api/pipeline/issues/${issueId}/stages/${stageId}/generate/progress`,@@ -21,7 +23,7 @@ vi.mock('../../ui/Toast', () => ({import TextStagePanel from './TextStagePanel';import toast from '../../ui/Toast';-import { generatePipelineStage, updatePipelineIssue } from '../../../services/api';+import { generatePipelineStage, updatePipelineIssue, getPipelineStageRecovery, recoverPipelineStage } from '../../../services/api';const makeIssue = (ideaOverrides = {}) => ({id: 'iss-1',@@ -46,6 +48,137 @@ describe('TextStagePanel', () => {vi.clearAllMocks();});+ it('offers explicit legacy takeover, preserves content and unlocks generation after success', async () => {+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'legacy-revision', ownership: 'unknown', live: false, recoverable: true });+ const issue = makeIssue({ status: 'generating', output: 'Preserved draft' });+ const stage = { ...issue.stages.idea, status: 'error' };+ recoverPipelineStage.mockResolvedValueOnce({ issue: { ...issue, stages: { idea: stage } }, stage });+ const onStageUpdate = vi.fn();+ const { rerender } = renderPanel(issue, { onStageUpdate });+ expect(screen.getByRole('button', { name: 'Generate' })).toBeDisabled();+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));+ expect(screen.getByText(/may still be running on another machine/)).toBeInTheDocument();+ expect(recoverPipelineStage).toHaveBeenCalledWith('iss-1', 'idea', 'legacy-revision', { silent: true });+ expect(onStageUpdate).toHaveBeenCalledWith('idea', stage, expect.any(Object));+ rerender(<TextStagePanel issue={{ ...issue, stages: { idea: stage } }} stageId="idea" />);+ expect(screen.getByRole('button', { name: 'Generate' })).toBeEnabled();+ expect(screen.getByLabelText('Output')).toHaveValue('Preserved draft');+ });++ it.each(['success', 'failure'])('ignores a pending takeover %s after switching issue and stage', async (outcome) => {+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'revision-a', ownership: 'unknown', live: false, recoverable: true });+ let resolveRecovery;+ let rejectRecovery;+ recoverPipelineStage.mockReturnValueOnce(new Promise((resolve, reject) => {+ resolveRecovery = resolve;+ rejectRecovery = reject;+ }));+ const issueA = makeIssue({ status: 'generating', output: 'Draft A' });+ const onStageUpdate = vi.fn();+ const { rerender } = renderPanel(issueA, { onStageUpdate });+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));+ const issueB = { id: 'iss-2', stages: { prose: { status: 'ready', input: '', output: 'Draft B', runHistory: [] } } };+ rerender(<TextStagePanel issue={issueB} stageId="prose" onStageUpdate={onStageUpdate} />);+ expect(screen.getByLabelText('Output')).toHaveValue('Draft B');+ await act(async () => {+ if (outcome === 'success') {+ const stage = { ...issueA.stages.idea, status: 'error' };+ resolveRecovery({ issue: { ...issueA, stages: { idea: stage } }, stage });+ } else {+ rejectRecovery(new Error('Stage A changed'));+ }+ });+ expect(onStageUpdate).not.toHaveBeenCalled();+ expect(toast.success).not.toHaveBeenCalled();+ expect(toast.error).not.toHaveBeenCalled();+ expect(screen.getByLabelText('Output')).toHaveValue('Draft B');+ expect(screen.getByRole('button', { name: 'Generate' })).toBeEnabled();+ });++ it.each(['success', 'failure'])('ignores takeover %s after leaving and returning to a replaced stage', async (outcome) => {+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'old-revision', ownership: 'unknown', live: false, recoverable: true });+ let resolveRecovery;+ let rejectRecovery;+ recoverPipelineStage.mockReturnValueOnce(new Promise((resolve, reject) => {+ resolveRecovery = resolve;+ rejectRecovery = reject;+ }));+ const issueA = makeIssue({ status: 'generating', output: 'Old draft A' });+ const onStageUpdate = vi.fn();+ const { rerender } = renderPanel(issueA, { onStageUpdate });+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));+ rerender(<TextStagePanel issue={{ id: 'iss-2', stages: { idea: { status: 'ready', output: 'Draft B' } } }} stageId="idea" onStageUpdate={onStageUpdate} />);+ const updatedIssueA = makeIssue({ status: 'ready', output: 'New draft A from another action' });+ rerender(<TextStagePanel issue={updatedIssueA} stageId="idea" onStageUpdate={onStageUpdate} />);+ await act(async () => {+ if (outcome === 'success') {+ const stage = { ...issueA.stages.idea, status: 'error' };+ resolveRecovery({ issue: { ...issueA, stages: { idea: stage } }, stage });+ } else {+ rejectRecovery(new Error('Old takeover failed'));+ }+ });+ expect(onStageUpdate).not.toHaveBeenCalled();+ expect(toast.success).not.toHaveBeenCalled();+ expect(toast.error).not.toHaveBeenCalled();+ expect(screen.getByLabelText('Output')).toHaveValue('New draft A from another action');+ expect(screen.getByRole('button', { name: 'Generate' })).toBeEnabled();+ });++ it.each(['success', 'failure'])('ignores takeover %s after the panel unmounts', async (outcome) => {+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'old-revision', ownership: 'unknown', live: false, recoverable: true });+ let settle;+ recoverPipelineStage.mockReturnValueOnce(new Promise((resolve, reject) => {+ settle = () => outcome === 'success' ? resolve({ stage: { status: 'error' } }) : reject(new Error('Old takeover failed'));+ }));+ const onStageUpdate = vi.fn();+ const { unmount } = renderPanel(makeIssue({ status: 'generating' }), { onStageUpdate });+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));+ unmount();+ await act(async () => settle());+ expect(onStageUpdate).not.toHaveBeenCalled();+ expect(toast.success).not.toHaveBeenCalled();+ expect(toast.error).not.toHaveBeenCalled();+ });++ it('disables takeover while preserving unsaved seed and output edits', async () => {+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'legacy-revision', ownership: 'unknown', live: false, recoverable: true });+ renderPanel(makeIssue({ status: 'generating', input: 'Saved seed', output: 'Saved draft' }));+ const takeover = await screen.findByRole('button', { name: 'Take over stage' });+ expect(takeover).toBeEnabled();+ await userEvent.type(screen.getByPlaceholderText('seed…'), ' with unsaved detail');+ await userEvent.type(screen.getByLabelText('Output'), '\nUnsaved second paragraph.');+ expect(takeover).toBeDisabled();+ await userEvent.click(takeover);+ expect(recoverPipelineStage).not.toHaveBeenCalled();+ expect(screen.getByPlaceholderText('seed…')).toHaveValue('Saved seed with unsaved detail');+ expect(screen.getByLabelText('Output')).toHaveValue('Saved draft\nUnsaved second paragraph.');+ });++ it.each(['A local generation is active.', 'The stage changed. Refresh it before taking over.'])(+ 'keeps generation blocked and content intact when takeover is refused: %s', async (message) => {+ getPipelineStageRecovery.mockResolvedValueOnce({ revision: 'shown-revision', ownership: 'unknown', live: false, recoverable: true });+ recoverPipelineStage.mockRejectedValueOnce(new Error(message));+ const onStageUpdate = vi.fn();+ renderPanel(makeIssue({ status: 'generating', output: 'Keep this draft' }), { onStageUpdate });+ await userEvent.click(await screen.findByRole('button', { name: 'Take over stage' }));+ await waitFor(() => expect(toast.error).toHaveBeenCalledWith(message));+ expect(onStageUpdate).not.toHaveBeenCalled();+ expect(screen.getByRole('button', { name: 'Generate' })).toBeDisabled();+ expect(screen.getByLabelText('Output')).toHaveValue('Keep this draft');+ },+ );++ it.each([+ { ownership: 'foreign', live: false, text: /belongs to another machine/ },+ { ownership: 'unknown', live: true, text: /active on this machine/ },+ ])('does not offer recovery for $ownership ownership with live=$live', async ({ ownership, live, text }) => {+ getPipelineStageRecovery.mockResolvedValueOnce({ ownership, live, recoverable: false });+ renderPanel(makeIssue({ status: 'generating' }));+ expect(await screen.findByText(text)).toBeInTheDocument();+ expect(screen.queryByRole('button', { name: 'Take over stage' })).not.toBeInTheDocument();+ });+it('Generate success: calls the generate API with the right shape and reflects the new content once the parent lifts it', async () => {const stage = { status: 'ready', input: '', output: 'Generated idea text.', runHistory: [] };generatePipelineStage.mockResolvedValue({ stage });@@ -122,9 +255,10 @@ describe('TextStagePanel', () => {expect(screen.getByRole('button', { name: /Save edits/i })).toBeDisabled();});- it('resume-on-mount: mounting with server-side "generating" status keeps Generate disabled without starting a fresh run', () => {+ it('resume-on-mount: mounting with server-side "generating" status keeps Generate disabled without starting a fresh run', async () => {const issue = makeIssue({ status: 'generating' });renderPanel(issue);+ await screen.findByText('A generation is active on this machine.');expect(screen.getByRole('button', { name: 'Generate' })).toBeDisabled();expect(generatePipelineStage).not.toHaveBeenCalled();diff --git a/client/src/services/apiPipeline.js b/client/src/services/apiPipeline.jsindex f52fba6076..dd34ad3a92 100644--- a/client/src/services/apiPipeline.js+++ b/client/src/services/apiPipeline.js@@ -143,6 +143,13 @@ export const deletePipelineIssue = (id, options = {}) =>request(`/pipeline/issues/${encodeURIComponent(id)}`, { method: 'DELETE', ...options });// ---- Stage operations ----+export const getPipelineStageRecovery = (issueId, stageId, options = {}) =>+ request(`/pipeline/issues/${encodeURIComponent(issueId)}/stages/${encodeURIComponent(stageId)}/recovery`, options);+export const recoverPipelineStage = (issueId, stageId, revision, options = {}) =>+ request(`/pipeline/issues/${encodeURIComponent(issueId)}/stages/${encodeURIComponent(stageId)}/recover`, {+ method: 'POST', body: JSON.stringify({ revision }), ...options,+ });+export const generatePipelineStage = (issueId, stageId, opts = {}, options = {}) =>request(`/pipeline/issues/${encodeURIComponent(issueId)}/stages/${encodeURIComponent(stageId)}/generate`, {method: 'POST',diff --git a/server/lib/hostControlRoutes.test.js b/server/lib/hostControlRoutes.test.jsindex d1dc7ebbd8..38c5200714 100644--- a/server/lib/hostControlRoutes.test.js+++ b/server/lib/hostControlRoutes.test.js@@ -520,6 +520,8 @@ describe('Pipeline authoring remainder policy (#10907)', () => {'PATCH /api/pipeline/tts/narrate/segment','POST /api/pipeline/editorial/custom-checks','POST /api/pipeline/issues/:id/auto-run-text/cancel',+ // Recovery only resets persisted state; generating again uses the gated route.+ 'POST /api/pipeline/issues/:id/stages/:stageId/recover','POST /api/pipeline/issues/:id/stages/:stageId/restore','POST /api/pipeline/issues/:id/stages/audio/cues/:cueIdx/render','POST /api/pipeline/issues/:id/stages/audio/extract-lines',diff --git a/server/lib/pipelineValidation.js b/server/lib/pipelineValidation.jsindex 7cc2d7eeb8..c7e2336b02 100644--- a/server/lib/pipelineValidation.js+++ b/server/lib/pipelineValidation.js@@ -32,6 +32,8 @@ import { CHECK_SCOPES, CHECK_SEVERITIES } from './editorial/checkInfra/taxonomy.import { SHOT_TYPES, SCREEN_DIRECTIONS } from './shotGrammar.js';import { characterEvolutionSchema } from './characterEvolutionValidation.js';+export const pipelineStageRecoverySchema = z.object({ revision: z.string().regex(/^[a-f0-9]{64}$/) });+// =============================================================================// WRITERS ROOM SCHEMAS// =============================================================================diff --git a/server/routes/pipeline/issues.js b/server/routes/pipeline/issues.jsindex aafcad991e..ad33435ff6 100644--- a/server/routes/pipeline/issues.js+++ b/server/routes/pipeline/issues.js@@ -51,6 +51,8 @@ import {import { ARC_ROLES } from '../../lib/storyArc.js';import { mapServiceError, countExtractedCanon } from './shared.js';+import textStageRecoveryRouter from './textStageRecovery.js';+const router = Router();// ---- Issue schemas ----@@ -432,6 +434,8 @@ router.delete('/issues/:id', asyncHandler(async (req, res) => {// Stage operations// =====================+router.use(textStageRecoveryRouter);+router.post('/issues/:id/stages/:stageId/generate', asyncHandler(async (req, res) => {const { id, stageId } = req.params;if (!issuesSvc.TEXT_STAGE_IDS.includes(stageId)) {diff --git a/server/routes/pipeline/textStageRecovery.js b/server/routes/pipeline/textStageRecovery.jsnew file mode 100644index 0000000000..1acbd66b46--- /dev/null+++ b/server/routes/pipeline/textStageRecovery.js@@ -0,0 +1,23 @@+import { Router } from 'express';+import { z } from 'zod';+import { asyncHandler, createServiceErrorMapper } from '../../lib/errorHandler.js';+import { validateRequest } from '../../lib/validation.js';+import { pipelineStageRecoverySchema } from '../../lib/pipelineValidation.js';+import { TEXT_STAGE_IDS, ERR_NOT_FOUND } from '../../services/pipeline/issuesShared.js';+import { getStageRecovery, recoverLegacyStage } from '../../services/pipeline/textStages.js';++const router = Router();+const paramsSchema = z.object({ id: z.string().min(1), stageId: z.enum(TEXT_STAGE_IDS) });+const mapError = createServiceErrorMapper({ [ERR_NOT_FOUND]: 404, PIPELINE_RECOVERY_CONFLICT: 409 });++router.get('/issues/:id/stages/:stageId/recovery', asyncHandler(async (req, res) => {+ const { id, stageId } = validateRequest(paramsSchema, req.params);+ res.json(await getStageRecovery(id, stageId).catch((err) => { throw mapError(err); }));+}));+router.post('/issues/:id/stages/:stageId/recover', asyncHandler(async (req, res) => {+ const { id, stageId } = validateRequest(paramsSchema, req.params);+ const { revision } = validateRequest(pipelineStageRecoverySchema, req.body ?? {});+ res.json(await recoverLegacyStage(id, stageId, revision).catch((err) => { throw mapError(err); }));+}));++export default router;diff --git a/server/services/pipeline/issueStages.js b/server/services/pipeline/issueStages.jsindex c172ab8cfd..1d6506f00a 100644--- a/server/services/pipeline/issueStages.js+++ b/server/services/pipeline/issueStages.js@@ -8,6 +8,7 @@* renumber/bulk write that rewrites the same issue — one mutex per series.*/+import { createHash } from 'node:crypto';import { isPlainObject } from '../../lib/objects.js';import { pickRenderedFilename } from '../../lib/renderSlot.js';import { emitRecordUpdated } from '../sharing/recordEvents.js';@@ -22,6 +23,11 @@ import {import { isStr } from '../../lib/textUtils.js';import { codedError } from '../../lib/codedError.js';+// Hash the complete sanitized stage, including content and history: timestamps alone+// miss same-tick edits and issue-level PATCHes that retain the stage timestamp.+export const textStageRevision = (stage) => createHash('sha256')+ .update(JSON.stringify(sanitizeTextStage(stage))).digest('hex');+/*** Partial update to a single stage on an issue. Use this from generators so* a stage write doesn't have to load the full issue, mutate, and re-validate.diff --git a/server/services/pipeline/textStages.js b/server/services/pipeline/textStages.jsindex 6adc47c597..0a94db2ccb 100644--- a/server/services/pipeline/textStages.js+++ b/server/services/pipeline/textStages.js@@ -18,6 +18,7 @@* emit is a no-op and generation is byte-for-byte the un-streamed path.*/+import { codedError } from '../../lib/codedError.js';import { randomUUID } from 'node:crypto';import { getInstanceId, UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';import { runStagedLLM, resolveStageContext } from '../stageRunner.js';@@ -25,7 +26,7 @@ import { getStage } from '../promptService.js';import { beginStageProgress, emitStageProgress, finishStageProgress } from './textStageProgress.js';import { getSeries } from './series.js';import { extractCanonFromProse, summarizeCanonExtraction } from '../universeCanon.js';-import { getIssue, listIssues, updateStage, updateStageWithLatest, assertStageUnlocked, TEXT_STAGE_IDS } from './issues.js';+import { getIssue, listIssues, updateStage, updateStageWithLatest, assertStageUnlocked, textStageRevision, TEXT_STAGE_IDS } from './issues.js';import { pickCanon } from './seriesCanon.js';import { getUniverse } from '../universeBuilder.js';import { compareIssuesByPosition, NO_LINKED_UNIVERSE_PLACEHOLDER } from './arcPlanner.js';@@ -38,7 +39,9 @@ import { usableInputTokens, trimContextToBudget, CHARS_PER_TOKEN } from '../../limport { escapeRegExp } from '../../lib/textUtils.js';// Separate from the advisory progress channel: overlapping runs each stay live.-const liveGenerations = new Set();+const liveGenerations = new Map();+const hasLiveGeneration = (issueId, stageId) => [...liveGenerations.values()]+ .some((run) => run.issueId === issueId && run.stageId === stageId);const STAGE_TO_TEMPLATE = Object.freeze({idea: 'pipeline-idea-expansion',@@ -885,8 +888,8 @@ export async function generateStage(issueId, stageId, options = {}) {throw new Error(`generateStage: unsupported stageId "${stageId}"`);}// Generation may overlap, but only the first run owns this stream.- const generation = { id: randomUUID(), instanceId: await getInstanceId() };- liveGenerations.add(generation.id);+ const generation = { id: randomUUID(), instanceId: null };+ liveGenerations.set(generation.id, { issueId, stageId });const ownsProgress = beginStageProgress(issueId, stageId);const emit = (payload) => {if (ownsProgress) emitStageProgress(issueId, stageId, payload);@@ -894,7 +897,10 @@ export async function generateStage(issueId, stageId, options = {}) {emit({ type: 'start', issueId, stageId, at: new Date().toISOString() });// Settle to an outcome object rather than try/catch so the terminal frame// ships on BOTH paths before the error resumes bubbling to the caller.- const outcome = await runTextStage(issueId, stageId, options, emit, generation)+ const outcome = await getInstanceId().then((instanceId) => {+ generation.instanceId = instanceId;+ return runTextStage(issueId, stageId, options, emit, generation);+ }).then((result) => ({ result }), (error) => ({ error }));liveGenerations.delete(generation.id);if (outcome.error) {@@ -979,6 +985,37 @@ async function runTextStage(issueId, stageId, options, emit, generation) {}}+const hasUnknownOwner = (stage) => !stage.generation?.instanceId+ || stage.generation.instanceId === UNKNOWN_INSTANCE_ID;++export async function getStageRecovery(issueId, stageId) {+ const issue = await getIssue(issueId);+ const stage = issue.stages[stageId];+ const instanceId = await getInstanceId();+ const ownership = hasUnknownOwner(stage) ? 'unknown'+ : (stage.generation.instanceId === instanceId ? 'local' : 'foreign');+ const live = hasLiveGeneration(issueId, stageId);+ return { revision: textStageRevision(stage), ownership, live,+ recoverable: stage.status === 'generating' && ownership === 'unknown' && !live };+}++export function recoverLegacyStage(issueId, stageId, revision) {+ return updateStageWithLatest(issueId, stageId, (current) => {+ if (hasLiveGeneration(issueId, stageId)) {+ throw codedError('A local generation is active. Wait for it to finish.', 'PIPELINE_RECOVERY_CONFLICT');+ }+ if (textStageRevision(current) !== revision) {+ throw codedError('The stage changed. Refresh it before taking over.', 'PIPELINE_RECOVERY_CONFLICT');+ }+ if (current.status !== 'generating' || !hasUnknownOwner(current)) {+ throw codedError('Only generating stages with unknown execution ownership can be taken over.', 'PIPELINE_RECOVERY_CONFLICT');+ }+ // Keep drafts, run history and editorial locks; this does not generate or publish.+ return { status: 'error', generation: null,+ errorMessage: 'Operator recovered unknown execution ownership — generate again.' };+ });+}+/** Only this install's abandoned generations may be demoted; legacy ownership is unknown. */export async function recoverStuckGeneratingStages() {const instanceId = await getInstanceId();diff --git a/server/services/pipeline/textStages.test.js b/server/services/pipeline/textStages.test.jsindex a6dd826491..84d9a1d5e5 100644--- a/server/services/pipeline/textStages.test.js+++ b/server/services/pipeline/textStages.test.js@@ -1,3 +1,8 @@+import express from 'express';+import { request } from '../../lib/testHelper.js';+import { errorMiddleware } from '../../lib/errorHandler.js';+import recoveryRouter from '../../routes/pipeline/textStageRecovery.js';+import { getInstanceId } from '../instanceIdentity.js';import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';import { readFileSync } from 'fs';import { dirname, join } from 'path';@@ -119,6 +124,61 @@ describe('pipeline text stage generator', () => {return { series, issue };}+ const recoveryApp = express();+ recoveryApp.use(express.json(), recoveryRouter, errorMiddleware);+ const recoveryPath = (id, action) => `/issues/${id}/stages/idea/${action}`;++ it('route takeover preserves drafts, history and locks, then allows a fresh generation after unlocking', async () => {+ const { issue } = await seed();+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', input: 'Seed', output: 'Draft',+ locked: true, lastRunId: 'old-run', runHistory: [{ runId: 'earlier', input: 'Prior seed', output: 'Prior draft' }] });+ const before = (await issuesSvc.getIssue(issue.id)).stages.idea;+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));+ expect(status.body).toMatchObject({ ownership: 'unknown', live: false, recoverable: true });+ const result = await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision });+ expect(result.status).toBe(200);+ expect(result.body.stage).toMatchObject({ status: 'error', input: before.input, output: before.output,+ runHistory: before.runHistory, lastRunId: before.lastRunId, locked: true });+ await expect(textStages.generateStage(issue.id, 'idea')).rejects.toThrow(/locked/i);+ await issuesSvc.updateStage(issue.id, 'idea', { locked: false });+ expect((await textStages.generateStage(issue.id, 'idea')).stage.status).toBe('ready');+ });++ it('route refuses a run started after the action was shown, before ownership is persisted', async () => {+ const { issue } = await seed();+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', output: 'Preserved' });+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));+ let releaseIdentity;+ vi.mocked(getInstanceId).mockImplementationOnce(() => new Promise((resolve) => { releaseIdentity = resolve; }));+ const running = textStages.generateStage(issue.id, 'idea');+ const refused = await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision });+ releaseIdentity('test-instance');+ await running;+ expect(refused.status).toBe(409);+ expect(refused.body.error).toMatch(/local generation is active/i);+ });++ it('route rejects a revision race even when a user edit retains the stage timestamp', async () => {+ const { issue } = await seed();+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', output: 'Original' });+ const before = (await issuesSvc.getIssue(issue.id)).stages.idea;+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));+ await issuesSvc.updateIssue(issue.id, { stages: { idea: { output: 'New edit' } } });+ const refused = await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision });+ expect(refused.status).toBe(409);+ expect((await issuesSvc.getIssue(issue.id)).stages.idea).toMatchObject({ output: 'New edit', updatedAt: before.updatedAt });+ });++ it('route identifies foreign execution and rejects takeover and malformed requests', async () => {+ const { issue } = await seed();+ await issuesSvc.updateStage(issue.id, 'idea', { status: 'generating', generation: { id: 'foreign-run', instanceId: 'other-instance' } });+ const status = await request(recoveryApp).get(recoveryPath(issue.id, 'recovery'));+ expect(status.body).toMatchObject({ ownership: 'foreign', recoverable: false });+ expect((await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: status.body.revision })).status).toBe(409);+ expect((await request(recoveryApp).post(recoveryPath(issue.id, 'recover')).send({ revision: null })).status).toBe(400);+ expect((await request(recoveryApp).post(`/issues/${issue.id}/stages/audio/recover`).send({ revision: status.body.revision })).status).toBe(400);+ });+it('generateStage moves a stage idle → ready and persists output + runId', async () => {const { issue } = await seed();const result = await textStages.generateStage(issue.id, 'idea', { seedInput: 'foundry mystery' });