Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { maintenance } from '../../lib/maintenanceAdmission.js';
/**
* Creative Director — production-plan advance loop (CDO Phase 2, #2184).
*
* The generalized sibling of completionHook.js#advanceAfterSceneSettled. For a
* DIRECTIVE-driven project (`project.directive` present), "what happens next" is
* a pure function of the plan DAG on the project record — never a stored cursor:
*
* - No plan yet → enqueue the planner agent (cd-plan) to write one.
* - A runnable step → dispatch it through the gated creative tool
* registry (`dispatchCreativeTool`), one at a time,
* sequentially, respecting `dependsOn`.
* - A long-running step → stays `running` until its media job settles (a
* completion event re-fires this loop).
* - A step failed → bounded re-plan (MAX_REPLAN_ROUNDS), then pause
* with residuals for human review.
* - A blocked step → pause with the block reason (never silently retry
* around a human-review pause).
* - All steps terminal → the project is complete.
*
* Legacy video projects (no `directive`/`plan`) never enter here — the treatment/
* scene flow in completionHook.js is unchanged. Idempotent + driven by CoS
* completions (the planner task), media-job events (long-running steps), and boot
* recovery — the same shape as the scene loop.
*/
import { getProject, updateProject, updatePlanStep, recordRun, updateRun } from './local.js';
import { enqueuePlanTask } from './agentBridge.js';
import { dispatchCreativeTool } from '../creative/toolRegistry.js';
import { listJobs, mediaJobEvents } from '../mediaJobQueue/index.js';
// Import the modules that DECLARE these, not the `seriesAutopilot.js` barrel that
// forwards them (#5920): a barrel re-entry from outside the package pulls the whole
// autopilot cluster in and is what closed the old import cycle through this file.
import { autopilotEvents, AUTOPILOT_TERMINAL_TYPES } from '../pipeline/seriesAutopilot/state.js';
import { isAutopilotActive } from '../pipeline/seriesAutopilot/session.js';
import { getSeries } from '../pipeline/series.js';
import { MAX_REPLAN_ROUNDS, PLAN_STEP_TERMINAL_SUCCESS } from '../../lib/creativeDirectorPresets.js';
import { blockedStageReason, closeDeliverableStreak, exhaustedDeliverableStreak } from './deliverableGate.js';
import { logFailureWithStack } from '../../lib/failureLogging.js';
// The registry tool that runs Series Autopilot as a plan step. A long-running
// step whose dispatch returns a run handle (a `runId`, not a media `jobId`)
// settles via the in-process autopilot event bus (CDO Phase 3, #2185).
const AUTOPILOT_TOOL_NAME = 'pipeline_startSeriesAutopilot';
const nowISO = () => new Date().toISOString();
// The registry tool a plan step uses to render a video. Its done result carries
// `{ jobId }`, and a video history entry's id === its jobId — so the last such
// step's jobId is the project's final video (see the `complete` branch below).
const VIDEO_RENDER_TOOL_NAME = 'media_enqueueVideoJob';
/**
* The jobId of the LAST done video-render step in a plan (or null). Used to
* promote a directive plan's produced video to the project's `finalVideoId` on
* completion so the CD overview surfaces it. Pure + last-wins: a multi-render
* plan resolves to its final cut in listed order.
*/
export function lastRenderedVideoJobId(plan) {
const steps = Array.isArray(plan?.steps) ? plan.steps : [];
for (let i = steps.length - 1; i >= 0; i -= 1) {
const step = steps[i];
if (step?.toolName === VIDEO_RENDER_TOOL_NAME && step.status === 'done' && step.result?.jobId) {
return step.result.jobId;
}
}
return null;
}
/**
* Pure next-step derivation over a plan DAG. Exported + side-effect-free so the
* ordering/dependency/terminal logic is unit-tested directly against fixtures.
*
* Returns one of:
* - `{ type: 'empty' }` — plan has no steps → nothing to execute.
* - `{ type: 'waiting', step }` — a step is `running`; wait for it to settle.
* - `{ type: 'blocked', step }` — a step is `blocked` (human-review pause).
* - `{ type: 'run', step }` — the next runnable pending step (deps all
* terminal-success), in listed order.
* - `{ type: 'stuck', steps }` — pending steps remain but none is runnable
* (a dependency failed or is missing).
* - `{ type: 'failed', steps }` — every step terminal and ≥1 failed.
* - `{ type: 'complete' }` — every step done/skipped.
*
* Sequential by construction: even when two pending steps are both runnable it
* returns the FIRST (array order) — v1 has no parallel branches.
*
* @param {{steps?: Array<object>}} plan
*/
export function deriveNextPlanAction(plan) {
const steps = Array.isArray(plan?.steps) ? plan.steps : [];
if (!steps.length) return { type: 'empty' };
const byId = new Map(steps.map((s) => [s.stepId, s]));
const running = steps.find((s) => s.status === 'running');
if (running) return { type: 'waiting', step: running };
const blocked = steps.find((s) => s.status === 'blocked');
if (blocked) return { type: 'blocked', step: blocked };
const depsSatisfied = (s) => (Array.isArray(s.dependsOn) ? s.dependsOn : []).every((id) => {
const dep = byId.get(id);
return dep && PLAN_STEP_TERMINAL_SUCCESS.has(dep.status);
});
const pending = steps.filter((s) => s.status === 'pending');
const runnable = pending.find(depsSatisfied);
if (runnable) return { type: 'run', step: runnable };
// No pending step is runnable and nothing is running → the plan can't progress
// (a dependency failed/missing, or a cycle). Surface as residuals.
if (pending.length) return { type: 'stuck', steps: pending };
const failed = steps.filter((s) => s.status === 'failed');
if (failed.length) return { type: 'failed', steps: failed };
return { type: 'complete' };
}
// A single cross-step result reference: `{{steps.<stepId>.result.<dotpath>}}`.
// The reference resolves to a value produced by an EARLIER (terminal-success)
// step's persisted `result` — the id-only summary `summarizeResult` stored (e.g.
// a created series' `result.id`). stepIds allow letters/digits/`_`/`-`; the path
// is a dotted key list into the result object.
// One source for the reference grammar; the anchored form (whole-value → raw
// type preserved) and the global form (embedded → stringified) derive from it so
// widening the id/path character classes can't diverge the two.
const STEP_REF_BODY = String.raw`\{\{\s*steps\.([\w-]+)\.result\.([\w.-]+)\s*\}\}`;
const STEP_REF_ANCHORED = new RegExp(`^${STEP_REF_BODY}$`);
const STEP_REF_GLOBAL = new RegExp(STEP_REF_BODY, 'g');
// Fast-path sentinel — the grammar's opening (`{{`, optional whitespace,
// `steps.`). MUST stay as tolerant as STEP_REF_BODY's prefix or a
// whitespace-padded reference the resolver WOULD match slips past the early-out
// and dispatches as a literal. Non-global so `.test()` carries no lastIndex.
const STEP_REF_SENTINEL = /\{\{\s*steps\./;
/**
* Resolve `{{steps.<stepId>.result.<path>}}` references in a plan step's `args`
* against the plan's already-completed steps (#2773). The executor otherwise
* dispatches `args` verbatim and never interpolated prior results — so a step
* that needs a just-minted id (the classic case: a `series` commission where
* `pipeline_startSeriesAutopilot` needs the id `pipeline_createSeries` mints)
* had no way to reference it, and could only ever create an empty series.
*
* A whole-string reference (`"{{steps.create-series.result.id}}"`) substitutes
* the RAW resolved value (type preserved); an embedded reference inside a longer
* string is stringified in place. References are resolved recursively through
* nested arrays/objects in `args`.
*
* Pure + side-effect-free (exported for direct unit testing). Returns
* `{ args, error }`: `error` is a non-null human-readable string when any
* reference points at an unknown step, a step that is not terminal-success, or a
* missing result key — the executor treats that as a planning error (fail the
* step → bounded re-plan) rather than dispatching an unresolved `{{…}}` literal.
*
* @param {{stepId?: string, args?: object}} step
* @param {{steps?: Array<object>}} plan
*/
export function resolvePlanStepArgs(step, plan) {
const args = step?.args || {};
// Fast path: the vast majority of steps carry no references. A cheap
// serialize-and-scan for the reference opener returns the args untouched
// without building the step index or walking/cloning the tree.
if (!STEP_REF_SENTINEL.test(JSON.stringify(args))) return { args, error: null };
const steps = Array.isArray(plan?.steps) ? plan.steps : [];
const byId = new Map(steps.map((s) => [s.stepId, s]));
const errors = [];
const resolveRef = (stepId, path) => {
const dep = byId.get(stepId);
if (!dep) { errors.push(`references unknown step "${stepId}"`); return undefined; }
if (!PLAN_STEP_TERMINAL_SUCCESS.has(dep.status)) {
errors.push(`references step "${stepId}" which is not complete (status: ${dep.status || 'pending'})`);
return undefined;
}
// Dry-run preview: a step dispatched in dry-run mode settles `done` with only
// `{ planned: true }` and mints no real id, so a reference into it has nothing
// to resolve. Substitute a synthetic placeholder rather than erroring, so the
// whole-plan preview walk completes (the downstream step is itself dry-run and
// never uses the value) instead of failing into a spurious re-plan.
if (dep.result?.planned === true) return `dry-run:${stepId}.${path}`;
let val = dep.result;
for (const seg of path.split('.')) val = val == null ? undefined : val[seg];
if (val === undefined || val === null) {
errors.push(`references missing result "${path}" on step "${stepId}"`);
return undefined;
}
return val;
};
const resolveString = (str) => {
if (!str.includes('{{')) return str;
const whole = STEP_REF_ANCHORED.exec(str);
if (whole) {
const v = resolveRef(whole[1], whole[2]);
return v === undefined ? str : v;
}
return str.replace(STEP_REF_GLOBAL, (m, stepId, path) => {
const v = resolveRef(stepId, path);
return v === undefined ? m : String(v);
});
};
const resolveValue = (val) => {
if (typeof val === 'string') return resolveString(val);
if (Array.isArray(val)) return val.map(resolveValue);
if (val && typeof val === 'object') {
const out = {};
for (const [k, v] of Object.entries(val)) out[k] = resolveValue(v);
return out;
}
return val;
};
const resolvedArgs = resolveValue(args);
// Belt-and-suspenders: the plan schema permits stepIds the reference grammar
// (`[\w-]+`) can't parse (a dot, colon, space, or Unicode char). Such a
// reference matches the sentinel but not STEP_REF_*, so it would survive
// resolution as a literal and dispatch unresolved. If any `{{steps.…}}` is left
// after a clean pass, fail the step (→ re-plan) instead — never dispatch a
// literal placeholder. Skip when errors already fired (they carry the real cause).
if (!errors.length && STEP_REF_SENTINEL.test(JSON.stringify(resolvedArgs))) {
errors.push('contains an unresolved step reference — use the form {{steps.<stepId>.result.<key>}} with a word/hyphen stepId');
}
return { args: resolvedArgs, error: errors.length ? `Step "${step?.stepId}" ${errors.join('; ')}` : null };
}
// In-memory dedup, same role as completionHook's inflight sets. `inflightPlanner`
// covers the updateProject→enqueue window for the planner; `inflightPlanStep`
// covers the getProject→recordRun window for a dispatch (released once the
// persisted `running` step+run guard re-entry). `planJobCleanups` holds the
// teardown for every armed long-running media-job listener so tests can drop them.
const inflightPlanner = new Set();
const inflightPlanStep = new Set();
const planJobCleanups = new Map();
async function pausePlanWithResidual(projectId, reason) {
await updateProject(projectId, { status: 'paused', failureReason: reason })
.catch((e) => console.log(`⚠️ CD plan pause for ${projectId} failed: ${e.message}`));
console.log(`⏸️ CD plan ${projectId} paused: ${reason}`);
}
async function finishRun(projectId, runId, status, failureReason) {
if (!runId) return;
await updateRun(projectId, runId, {
status,
completedAt: nowISO(),
...(failureReason ? { failureReason } : {}),
}).catch((e) => console.log(`⚠️ CD plan finishRun ${runId} on ${projectId} failed: ${e.message}`));
}
// Compact, id-only summary of a tool result so the plan step's `result` never
// bloats with a full render payload / record body (mirrors the run ledger's
// digest-not-payload rule).
function summarizeResult(result) {
if (!result || typeof result !== 'object') return {};
const out = {};
for (const k of ['id', 'jobId', 'seriesId', 'issueId', 'universeId', 'name', 'status']) {
if (result[k] != null) out[k] = result[k];
}
return out;
}
// Flip the project to `planning` and hand the planner agent a fresh task. Shared
// by the no-plan-yet branch and the bounded re-planner so they can never drift on
// the in-flight bookkeeping. Callers own their own pre-guards AND must have taken
// `inflightPlanner` already (this releases it).
async function dispatchPlanner(projectId) {
await updateProject(projectId, { status: 'planning' })
.catch((e) => { inflightPlanner.delete(projectId); throw e; });
const fresh = await getProject(projectId);
await enqueuePlanTask(fresh).finally(() => inflightPlanner.delete(projectId));
}
/**
* #4146 — bounded blocked-stage gate in front of every planner dispatch.
*
* The planner agent's deliverable is `PATCH /:id/plan`. When it exits cleanly
* without PATCHing, completionHook now records the run `failed` — which is what
* lets the guard above re-dispatch LIVE rather than at boot. But re-handing the
* SAME task to a model that has already demonstrated it cannot perform the PATCH
* just burns the provider in a tight loop (and the re-plan route can't even be
* bounded by MAX_REPLAN_ROUNDS: `replanRounds` only advances when a plan actually
* lands). After MAX_CONSECUTIVE_MISSED_DELIVERABLES empty completions, surface
* the stage to the user instead — and close the streak so a Resume after
* switching models starts from a clean budget.
*
* Returns true when the caller must NOT dispatch.
*/
async function plannerBlockedOnEmptyRuns(project) {
const streak = exhaustedDeliverableStreak(project.runs, 'plan');
if (!streak) return false;
await closeDeliverableStreak(project.id, project.runs, 'plan');
await pausePlanWithResidual(project.id, blockedStageReason('plan', streak));
return true;
}
async function enqueuePlannerOnce(project) {
const projectId = project.id;
const hasInflightPlanRun = (project.runs || []).some(
(r) => r.kind === 'plan' && r.status !== 'completed' && r.status !== 'failed',
);
if (hasInflightPlanRun || inflightPlanner.has(projectId)) return;
if (await plannerBlockedOnEmptyRuns(project)) return;
inflightPlanner.add(projectId);
return dispatchPlanner(projectId);
}
/**
* The generalized advance loop. Pure next-step over the plan, idempotent, safe
* to call from a CoS completion, a media-job settle, boot recovery, or a manual
* resume — it re-reads project state and derives the one next action every time.
*
* @param {string} projectId
*/
export async function advanceAfterPlanStepSettled(projectId) {
const project = await getProject(projectId).catch(() => null);
if (!project) return;
if (project.workspace === 'video') {
const { videoReviewAllowsDispatch } = await import('./videoReview.js');
if (!await videoReviewAllowsDispatch(projectId, [])) return;
}
if (project.status === 'paused' || project.status === 'failed') return;
// Legacy video project — the scene loop owns it; never plan-advance it.
if (!project.directive) return;
// No plan yet → enqueue the planner (mirrors the no-treatment branch).
if (!Array.isArray(project.plan?.steps)) {
return enqueuePlannerOnce(project);
}
const action = deriveNextPlanAction(project.plan);
switch (action.type) {
case 'waiting':
// A step is running (a long-running job in flight). Its settle re-fires us.
return;
case 'blocked':
return pausePlanWithResidual(
projectId,
`Step "${action.step.stepId}" is blocked: ${action.step.result?.reason || 'awaiting human review'}`,
);
case 'stuck':
return pausePlanWithResidual(
projectId,
`No runnable step — ${action.steps.length} pending step(s) blocked by a failed or missing dependency`,
);
case 'failed':
// Safety net: a step is 'failed' but the executor didn't route it (e.g. a
// hand-edited/legacy record). Send it through the failure handler.
return handlePlanStepFailure(projectId, action.steps[0]);
case 'empty':
case 'complete': {
if (project.workspace === 'video') {
const { runStitch } = await import('./stitchRunner.js');
return runStitch(projectId);
}
// Promote the plan's produced video so the CD overview has an artifact to
// show. Unlike the legacy scene/stitch flow (stitchRunner sets
// finalVideoId), the directive-plan flow historically only flipped status
// to `complete` — so a finished video commission looked like it "did
// nothing" even though a render landed in the media library. A video
// history entry's id === its jobId, so the last done video-render step's
// jobId IS the final video id. Null when already set or a video-less plan
// (e.g. a comic) — which also keeps a re-entry from re-writing state.
const videoId = project.finalVideoId ? null : lastRenderedVideoJobId(project.plan);
if (project.status !== 'complete' || videoId) {
const patch = { status: 'complete', ...(videoId ? { finalVideoId: videoId } : {}) };
await updateProject(projectId, patch)
.catch((e) => console.log(`⚠️ CD plan complete for ${projectId} failed: ${e.message}`));
console.log(`✅ CD plan ${projectId} complete${videoId ? ` (final video ${videoId})` : ''}`);
}
return;
}
case 'run':
return runPlanStep(project, action.step);
default:
return;
}
}
async function runPlanStep(project, step) {
step = { ...step, expectedProductionRevision: project.workspace === 'video' ? project.videoWorkRevision || 0 : undefined };
if (project.workspace === 'video') {
const { videoReviewAllowsDispatch } = await import('./videoReview.js');
if (!await videoReviewAllowsDispatch(project.id, ['script-shot-plan', 'references'])) return;
}
const projectId = project.id;
const key = `${projectId}:${step.stepId}`;
if (inflightPlanStep.has(key)) return;
// A persisted 'running' run for this step means a concurrent advance already
// dispatched it — don't double-dispatch.
const alreadyRunning = (project.runs || []).some(
(r) => r.kind === 'plan-step' && r.stepId === step.stepId && r.status === 'running',
);
if (alreadyRunning) return;
inflightPlanStep.add(key);
let run;
try {
if (project.status !== 'rendering') await updateProject(projectId, { status: 'rendering' });
await updatePlanStep(projectId, step.stepId, { ...(step.expectedProductionRevision === undefined ? {} : { expectedProductionRevision: step.expectedProductionRevision }), status: 'running', startedAt: nowISO() });
run = await recordRun(projectId, {
kind: 'plan-step', stepId: step.stepId, toolName: step.toolName, status: 'running',
});
} finally {
// The persisted 'running' step + run row now guard re-entry — release the
// in-memory key so a raced advance sees the persisted state and bails.
inflightPlanStep.delete(key);
}
// Resolve cross-step result references in the args (#2773) so this step can
// consume a prior step's minted id (e.g. a created series' `result.id`). A
// reference that can't be resolved is a planning error (a mis-authored plan) →
// fail the step and route through bounded re-plan, never dispatch a literal
// `{{…}}`. The RESOLVED step flows onward so downstream settle logic (e.g. the
// autopilot listener's `step.args.seriesId`) sees the concrete id, not the ref.
const resolved = resolvePlanStepArgs(step, project.plan);
if (resolved.error) {
await finishRun(projectId, run?.runId, 'failed', resolved.error);
const bumped = (step.retryCount || 0) + 1;
await updatePlanStep(projectId, step.stepId, { ...(step.expectedProductionRevision === undefined ? {} : { expectedProductionRevision: step.expectedProductionRevision }), status: 'failed', retryCount: bumped, result: { error: resolved.error } });
console.error(`❌ CD plan ${projectId}: step "${step.stepId}" ${resolved.error}`);
return handlePlanStepFailure(projectId, { ...step, retryCount: bumped });
}
const resolvedStep = { ...step, args: resolved.args };
console.log(`▶️ CD plan ${projectId}: dispatching step "${step.stepId}" (${step.toolName})`);
// Outside the request lifecycle — catch so a throw becomes a settle, never an
// unhandled rejection.
const targetAbility = project.directive?.constraints?.targetAbility;
const dispatch = await dispatchCreativeTool(resolvedStep.toolName, resolvedStep.args, {
projectId,
...(project.workspace === 'video' ? { videoStepId: step.stepId } : {}),
...(targetAbility ? { targetAbility } : {}),
})
.catch((err) => ({ ok: false, threw: true, error: err.message }));
return settlePlanStepDispatch(projectId, resolvedStep, run?.runId, dispatch);
}
async function settlePlanStepDispatch(projectId, step, runId, dispatch) {
// Gate rejection (autonomy off / over budget / wrapped-service refusal): the
// step did no work. Block it + pause so a human can raise the budget / flip
// autonomy and Resume — the CD never silently retries around a governance stop.
if (!dispatch || (dispatch.ok !== true && !dispatch.threw)) {
const reason = dispatch?.reason || 'rejected';
await finishRun(projectId, runId, 'failed', reason);
await updatePlanStep(projectId, step.stepId, { ...(step.expectedProductionRevision === undefined ? {} : { expectedProductionRevision: step.expectedProductionRevision }), status: 'blocked', result: { reason } });
return pausePlanWithResidual(
projectId,
`Step "${step.stepId}" (${step.toolName}) rejected by the creative gate: ${reason}`,
);
}
// Real tool error → mark failed, then bounded re-plan or pause.
if (dispatch.threw) {
const error = dispatch.error || 'tool error';
await finishRun(projectId, runId, 'failed', error);
const bumped = (step.retryCount || 0) + 1;
await updatePlanStep(projectId, step.stepId, { ...(step.expectedProductionRevision === undefined ? {} : { expectedProductionRevision: step.expectedProductionRevision }), status: 'failed', retryCount: bumped, result: { error } });
return handlePlanStepFailure(projectId, { ...step, retryCount: bumped });
}
// dry-run — the whole plan is walked as a preview (no side effects executed).
if (dispatch.planned || dispatch.mode === 'dry-run') {
await finishRun(projectId, runId, 'completed');
await updatePlanStep(projectId, step.stepId, { ...(step.expectedProductionRevision === undefined ? {} : { expectedProductionRevision: step.expectedProductionRevision }), status: 'done', result: { planned: true } });
return advanceAfterPlanStepSettled(projectId);
}
// Long-running (media render / job): stay `running` until the underlying job
// settles; a completion event re-fires the loop.
if (dispatch.longRunning) {
const jobId = dispatch.result?.jobId;
if (jobId) return armPlanJobListener(projectId, step.stepId, jobId, runId, step.expectedProductionRevision);
// Series Autopilot returns a run handle (`runId`), not a media jobId — settle
// this step off the in-process autopilot event bus (CDO Phase 3, #2185): the
// step stays `running` until the underlying autopilot run reaches a terminal
// frame. A pause surfaces as a BLOCKED step (never auto-retried); an error
// routes through the failure handler. `alreadyRunning` means the tool
// attached to a live run instead of double-starting — we observe it the same.
const seriesId = step.args?.seriesId;
if (step.toolName === AUTOPILOT_TOOL_NAME && seriesId && dispatch.result?.runId) {
return armAutopilotListener(projectId, step.stepId, seriesId, runId, dispatch.result, step.expectedProductionRevision);
}
// A long-running tool with no observable handle at all — mark done on start so
// the plan makes progress; the underlying work continues on its own.
console.log(`⚠️ CD plan ${projectId}: step "${step.stepId}" (${step.toolName}) is long-running with no observable handle — marking done on start`);
await finishRun(projectId, runId, 'completed');
await updatePlanStep(projectId, step.stepId, { ...(step.expectedProductionRevision === undefined ? {} : { expectedProductionRevision: step.expectedProductionRevision }), status: 'done', result: summarizeResult(dispatch.result) });
return advanceAfterPlanStepSettled(projectId);
}
// Synchronous success (free / llm non-long-running).
await finishRun(projectId, runId, 'completed');
await updatePlanStep(projectId, step.stepId, { ...(step.expectedProductionRevision === undefined ? {} : { expectedProductionRevision: step.expectedProductionRevision }), status: 'done', result: summarizeResult(dispatch.result) });
console.log(`✅ CD plan ${projectId}: step "${step.stepId}" done`);
return advanceAfterPlanStepSettled(projectId);
}
// Bounded re-planning (autopilot's convergence-pause contract): on a step
// failure the planner may revise the remaining steps at most MAX_REPLAN_ROUNDS
// times, then the project pauses with residuals for human review.
async function handlePlanStepFailure(projectId, step) {
const project = await getProject(projectId).catch(() => null);
if (project?.workspace === 'video' && (project.videoExecution?.attempts || []).some(attempt => attempt.status === 'uncertain')) {
return pausePlanWithResidual(projectId, 'A Video submission is uncertain. Reconcile its job before retrying.');
}
if (!project || project.status === 'paused' || project.status === 'failed') return;
const rounds = project.plan?.replanRounds || 0;
if (rounds < (project.workspace === 'video' ? project.videoExecution?.limits?.maxReplans ?? 0 : MAX_REPLAN_ROUNDS) && !inflightPlanner.has(projectId)) {
// MAX_REPLAN_ROUNDS alone cannot bound this path: `replanRounds` is bumped by
// applyPlan, so a planner that never PATCHes leaves it at the same value and
// this branch would re-fire forever (#4146). The empty-run gate is what
// terminates that loop.
if (await plannerBlockedOnEmptyRuns(project)) return;
console.log(`🔁 CD plan ${projectId}: step "${step.stepId}" failed — re-planning (round ${rounds + 1}/${MAX_REPLAN_ROUNDS})`);
inflightPlanner.add(projectId);
await dispatchPlanner(projectId);
return;
}
return pausePlanWithResidual(
projectId,
`Step "${step.stepId}" (${step.toolName || 'tool'}) failed after ${rounds} re-plan round(s) — paused for human review`,
);
}
// Register a one-shot listener that settles a long-running plan step when its
// media job reaches a terminal state. Idempotent teardown; closes the race where
// the job already settled between dispatch returning and the listener attaching.
// No local backstop timer needed here (unlike armAutopilotListener below): every
// media job carries its own per-job idle watchdog (mediaJobQueue/index.js) that
// unconditionally forces a 'completed'/'failed'/'canceled' emit within a bounded
// window regardless of job kind, so this listener can never wait forever.
function armPlanJobListener(projectId, stepId, jobId, runId, expectedProductionRevision) {
const key = `${projectId}:${stepId}`;
let fired = false;
const teardown = () => {
mediaJobEvents.off('completed', settle);
mediaJobEvents.off('failed', settle);
mediaJobEvents.off('canceled', settle);
planJobCleanups.delete(key);
};
function settle(job) {
if (fired || job?.id !== jobId) return;
fired = true;
teardown();
const success = job.status === 'completed';
// Runs outside the request lifecycle (media-queue emitter) — never throw out.
maintenance.continueSettlement(async () => {
await finishRun(projectId, runId, success ? 'completed' : 'failed', success ? undefined : `job ${job.status}`);
if (success) {
await updatePlanStep(projectId, stepId, { ...(expectedProductionRevision === undefined ? {} : { expectedProductionRevision }), status: 'done', result: { jobId } });
await advanceAfterPlanStepSettled(projectId);
} else {
await updatePlanStep(projectId, stepId, { ...(expectedProductionRevision === undefined ? {} : { expectedProductionRevision }), status: 'failed', result: { jobId, jobStatus: job.status } });
await handlePlanStepFailure(projectId, { stepId, toolName: '(long-running job)', retryCount: 0 });
}
}).catch((e) => logFailureWithStack(`❌ CD plan ${projectId} job settle for ${stepId} failed`, e));
}
mediaJobEvents.on('completed', settle);
mediaJobEvents.on('failed', settle);
mediaJobEvents.on('canceled', settle);
planJobCleanups.set(key, teardown);
console.log(`⏳ CD plan ${projectId}: step "${stepId}" waiting on job ${String(jobId).slice(0, 8)}`);
// The job may already be terminal (settled between dispatch and attach) — its
// event fired and won't repeat. Re-check and settle immediately if so.
const current = listJobs().find((j) => j.id === jobId);
if (current && (current.status === 'completed' || current.status === 'failed' || current.status === 'canceled')) {
settle(current);
}
}
// Compact summary of an autopilot terminal frame for the plan step `result` —
// carries the run id + the "clean complete but with residual concerns" counters
// the autopilot's own `complete` frame surfaces (#1572/#1573), so a plan reader
// sees the same qualifications an SSE client would.
function summarizeAutopilotResult(payload) {
const out = {};
for (const k of ['runId', 'steps', 'craftGapIssues', 'craftGapFindings', 'editorialCheckErrors']) {
if (payload?.[k] != null) out[k] = payload[k];
}
return out;
}
// Map an autopilot terminal frame onto the plan step's persisted state.
// - complete → step `done`, advance the plan.
// - paused / canceled → step `blocked` (human-review pause; never auto-retried).
// - error → step `failed`, route through bounded re-plan.
// Runs off the event bus / a marker read — outside any request lifecycle — so it
// must never throw out (the caller wraps it, but keep it self-contained).
async function settleAutopilotStep(projectId, stepId, runId, payload, expectedProductionRevision) {
const type = payload?.type;
if (type === 'complete') {
await finishRun(projectId, runId, 'completed');
await updatePlanStep(projectId, stepId, { ...(expectedProductionRevision === undefined ? {} : { expectedProductionRevision }), status: 'done', result: summarizeAutopilotResult(payload) });
console.log(`✅ CD plan ${projectId}: autopilot step "${stepId}" complete`);
return advanceAfterPlanStepSettled(projectId);
}
if (type === 'paused' || type === 'canceled') {
const reason = payload?.reason
|| (type === 'canceled' ? 'autopilot run was canceled' : 'autopilot paused for human review');
await finishRun(projectId, runId, 'failed', reason);
// BLOCKED, not failed — a pause is a human-review gate. The advance loop sees
// the blocked step and pauses the whole plan with the reason; it is never
// silently retried around (the autopilot pause contract).
await updatePlanStep(projectId, stepId, { ...(expectedProductionRevision === undefined ? {} : { expectedProductionRevision }),
status: 'blocked',
result: { reason, residualFindings: payload?.residualFindings || [], pauseKind: payload?.pauseKind || null },
});
console.log(`⏸️ CD plan ${projectId}: autopilot step "${stepId}" blocked: ${reason}`);
return advanceAfterPlanStepSettled(projectId);
}
// error frame.
const error = payload?.error || 'autopilot run error';
await finishRun(projectId, runId, 'failed', error);
await updatePlanStep(projectId, stepId, { ...(expectedProductionRevision === undefined ? {} : { expectedProductionRevision }), status: 'failed', result: { error } });
return handlePlanStepFailure(projectId, { stepId, toolName: '(series autopilot)', retryCount: 0 });
}
// Read the persisted autopilot marker for the race where the run reached a
// terminal state between dispatch returning and our listener attaching (its
// event fired and won't repeat). Returns a synthetic terminal frame or null when
// the run is still `running`/absent (the live listener will catch the real one).
async function readAutopilotMarker(seriesId) {
const series = await getSeries(seriesId).catch(() => null);
const marker = series?.autopilot;
if (!marker) return null;
if (marker.status === 'done') {
return { type: 'complete', runId: marker.runId, craftGapIssues: marker.craftGapIssues, craftGapFindings: marker.craftGapFindings, editorialCheckErrors: marker.editorialCheckErrors };
}
if (marker.status === 'paused') {
return { type: 'paused', runId: marker.runId, reason: marker.lastError, residualFindings: marker.residualFindings || [], pauseKind: marker.pauseKind || null };
}
if (marker.status === 'error') {
return { type: 'error', runId: marker.runId, error: marker.lastError };
}
return null; // 'running' or unknown — not terminal.
}
// Backstop for an autopilot run that goes fully silent after this listener
// attaches — a bug in a step runner, an unbounded internal await, or an
// event-bus mismatch would otherwise leave this listener (and the plan step)
// wedged in `running` forever, since unlike armPlanJobListener's media job
// (always terminated by mediaJobQueue's own per-job idle watchdog,
// server/services/mediaJobQueue/index.js) a whole autopilot run has no
// underlying watchdog of its own. Modeled as IDLE time, not a fixed wall-clock
// cap (mirrors mediaJobQueue's own idle-vs-wall-time choice) because a healthy
// run can legitimately drive many steps for hours — every step:start/
// step:complete/note frame the orchestrator broadcasts (seriesAutopilot/
// orchestrator.js) counts as activity and re-arms the window; only a run that
// stops emitting anything at all trips it.
const AUTOPILOT_LISTENER_IDLE_MS = 45 * 60 * 1000;
// Attach a listener to the in-process autopilot event bus for `seriesId`, settling
// the plan step on the first terminal frame. Idempotent teardown (reuses the
// shared planJobCleanups map so __resetPlanInflightState drops it too). Closes the
// already-terminal race via the persisted marker, mirroring armPlanJobListener.
function armAutopilotListener(projectId, stepId, seriesId, runId, apResult, expectedProductionRevision) {
const key = `${projectId}:${stepId}`;
let fired = false;
let idleTimer;
const teardown = () => {
autopilotEvents.off(seriesId, handler);
clearTimeout(idleTimer);
planJobCleanups.delete(key);
};
const armIdleTimer = () => {
clearTimeout(idleTimer);
idleTimer = setTimeout(() => {
if (fired) return;
fired = true;
teardown();
console.log(`⏱️ CD plan ${projectId}: step "${stepId}" autopilot run for series ${String(seriesId).slice(0, 8)} went silent for ${AUTOPILOT_LISTENER_IDLE_MS}ms — settling failed`);
// Runs outside the request lifecycle (in-process event bus) — never throw out.
(async () => {
await finishRun(projectId, runId, 'failed', `autopilot run stalled — no activity for ${Math.round(AUTOPILOT_LISTENER_IDLE_MS / 60000)}m`);
await updatePlanStep(projectId, stepId, { ...(expectedProductionRevision === undefined ? {} : { expectedProductionRevision }), status: 'failed', result: { error: 'autopilot run stalled (idle backstop)' } });
await handlePlanStepFailure(projectId, { stepId, toolName: '(series autopilot)', retryCount: 0 });
})().catch((e) => logFailureWithStack(`❌ CD plan ${projectId} autopilot idle-backstop settle for ${stepId} failed`, e));
}, AUTOPILOT_LISTENER_IDLE_MS);
idleTimer.unref?.();
};
function handler(payload) {
if (fired || !payload) return;
if (!AUTOPILOT_TERMINAL_TYPES.has(payload.type)) {
// Non-terminal frame (step:start, step:complete, note, …) — the run is
// alive and making progress; re-arm the idle window rather than settling.
armIdleTimer();
return;
}
fired = true;
teardown();
settleAutopilotStep(projectId, stepId, runId, payload, expectedProductionRevision)
.catch((e) => logFailureWithStack(`❌ CD plan ${projectId} autopilot settle for ${stepId} failed`, e));
}
autopilotEvents.on(seriesId, handler);
planJobCleanups.set(key, teardown);
armIdleTimer();
console.log(`⏳ CD plan ${projectId}: step "${stepId}" attached to autopilot run for series ${String(seriesId).slice(0, 8)}${apResult?.alreadyRunning ? ' (already running)' : ''}`);
// The run may already have reached a terminal state (settled between dispatch
// returning and this attach) — its event fired and won't repeat. Re-check via
// isAutopilotActive + the persisted marker and settle immediately if so.
if (!isAutopilotActive(seriesId)) {
(async () => {
const marker = await readAutopilotMarker(seriesId);
if (marker && !fired) {
fired = true;
teardown();
await settleAutopilotStep(projectId, stepId, runId, marker, expectedProductionRevision);
}
})().catch((e) => logFailureWithStack(`❌ CD plan ${projectId} autopilot marker settle for ${stepId} failed`, e));
}
}
// Test-only: clear module-level dedup + tear down any armed media-job listeners
// so a suite that leaves a long-running step pending can't leak a live listener
// into a later test.
export function __resetPlanInflightState() {
for (const teardown of planJobCleanups.values()) teardown();
planJobCleanups.clear();
inflightPlanner.clear();
inflightPlanStep.clear();
}