Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

12.0 KB
Raw Blame
/**
 * Creative Director — boot-time recovery.
 *
 * Server restarts (deploys, watchers, OOM kills) abort any in-flight render
 * and tear down the in-memory listeners that runSceneRender attaches. Without
 * recovery the project sits in `rendering` / `stitching` forever — its scene
 * status fields point at jobs that the queue already reclassified as
 * 'failed (interrupted by restart)' but nothing fires advanceAfterSceneSettled
 * to push the project forward.
 *
 * On boot, after the media-job queue reloads its persisted state, we:
 *   1. Find every project that's mid-flight (status in planning/rendering/
 *      stitching).
 *   2. Reset any scenes stuck in `rendering` or `evaluating` back to `pending`
 *      — their listeners are gone, the render is dead, the only sane next
 *      action is to redo them.
 *   3. Mark any persisted `runs[]` row in `running` state as failed —
 *      the agent task behind it died with the previous process. Without
 *      this, advanceAfterSceneSettled's persisted-runs guard (which treats
 *      any non-terminal `treatment` run as "another worker is on it")
 *      would silently refuse to enqueue a replacement and the project
 *      would stay stuck in `planning` forever.
 *   4. Call `advanceAfterSceneSettled` to resume each project. It picks up
 *      from wherever the project stopped: re-renders pending scenes, fires a
 *      fresh evaluate task, runs the stitch, etc.
 */

import { listProjects, updateScene, updatePlanStep, updateProject } from './local.js';
import { listJobs, cancelJob } from '../mediaJobQueue/index.js';
import { inflightRuns, ownedLiveJobs, retireRuns } from './stopProject.js';
import { logFailureWithStack } from '../../lib/failureLogging.js';

// Boot-time coordination: cos.start() (entered from cos.init() when
// alwaysOn/autoStart is configured) calls resetOrphanedTasks(), which
// would respawn stale CD treatment/evaluate tasks BEFORE
// recoverInFlightProjects() has had a chance to retire them via retireRuns.
// Two concurrent agents would then race on the same project. Expose a
// promise that cos.start() awaits before its resetOrphanedTasks call,
// so the order is always:
//   1. recoverInFlightProjects retires stale CD tasks (status='completed').
//   2. resetOrphanedTasks runs and finds nothing CD-related to respawn.
let resolveRecoveryDone;
export const cdRecoveryDone = new Promise((resolve) => {
  resolveRecoveryDone = resolve;
});

// Projects that should be auto-advanced on boot — the user expects them to
// keep running.
const RECOVERABLE_STATUSES = new Set(['planning', 'rendering', 'stitching']);
// Projects whose stale state still needs cleanup but should NOT auto-advance
// on boot. `paused` is here because the user pressed Pause; we still need to
// wipe the dead in-flight state behind the pause so Resume picks up cleanly,
// but we don't want to fire a fresh agent task before the user clicks
// Resume themselves.
const CLEANUP_ONLY_STATUSES = new Set(['paused']);
const STUCK_SCENE_STATUSES = new Set(['rendering', 'evaluating']);

export async function recoverInFlightProjects() {
  // Stamp the recovery start so we can distinguish boot-snapshot jobs
  // (queued by the dead worker before restart, must be canceled) from
  // jobs the user enqueued AFTER recovery began (e.g. clicked Resume
  // mid-recovery — the new job must NOT be canceled as an "orphan").
  // recoverInFlightProjects runs fire-and-forget from index.js and the
  // user can interact during that window.
  const recoveryStartedAt = Date.now();
  const projects = await listProjects();
  const needsCleanup = projects.filter(
    (p) => RECOVERABLE_STATUSES.has(p.status) || CLEANUP_ONLY_STATUSES.has(p.status),
  );
  if (!needsCleanup.length) {
    // Must resolve the gate even on the no-op path or cos.start's
    // resetOrphanedTasks await would sit on its 60s timeout for nothing
    // every daemon start/auto-start.
    resolveRecoveryDone();
    return { resumed: 0 };
  }

  const { advanceAfterSceneSettled } = await import('./completionHook.js');
  const { advanceAfterPlanStepSettled } = await import('./planAdvance.js');
  let resumed = 0;
  const completedAt = new Date().toISOString();
  for (const project of needsCleanup) {
    if (project.workspace === 'video') {
      if (project.videoReplica || !project.videoOwnerInstanceId) continue;
      const { reconcileVideoExecution } = await import('./videoExecution.js');
      await retireRuns(project.id, { runs: inflightRuns(project), reason: 'Video production paused after restart', taskMetadata: { interruptedByRestart: 'true' } });
      await reconcileVideoExecution(project.id, { restarting: true });
      continue;
    }
    // CDO Phase 2 (#2184): a directive-driven project's plan steps stuck in
    // `running` lost their in-memory dispatch/job listener when the process
    // died — reset them to `pending` so the advance loop re-dispatches. (Their
    // stale `plan-step` run rows are reaped by the staleRuns pass below, same as
    // treatment/evaluate runs.)
    const planSteps = Array.isArray(project.plan?.steps) ? project.plan.steps : [];
    const stuckPlanSteps = planSteps.filter((s) => s.status === 'running');
    for (const step of stuckPlanSteps) {
      await updatePlanStep(project.id, step.stepId, { status: 'pending' })
        .catch((e) => console.log(`⚠️ CD recovery: reset plan step ${step.stepId} of ${project.id} failed: ${e.message}`));
    }
    if (stuckPlanSteps.length) {
      console.log(`🔄 CD recovery: ${project.id} reset ${stuckPlanSteps.length} stuck plan step(s) to pending`);
    }

    const scenes = project.treatment?.scenes || [];
    const stuck = scenes.filter((s) => STUCK_SCENE_STATUSES.has(s.status));
    for (const scene of stuck) {
      // Paused projects with a completed render (renderedJobId set) retain their
      // `evaluating` status so completionHook's resume path can re-fire the
      // evaluator against the existing clip instead of forcing a full re-render.
      // Resetting these to `pending` would discard the already-rendered video.
      if (project.status === 'paused' && scene.renderedJobId) continue;
      await updateScene(project.id, scene.sceneId, { status: 'pending' })
        .catch((e) => console.log(`⚠️ CD recovery: reset scene ${scene.sceneId} of ${project.id} failed: ${e.message}`));
    }
    const resetCount = stuck.filter(
      (s) => !(project.status === 'paused' && s.renderedJobId),
    ).length;
    if (resetCount) {
      console.log(`🔄 CD recovery: ${project.id} reset ${resetCount} stuck scene(s) to pending`);
    }
    // Reap stale `running` agent-run rows AND retire the underlying CoS task.
    // The agent task behind each run died with the previous process, but
    // cos.js#resetOrphanedTasks would otherwise see `in_progress` task rows on
    // boot and respawn them — racing the fresh treatment/evaluate task this
    // recovery path will cause to enqueue. `retireRuns` owns that contract (the
    // `'internal'` taskType and the supported terminal status are load-bearing —
    // its comments carry the why); we only supply the audit stamp. Nothing can be
    // alive to kill here: this runs at boot, in a process that just started.
    const staleRuns = inflightRuns(project).filter((r) => r.status === 'running');
    if (staleRuns.length) {
      await retireRuns(project.id, {
        runs: staleRuns,
        reason: 'interrupted by restart',
        taskMetadata: { interruptedByRestart: 'true', recoveredAt: completedAt },
      });
      console.log(`🔄 CD recovery: ${project.id} reaped ${staleRuns.length} stale running run(s)`);
    }
    // Cancel any media-queue jobs owned by this project that
    // initMediaJobQueue() restored from disk. Two reasons: (1) For paused
    // projects the user explicitly stopped, the queued render shouldn't
    // burn GPU on resume. (2) For recovering projects (planning/rendering/
    // stitching) the recovery path resets their scenes to `pending` and
    // advance() will enqueue a fresh render — leaving the prior job alive
    // would race two completions for the same `cd:<projectId>:<sceneId>`
    // owner.
    //
    // We must cancel BOTH queued AND running, not just queued. The queue
    // worker is started by initMediaJobQueue() (see server/index.js boot
    // order) and dequeues restored jobs immediately — by the time this
    // recovery path runs, what was `queued` on disk may already have
    // transitioned to `running` in memory. Cancelling running CD jobs
    // here SIGTERMs the gen process so the freshly-enqueued render
    // doesn't have to compete for GPU memory with a doomed sibling.
    // (Jobs reclassified `failed (interrupted by restart)` by the queue's
    // own boot recovery are already terminal and listJobs returns them
    // archived — they won't match either filter.)
    //
    // Filter to jobs queued BEFORE recovery started — recoverInFlightProjects
    // runs fire-and-forget from index.js, so the user can have already
    // clicked Resume on a paused project and enqueued a fresh render
    // while we're iterating projects here. queuedAt > recoveryStartedAt
    // means it's a brand-new user-initiated job, not a stale snapshot
    // entry, and canceling it would silently kill the user's action.
    // Match BOTH owner conventions: the scene loop tags renders `cd:<id>:<sceneId>`;
    // a Phase-2 plan render step tags media jobs `creative-director:<id>` (the
    // registry's resolveOwner). Both must be canceled so the reset step's fresh
    // dispatch doesn't race a doomed sibling for the same output.
    const orphaned = ownedLiveJobs(listJobs(), project.id)
      .filter((j) => {
        const qa = new Date(j.queuedAt || 0).getTime();
        return qa > 0 && qa < recoveryStartedAt;
      });
    for (const job of orphaned) {
      await cancelJob(job.id)
        .catch((e) => console.log(`⚠️ CD recovery: cancel orphaned ${job.status} job ${job.id} for ${project.id} failed: ${e.message}`));
    }
    if (orphaned.length) {
      console.log(`🔄 CD recovery: ${project.id} canceled ${orphaned.length} orphaned job(s) (mix of queued + already-dequeued, all from pre-recovery snapshot)`);
    }
    // Only auto-advance projects the user expects to still be running.
    // `paused` projects skip this — the cleanup above is enough to make a
    // future Resume click work.
    if (RECOVERABLE_STATUSES.has(project.status)) {
      // CDO Phase 2 (#2184): a directive-driven project resumes through the
      // generalized plan advance loop (deriveNextPlanAction re-derives the next
      // step from the reset plan). Legacy video projects stay on the scene loop.
      if (project.directive) {
        advanceAfterPlanStepSettled(project.id)
          .catch((e) => logFailureWithStack(`❌ CD recovery: plan advance for ${project.id} failed`, e));
        resumed += 1;
        continue;
      }
      // Special case: a project that was mid-stitch when the server died
      // still has status='stitching'. advanceAfterSceneSettled exits
      // early when it sees that status (its stitch dedup guard), so it
      // would never re-fire runStitch. Flip back to 'rendering' so the
      // function re-evaluates from scratch — it'll find all scenes
      // accepted, no inflight, no finalVideoId, and call runStitch
      // (which sets status back to 'stitching' itself before starting).
      if (project.status === 'stitching' && !project.finalVideoId) {
        await updateProject(project.id, { status: 'rendering' })
          .catch((e) => console.log(`⚠️ CD recovery: reset stitching→rendering for ${project.id} failed: ${e.message}`));
      }
      advanceAfterSceneSettled(project.id)
        .catch((e) => logFailureWithStack(`❌ CD recovery: advance for ${project.id} failed`, e));
      resumed += 1;
    }
  }
  console.log(`🔄 CD recovery: resumed ${resumed} in-flight project(s)`);
  resolveRecoveryDone();
  return { resumed };
}

// Test/resume helper: callers that don't run recoverInFlightProjects (unit
// tests, scripts) should still resolve the gate so any await on
// cdRecoveryDone in cos.init doesn't hang forever. Idempotent.
export function markRecoveryDone() {
  resolveRecoveryDone();
}