Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

24.1 KB
Raw Blame
/**
 * Series Autopilot — run session plumbing (#2842 split of seriesAutopilot.js):
 * SSE client attach/broadcast, cancellation, cleanup, the persisted run marker,
 * CoS gap filing, pause notifications, provider option shaping and the budget gate.
 */

import { attachSseClient, broadcastSse, SSE_CLEANUP_DELAY_MS } from '../../../lib/sseUtils.js';
import * as cosTaskStore from '../../cosTaskStore.js';
import { getDomainBudgetStatus, recordDomainUsage } from '../../domainUsage.js';
import { addNotification, removeByMetadata, NOTIFICATION_TYPES, PRIORITY_LEVELS } from '../../notifications.js';
import { getSeries, updateSeries } from '../series.js';
import * as volumeBeatsRunner from '../volumeBeatsRunner.js';
import * as autoRunner from '../autoRunner.js';
import { patchRunMetadata, stopRun } from '../../runner.js';
import { runs, autopilotEvents, noteSignal, noteProgress, snapshotProgress } from './state.js';

// ---------------------------------------------------------------------------
// Run registry helpers (mirror editorialAnalysisRunner.js).
// ---------------------------------------------------------------------------

export function isAutopilotActive(seriesId) {
  const run = runs.get(seriesId);
  return !!run && !run.finished;
}

export function isAutopilotPauseRequested(seriesId) {
  const run = runs.get(seriesId);
  return !!run && !run.finished && run.pauseRequested === true;
}

// The `start` frame of an IN-FLIGHT run (null when none is active). Everything
// a client needs to describe a run it didn't watch begin — mode (the dry-run
// badge), target, the resolved run provider/model — lives on that frame, but
// attachSseClient replays only the LAST payload, so a client attaching mid-run
// never sees it. Kept whole (rather than rescuing one field at a time through
// bespoke status keys) so a new start-frame field needs no new plumbing.
export function activeRunStart(seriesId) {
  const run = runs.get(seriesId);
  if (!run || run.finished) return null;
  return run.startPayload || null;
}

// The live progress snapshot of an IN-FLIGHT run (null when none is active) —
// the milestone map's cursor: completed counts per step kind, the step running
// now, and what each gate last verified. Same object the `progress` frame
// carries, so a client attaching mid-run starts from the run's real position
// instead of an empty map.
export function activeRunProgress(seriesId) {
  const run = runs.get(seriesId);
  if (!run || run.finished) return null;
  return snapshotProgress(run);
}

export function attachClient(seriesId, res) {
  return attachSseClient(runs, seriesId, res);
}

export function cancelSeriesAutopilot(seriesId) {
  const run = runs.get(seriesId);
  if (!run || run.finished) return false;
  run.cancelRequested = true;
  // Emit an immediate acknowledgement frame so the UI can switch to a
  // "cancelling…" state right away. Cancellation is cooperative and checked
  // between steps (the terminal `canceled` frame follows once the active
  // step/LLM call returns) — without this ack the user sees no feedback until
  // the loop unwinds, which can be the length of a long in-flight LLM call (#1617).
  broadcastSse(run, { type: 'cancel:acknowledged', runId: run.runId, requestedAt: new Date().toISOString() });
  // Propagate to the currently-delegated child so cancel is responsive
  // mid-step instead of only between steps.
  const child = run.activeChild;
  if (child?.kind === 'beats') volumeBeatsRunner.cancelVolumeBeatsRun(child.id);
  else if (child?.kind === 'text') autoRunner.cancelAutoRun(child.id);
  // Direct staged-LLM steps (arc, episode plan, foundation judge/repair) do
  // not have a delegated child coordinator. Stop their concrete run too; the
  // prompt runner reports fallback run ids through the same lifecycle hook.
  if (run.activeLlmRunId) {
    stopRun(run.activeLlmRunId).catch((err) => {
      console.log(`⚠️ autopilot: active LLM stop failed for ${seriesId.slice(0, 12)}: ${err.message}`);
    });
  }
  return true;
}

// Graceful control for quality-first runs: finish the current top-level step
// (and any provisional repair + independent verification transaction inside
// it), then persist a resumable pause. Unlike Cancel this deliberately does not
// stop the active provider run or delegated child.
export function pauseSeriesAutopilot(seriesId) {
  const run = runs.get(seriesId);
  if (!run || run.finished) return false;
  run.pauseRequested = true;
  broadcastSse(run, {
    type: 'pause:acknowledged',
    runId: run.runId,
    requestedAt: new Date().toISOString(),
  });
  return true;
}

function emitFrame(run, seriesId, payload, sseOpts) {
  broadcastSse(run, payload, sseOpts);
  // Retain the diagnosable frames for the opt-in self-improvement pass. A single
  // tap here (rather than instrumenting each step runner) means any telemetry
  // frame a future step emits is diagnosable evidence for free. No-op unless the
  // run opted in — see state.js#noteSignal (it lives with the run registry it
  // mutates, which is what keeps selfImprove.js out of this module's imports).
  noteSignal(run, payload);
  // Mirror every frame onto the in-process bus (CDO Phase 3, #2185) so a
  // server-side consumer (CD plan step) sees the same progress/pause/terminal
  // frames as an SSE client. Emit is best-effort — a listener throw must never
  // abort the run (this runs inside the fire-and-forget coordinator).
  try {
    autopilotEvents.emit(seriesId, payload);
  } catch (err) {
    console.log(`⚠️ autopilot: event emit failed for ${seriesId.slice(0, 12)}: ${err.message}`);
  }
}

export function broadcast(seriesId, payload) {
  const run = runs.get(seriesId);
  if (!run) return;
  emitFrame(run, seriesId, payload);
  // Milestone map: fold the frame into the run's progress snapshot and, when it
  // moved, follow it with the snapshot itself. Emitted as its own frame (rather
  // than stamped onto every frame) so the existing frame shapes — and the
  // diagnosis evidence read off them — are untouched, and NOT retained for SSE
  // replay: it follows nearly every meaningful frame, so retaining it would take
  // the single replay slot and leave a late-attaching client with a snapshot
  // instead of the frame that says what the run is doing. That client reads the
  // same snapshot off the status route.
  if (noteProgress(run, payload)) {
    emitFrame(run, seriesId, { type: 'progress', runId: run.runId, ...snapshotProgress(run) }, { retain: false });
  }
}

// Broadcast the run's `start` frame AND retain it on the run record, so a
// client attaching mid-run can still read it back via `activeRunStart`.
export function broadcastStart(seriesId, payload) {
  const run = runs.get(seriesId);
  if (run) run.startPayload = payload;
  broadcast(seriesId, payload);
}

export function scheduleCleanup(seriesId, record) {
  record.cleanupTimer = setTimeout(() => {
    if (runs.get(seriesId) !== record) return;
    for (const c of record.clients) c.end();
    runs.delete(seriesId);
  }, SSE_CLEANUP_DELAY_MS);
}

// Thin persisted marker for resume/paused UI + boot recovery. NOT a step
// cursor — see module header. Best-effort; a marker write must never abort a run.
export async function persistMarker(seriesId, patch) {
  const run = runs.get(seriesId);
  const resumable = ['running', 'paused', 'error'].includes(patch.status) && run?.options
    ? {
      includeVisual: run.options.includeVisual !== false,
      ...(run.options.productionScope ? { productionScope: run.options.productionScope } : {}),
      fileGaps: run.options.fileGaps === true,
      ...(run.options.providerOverride ? { providerOverride: run.options.providerOverride } : {}),
      ...(run.options.modelOverride ? { modelOverride: run.options.modelOverride } : {}),
      ...(run.options.effortOverride ? { effortOverride: run.options.effortOverride } : {}),
      ...(run.options.judgeLlm ? { judgeLlm: run.options.judgeLlm } : {}),
      ...(run.options.stageLlm ? { stageLlm: run.options.stageLlm } : {}),
      autoSelectModels: run.options.autoSelectModels === true,
      overrideStagePins: run.options.overrideStagePins === true,
    }
    : null;
  // Milestone map (#4140). Both halves the Autonomous card draws it from live
  // ONLY on the in-memory run record — the projected plan on the retained
  // `start` frame, and the progress snapshot folded onto the record — so a run
  // that paused overnight came back as a resume banner with no map beside it.
  // Carry them on the marker, which outlives the run.
  //
  // Stamped on EVERY marker write that has a plan, not only the terminals: the
  // marker is wholesale-replaced per write, so a terminals-only stamp would
  // leave the map missing for the one interruption that skips a terminal write
  // entirely — a hard restart, whose `running` marker the boot recovery demotes
  // to `paused` by spreading whatever the marker already held. The plan is ~20
  // small rows and the snapshot is keyed by the same step vocabulary, so this
  // costs a couple of KB on a series record that is already tens.
  const plan = Array.isArray(run?.startPayload?.plan) ? run.startPayload.plan : null;
  const progress = plan ? snapshotProgress(run) : null;
  await updateSeries(seriesId, {
    autopilot: {
      ...patch,
      ...(resumable ? { resumeOptions: resumable } : {}),
      ...(plan ? { plan } : {}),
      ...(progress ? { progress } : {}),
      updatedAt: new Date().toISOString(),
    },
  }).catch((err) => {
    console.log(`⚠️ autopilot: marker write failed for ${seriesId.slice(0, 12)}: ${err.message}`);
  });
}

// File a CoS task for a capability/quality gap the autopilot can't resolve on
// its own (a script that won't parse, a render that keeps failing, a stalled
// verify, a run-ending error). Opt-in via `options.fileGaps`; never fires in
// dry-run. The first description line is kept STABLE per (series, gapKind,
// issue) so cosTaskStore.addTask's pending/in_progress dedup collapses repeats
// instead of spamming a task per page / per run. Best-effort — a task-store
// failure must never abort the autopilot.
//
// No `app` is passed. `metadata.app` is WORKSPACE ROUTING — it must name a
// record in `data/apps.json` — not a feature tag. These gaps are work on
// PortOS's own pipeline code, so the correct workspace is the PortOS root,
// which is exactly what an absent `app` resolves to. Passing the feature name
// `'pipeline'` made every gap task unrunnable once the #3180 guard landed:
// `prepareAgentWorkspace` refuses to spawn an agent whose app doesn't resolve
// to a repo path, so the task was filed and then blocked at every spawn
// attempt. Same reasoning as the investigation tasks in agentErrorAnalysis.js.
export async function fileGap(record, sId, { gapKind, issueId = null, summary, context = '' }) {
  if (!record.options.fileGaps || record.mode !== 'execute') return;
  const idTag = `series ${sId}${issueId ? ` issue ${issueId}` : ''}`;
  const description = `Autopilot ${gapKind} gap — ${idTag}\n\n${summary}`;
  const result = await cosTaskStore.addTask({
    description,
    context,
    autopilotGapSeriesId: sId,
    autopilotGapKind: gapKind,
  }, 'user')
    .catch((err) => { console.log(`⚠️ autopilot: fileGap (${gapKind}) failed: ${err.message}`); return null; });
  if (result && !result.duplicate) {
    broadcast(sId, { type: 'gap:filed', gapKind, issueId, taskId: result.id });
  }
}

// Pause escalation (#1615): post an in-app notification when a run pauses so the
// user is told actively, not only when they happen to open the status page. The
// SSE `paused` frame still fires for an attached client; this is the persistent
// out-of-band signal for a user who isn't watching. Opt-out via
// `options.notifyOnPause` / the persisted setting (default on); never fires in
// dry-run. Prior pause notifications for this series are cleared first so a
// resume→pause cycle leaves exactly one current banner instead of a stack, and
// the metadata field is series-scoped so removeByMetadata can't touch unrelated
// notifications. Best-effort — a notification failure must never abort the run.
// Drop any pause banner for this series. Called before posting a fresh one (so a
// resume→pause cycle leaves exactly one) AND when a new execute run starts (so a
// run resumed from a pause that then completes/errors doesn't leave a stale
// "paused" banner + dead resume link). Series-scoped metadata so it can't touch
// unrelated notifications. Best-effort.
export async function clearPauseNotice(sId) {
  await removeByMetadata('autopilotPauseSeriesId', sId).catch(() => {});
}

// The gap task's counterpart to clearPauseNotice: retire any gap task still
// QUEUED for this series when a fresh execute run starts.
//
// A gap task is a snapshot of one pause ("needs human review of the residual
// findings before it can continue"). Once a run is moving again that premise is
// void, but nothing used to retire the task — so CoS kept dispatching agents
// against findings the run had already repaired. On this install that burned ten
// agent sessions across two series, each re-fixing what the previous one fixed.
//
// Worse than waste: the resumed run OWNS the fields those findings name (the
// foundation gate repairs worldbuilding/character/craft itself, and the arc gate
// rewrites `arc.summary` + `season.synopsis`), so an agent hand-editing them
// concurrently clobbers the run's own repair.
//
// Self-correcting by construction: the gate re-judges from live content on the
// way past, and re-files a gap with FRESH findings if it still fails. Clearing
// early can only drop a task whose findings are about to be re-derived.
//
// Scoped to `pending` on purpose — an `in_progress` task has an agent attached,
// and flipping it mid-run would strand that agent and re-open the dedup slot.
// Status flip, never deletion, so the retirement federates (the #2619 precedent
// in cosTaskStore.sweepResolvedFailureTasks). Best-effort: this must never
// abort a run.
export async function clearGapTasks(sId) {
  const { tasks = [] } = await cosTaskStore.getUserTasks().catch(() => ({ tasks: [] }));
  // Legacy gap tasks (filed before `autopilotGapSeriesId` existed — every
  // install upgrading into this has some) carry no metadata handle, so fall back
  // to the first line, which fileGap builds deterministically above.
  const isGapFor = (t) => {
    if (t.metadata?.autopilotGapSeriesId) return t.metadata.autopilotGapSeriesId === sId;
    const line = cosTaskStore.firstLine(t.description);
    return line.startsWith('Autopilot ') && line.includes(` gap — series ${sId}`);
  };
  const stale = tasks.filter((t) => t.status === 'pending' && isGapFor(t));
  let retired = 0;
  for (const task of stale) {
    const updated = await cosTaskStore.updateTask(task.id, {
      status: 'completed',
      metadata: {
        resolution: 'auto-expired',
        autoExpiredReason: 'autopilot-resumed',
        autoExpiredAt: new Date().toISOString(),
      },
    }, 'user').catch(() => null);
    if (updated && !updated.error) retired++;
  }
  if (retired > 0) {
    console.log(`🧹 autopilot: retired ${retired} stale gap task(s) for ${sId.slice(0, 12)} on resume`);
  }
  return retired;
}

export async function notifyPause(record, sId, { reason, pauseKind = null, currentStep = null }) {
  if (record.options.notifyOnPause === false || record.mode !== 'execute') return;
  const series = await getSeries(sId).catch(() => null);
  const seriesName = series?.name || 'a series';
  await clearPauseNotice(sId);
  await addNotification({
    type: NOTIFICATION_TYPES.AUTOPILOT_PAUSED,
    title: `Autopilot paused — ${seriesName}`,
    description: reason || 'The run paused and needs human review before it can continue.',
    priority: PRIORITY_LEVELS.HIGH,
    link: `/pipeline/series/${sId}`,
    metadata: {
      autopilotPauseSeriesId: sId,
      runId: record.runId,
      category: 'autopilot-paused',
      pauseKind,
      currentStep,
    },
  }).catch((err) => { console.log(`⚠️ autopilot: pause notification failed for ${sId.slice(0, 12)}: ${err.message}`); });
}

// Series Autopilot threads BOTH its run provider AND its run model as SOFT
// defaults, NOT hard overrides — so a deliberate per-stage pin (Prompts page /
// stage-config.json) still wins for that stage, matching what verifyComicScript
// already does (#1514 for provider; #1558 for model). Each run-level value lands
// on stageRunner's soft channel (`providerDefault` tier 3 / `modelDefault`): it
// applies only to UNPINNED stages and soft-falls-through (to the active provider
// / the provider's default model) when unavailable, rather than throwing
// PROVIDER_OVERRIDE_UNAVAILABLE or beating a stage's deliberate pin the way a
// hard override would. For the model dimension "unpinned" means a stage carrying
// only a *tier* value (default/quick/coding/heavy) — the run model overrides the
// tier but still loses to a deliberate explicit-model pin (see
// stageRunner.resolveModelHint). Before #1558 the model was threaded as a hard
// `modelOverride`, which let the run model beat even an explicit stage pin.
//
// The run's reasoning effort (#3641) rides the same soft channel as a third
// dimension: `effortDefault` applies only to stages with no `stage.effort` pin,
// and the runner clamps it to (or drops it for) whatever provider actually runs.
//
// The run's `overrideStagePins` option does NOT change the channels below — the
// values here still come from the per-role/per-stage routes. It only stops
// stageRunner from consulting the stage-config pins that would otherwise
// outrank them (lib/stagePinPolicy.js).
//
// Two shapes because the delegated services disagree on field names: the
// arc/episode/verify passes take `providerDefault`/`modelDefault`; the child
// runners (volumeBeatsRunner, autoRunner) and the `providerId`-style services
// take `providerIdDefault`/`modelIdDefault`. Each maps its incoming defaults to
// stageRunner's `providerDefault`/`modelDefault` at the leaf call while keeping
// its existing hard `providerOverride`/`providerId` + `modelOverride`/`model`
// params untouched for manual route callers.
const inheritLlmRoute = (route, base) => {
  if (!route || typeof route !== 'object') return base;
  const providerOverride = route.providerOverride || base.providerOverride;
  // A model id belongs to its provider. When a route changes provider and
  // leaves model blank, use that provider's default instead of carrying a model
  // across. Same-provider and effort-only routes deliberately inherit it.
  const providerChanged = !!route.providerOverride
    && route.providerOverride !== base.providerOverride;
  return {
    providerOverride,
    modelOverride: route.modelOverride || (providerChanged ? undefined : base.modelOverride),
    effortOverride: route.effortOverride || base.effortOverride,
  };
};

export const roleLlm = (record, role = 'creative') => {
  const options = record?.options || {};
  const base = {
    providerOverride: options.providerOverride,
    modelOverride: options.modelOverride,
    effortOverride: options.effortOverride,
  };
  const roleRoute = role === 'judge'
    ? inheritLlmRoute(options.judgeLlm, base)
    : base;
  const inheritedStep = { pilotDraft: 'textStages', pilotReview: 'editorialReview' }[record?.currentStep];
  const stageRoute = options.stageLlm?.[record?.currentStep]?.[role]
    || options.stageLlm?.[inheritedStep]?.[role];
  if (stageRoute && typeof stageRoute === 'object' && Object.keys(stageRoute).length > 0) {
    return inheritLlmRoute(stageRoute, roleRoute);
  }
  // Evidence-based routing (`autoSelectModels`) only fills in for a role left
  // ENTIRELY unrouted — read off `roleRoute`, the route this role would
  // otherwise run on, so the rule has one definition and cannot desync from the
  // inheritance above it.
  //
  // That inheritance is the whole point: with no separate judge configured, the
  // judge route IS the run route, so choosing a run provider/model chooses the
  // judge too — which is exactly what the Options panel promises when "Use a
  // separate model for judging and verification" is unchecked. Deriving this
  // from `options.judgeLlm` alone instead left a run that picked ONE
  // provider/model with creation on the chosen route while every judge/verify
  // call (pipeline-arc-verify above all, plus volume verify, the foundation
  // judge and the editorial checks) was silently re-pointed at the learned one.
  // Reading `roleRoute` also covers the run route the series' own `series.llm`
  // supplied (orchestrator.js resolves that into these options before the run
  // starts), so a series that names its provider keeps it here too — and keeps
  // it identically across a pause, since that resolved route is what the resume
  // marker persists.
  const routeChosen = !!(
    roleRoute.providerOverride || roleRoute.modelOverride || roleRoute.effortOverride
  );
  const learned = options.autoSelectModels === true && !routeChosen
    ? options.modelRecommendations?.[record?.currentStep]?.[role]
    : null;
  if (learned) {
    return {
      providerOverride: learned.providerOverride,
      modelOverride: learned.modelOverride,
      effortOverride: learned.effortOverride,
    };
  }
  return roleRoute;
};

const runLifecycle = (record, role) => ({
  onRunCreated: (runId) => {
    record.activeLlmRunId = runId;
    patchRunMetadata(runId, {
      autopilotSystem: 'series',
      seriesId: record.seriesId,
      autopilotRunId: record.runId || null,
      pipelineStage: record.currentStep || 'unknown',
      pipelineRole: role,
    }).catch(() => {});
    // Close the race where Stop lands after createRun but before provider
    // execution registers its child process.
    if (record.cancelRequested) stopRun(runId).catch(() => {});
  },
  onRunSettled: (runId) => {
    if (record.activeLlmRunId === runId) record.activeLlmRunId = null;
  },
});

export const providerOverrideOpts = (record, role = 'creative') => {
  const llm = roleLlm(record, role);
  return {
    providerDefault: llm.providerOverride,
    modelDefault: llm.modelOverride,
    effortDefault: llm.effortOverride,
    ...runLifecycle(record, role),
  };
};
export const providerIdOpts = (record, role = 'creative') => {
  const llm = roleLlm(record, role);
  return {
    providerIdDefault: llm.providerOverride,
    modelIdDefault: llm.modelOverride,
    effortIdDefault: llm.effortOverride,
    ...runLifecycle(record, role),
    // Multi-candidate draft gate (#2169): bill one cos action per re-roll and stop
    // re-rolling when the daily budget is spent. Only ever invoked by
    // generateStage's runDraftGate on a judgeable stage with draftAttempts > 1 — a
    // no-op for every other stage/run. Check-then-bill so a skipped (budget-out)
    // attempt isn't charged. Returns false to halt further attempts (keep the best
    // so far); true when the attempt may proceed.
    chargeAction: async () => {
      const budget = await getDomainBudgetStatus('cos');
      if (!budget.withinBudget) return false;
      await recordDomainUsage('cos', { actions: 1 });
      return true;
    },
  };
};

// Non-deletion guarantee for every arc-rewriting call this run makes. Once the
// unlock pre-pass has cleared the per-season locks (see ./unlockPass.js), those
// locks can no longer stop an LLM-proposed arc from DROPPING a volume — and
// "unlock so the autopilot can edit" must never become "unlock so it can
// delete". Lives here beside the other record→options fragments so every arc
// path (generateArc, arc auto-resolve, the foundation gate's structure fix)
// spreads the SAME rule instead of each inlining its own copy.
export const seasonPreserveOpts = (record) => ({
  preserveDroppedSeasons: record.options.unlockForRun === true,
});

// Pause result when the cos action budget is exhausted, else null. Used to gate
// EACH billable call inside the multi-call verify/editorial convergence loops —
// the conductor's per-step budget check only fires once before the step, so
// without this a single step could bill several actions past the daily cap.
// gapFiled:true so a budget pause doesn't also file a generic stalled gap
// (mirrors the conductor's own loop-level budget pause, which files none).
export async function budgetPause() {
  const budget = await getDomainBudgetStatus('cos');
  if (budget.withinBudget) return null;
  return { pause: true, gapFiled: true, pauseKind: 'budget', reason: `daily cos ${budget.exceeded || 'actions'} budget reached` };
}