Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
28.7 KB
Raw Blame
// src/core/remediation/run.ts
// v0.41.18.0 (A1, codex finding #2). Extracted from doctor.ts:runRemediate
// so onboard + MCP run_onboard call the same orchestrator without parsing
// argv or invoking process.exit / console.* directly.
//
// The orchestrator wraps the plan loop with:
//   - BudgetTracker (auto-installed via withBudgetTracker)
//   - Checkpoint resume per A4 amended (matching plan_hash only)
//   - D5 dependency cascade (failed step aborts dependents)
//   - D7 per-step recheck (re-compute plan from fresh health)
//   - Hooks for caller observability (no console.* in the library)

import crypto from 'crypto';
import type { BrainEngine } from '../engine.ts';
import { queueWorkerAlive } from '../minions/no-worker.ts';
import {
  computeRecommendations,
} from '../brain-score-recommendations.ts';
import type { RemediationStep } from '../remediation-step.ts';
import { loadRecommendationContext } from './context.ts';
import { computeRemediationPlan } from './plan.ts';
import { planRepairStepsReport, previewFailureField, runRepairSteps, type RepairPlanStep, type RepairPreviewFailure, type RepairStepResult } from './repairs.ts';
import { OperationError } from '../ops/contract.ts';
import { isManualOnlyStep, manualOnlyFix } from './manual-only.ts';
import type { RemediationCheckpoint } from '../remediation-checkpoint.ts';
import type {
  ManualOnlySkippedStep,
  RemediationHooks,
  RemediationOpts,
  RemediationResult,
  StepResult,
} from './types.ts';

/**
 * Submit ordered Remediation jobs sequentially per D3, with D5 cascade
 * on failure and D7 scoped recheck between steps.
 *
 * PGLite path: synchronous in-process execution (no durable queue).
 *
 * Returns a RemediationResult; never throws on BudgetExhausted (the
 * exhaustion snapshot lives on result.budget_exhausted instead).
 * Other thrown errors propagate.
 *
 * Callers decide exit codes from the result.
 */
/**
 * E3: the job steps a run executes. An unreachable score target skips the
 * PAID steps only (their ids land in `job_steps_skipped.skipped`); free steps
 * still run so the brain improves as far as it can — when a worker serves the
 * queue. With no worker (PGLite, or no supervisor) they would wait out their
 * timeout, so every job step is skipped as before.
 */
function planJobSteps(planned: RemediationStep[], manifest: { job_ids: string[] } | undefined, unreachable?: { target: number; ceiling: number }) {
  const remediable = planned.filter((r) => r.status === 'remediable' && (!manifest || manifest.job_ids.includes(r.id)));
  const workerRuns = !unreachable || queueWorkerAlive('default') === true;
  const freeRecs = workerRuns ? remediable.filter((r) => (r.est_usd_cost ?? 0) === 0) : [];
  if (!unreachable) return { recs: remediable, freeRecs, jobStepsSkipped: undefined };
  return {
    recs: freeRecs, freeRecs,
    jobStepsSkipped: { reason: 'target_unreachable' as const, ...unreachable, skipped: remediable.filter((r) => !freeRecs.includes(r)).map((r) => r.id) },
  };
}

/**
 * Holds manual-only steps out of a run's job steps. `withoutManualOnly`
 * returns the other steps and records each remediable manual-only one once
 * (`manual_only_skipped`), naming the rows of its job an earlier run already
 * queued: they still run once a worker picks them up.
 */
function manualOnlySkips(engine: BrainEngine) {
  const skipped: ManualOnlySkippedStep[] = [];
  const withoutManualOnly = async (steps: RemediationStep[]): Promise<RemediationStep[]> => {
    const manual = steps.filter((s) => isManualOnlyStep(s) && s.status === 'remediable' && !skipped.some((m) => m.id === s.id));
    if (manual.length) {
      const { cliRenderContext, renderAction } = await import('../agent-output.ts');
      const queued = await engine.executeRaw<{ id: number | string; name: string; status: string }>(
        `SELECT id, name, status FROM minion_jobs
          WHERE name = ANY($1::text[]) AND status NOT IN ('completed', 'failed', 'dead', 'cancelled')
          ORDER BY id`,
        [[...new Set(manual.map((s) => s.job))]],
      ).catch(() => []);
      for (const step of manual) {
        const jobs = queued.filter((q) => q.name === step.job).map((q) => ({ id: Number(q.id), status: q.status }));
        skipped.push({
          code: 'manual_only_skipped',
          id: step.id, job: step.job, params: step.params,
          ...(step.est_usd_cost !== undefined ? { est_usd_cost: step.est_usd_cost } : {}),
          why: `${step.job} is manual-only, so this automatic run did not submit it`
            + (jobs.length ? `; ${jobs.length} ${step.job} job(s) an earlier run queued (${jobs.map((j) => `#${j.id} ${j.status}`).join(', ')}) still run when a worker picks them up unless cancelled with gbrain jobs cancel <id>.` : '.'),
          fix: renderAction(manualOnlyFix(step), cliRenderContext()),
          ...(jobs.length ? { queued_jobs: jobs } : {}),
        });
      }
    }
    return steps.filter((s) => !isManualOnlyStep(s));
  };
  return { withoutManualOnly, field: () => (skipped.length ? { manual_only_skipped: skipped } : {}) };
}

/**
 * How a submitted step is awaited. PGLite has no background worker, and a
 * Postgres queue may have none running: when the caller allows it and no
 * registered worker serves the queue, the step runs in this process (as `jobs submit --follow` does)
 * instead of waiting out its timeout for a worker that never comes.
 */
function stepWaiter(engine: BrainEngine, queue: InstanceType<typeof import('../minions/queue.ts').MinionQueue>, allowInline: boolean) {
  const worker = allowInline ? queueWorkerAlive('default') : true;
  const inline = engine.kind === 'pglite' ? worker !== true : worker === false;
  return async (jobId: number, waitOpts: { pollMs: number; timeoutMs: number }) => {
    if (inline) return runJobInline(engine, queue, jobId, waitOpts);
    const { waitForCompletion } = await import('../minions/wait-for-completion.ts');
    return waitForCompletion(queue, jobId, waitOpts);
  };
}

/** One submitted job, executed by an in-process worker until it is terminal. */
async function runJobInline(
  engine: BrainEngine,
  queue: InstanceType<typeof import('../minions/queue.ts').MinionQueue>,
  jobId: number,
  waitOpts: { pollMs: number; timeoutMs: number },
) {
  const { MinionWorker } = await import('../minions/worker.ts');
  const { registerBuiltinHandlers } = await import('../../commands/jobs.ts');
  const { waitForCompletion } = await import('../minions/wait-for-completion.ts');
  const worker = new MinionWorker(engine, { queue: 'default', pollInterval: 100, healthCheckInterval: 0 });
  await registerBuiltinHandlers(worker, engine);
  const running = worker.start();
  try {
    return await waitForCompletion(queue, jobId, waitOpts);
  } finally {
    worker.stop();
    await running;
  }
}

export async function runRemediation(
  engine: BrainEngine,
  opts: RemediationOpts = {},
  hooks: RemediationHooks = {},
): Promise<RemediationResult> {
  const targetScore = opts.targetScore ?? 90;
  const maxJobs = opts.maxJobs ?? Infinity;
  const dryRun = opts.dryRun ?? false;
  const resumeMode = opts.resume ?? false;
  const resumePlanHash = opts.resumePlanHash;
  const repairs = opts.repairs;
  if (repairs?.include && repairs.remote !== false) {
    throw new OperationError('permission_denied', 'Repair steps are PROTECTED: only a trusted local caller on the brain host can include them.',
      'On the brain host, run: gbrain doctor --remediation-plan');
  }

  // Lazy-load orchestration deps so the library entry-point doesn't pay
  // their cost on a --dry-run shortcut path (or when callers only need
  // computeRemediationPlan).
  const {
    BudgetTracker,
    BudgetExhausted,
  } = await import('../budget/budget-tracker.ts');
  const { withBudgetTracker } = await import('../ai/gateway.ts');
  const {
    computePlanHash,
    saveRemediationCheckpoint,
    loadRemediationCheckpoint,
    listRemediationCheckpoints,
    clearRemediationCheckpoint,
  } = await import('../remediation-checkpoint.ts');

  const ctx = await loadRecommendationContext(engine);
  const extraRemediations = opts.extraRemediations ?? [];
  const brainId = repairs ? (await (await import('../repair/core.ts')).resolveRepairScope(engine)).brain_id : undefined;
  let previewFailures: RepairPreviewFailure[] = [];
  const manualOnly = manualOnlySkips(engine);
  const synthetic = (score: number, extra: Partial<RemediationResult> = {}): RemediationResult => ({
    doctor_run_id: crypto.randomUUID(), brain_score_initial: score, brain_score_final: score, brain_score_target: targetScore,
    target_reached: false, submitted: [], aborted_count: 0, ...previewFailureField(previewFailures), ...manualOnly.field(), ...extra,
  });

  // Resume loads its checkpoint first: a checkpoint that records a manifest
  // binds the run to its brain, cap, consent and original steps.
  let cp: RemediationCheckpoint | null = null;
  if (resumeMode) {
    cp = resumePlanHash ? loadRemediationCheckpoint(resumePlanHash) : null;
    if (!cp && !resumePlanHash) {
      for (const e of listRemediationCheckpoints()) {
        const candidate = loadRemediationCheckpoint(e.plan_hash);
        if (candidate?.manifest && brainId && candidate.brain_id === brainId) { cp = candidate; break; }
      }
    }
    if (cp?.manifest && brainId && cp.brain_id !== brainId) {
      hooks.onResumeBrainMismatch?.(cp.plan_hash, cp.brain_id ?? 'unknown', brainId);
      const health = await engine.getHealth();
      return synthetic(health.brain_score, { resume_refused: { reason: 'brain_mismatch', checkpoint_brain_id: cp.brain_id ?? 'unknown', brain_id: brainId, plan_hash: cp.plan_hash } });
    }
  }
  const manifest = cp?.manifest;
  // The cap is cumulative: a resume without --max-usd reuses the recorded cap,
  // and spend already settled by the original run and earlier resumes counts.
  const maxUsd = opts.maxUsd ?? (manifest ? cp?.max_usd ?? undefined : undefined);
  const spentBefore = manifest ? cp?.spent_usd ?? 0 : 0;
  if (manifest) hooks.onResumeCap?.(maxUsd ?? null, spentBefore);
  const includeRepairs = repairs !== undefined && (repairs.include || (manifest !== undefined && cp?.include_repairs === true));

  // Pre-flight ceiling check via the shared plan computation. The score target
  // governs job steps only; repair steps are planned independently of it.
  const initialPlan = await computeRemediationPlan(engine, { targetScore, extraRemediations });
  const repairReport = repairs ? await planRepairStepsReport(engine, { noEmbed: repairs.noEmbed, kinds: manifest ? manifest.repair_kinds as RepairPlanStep['kind'][] : undefined, registry: repairs.registry }) : { steps: [], previewFailures: [] };
  let repairSteps: RepairPlanStep[] = repairReport.steps;
  previewFailures = repairReport.previewFailures;
  // Embeddings a budget stop left behind after re-sealing; the re-sealed pages no longer show up in a repair plan.
  let pendingEmbedSources = includeRepairs && manifest ? [...(cp?.pending_embed_sources ?? [])] : [];
  const initialHealth = await engine.getHealth();
  const { freeRecs, jobStepsSkipped, recs: plannedJobSteps } = planJobSteps(await manualOnly.withoutManualOnly(computeRecommendations(initialHealth, ctx, extraRemediations)), manifest,
    initialPlan.target_unreachable ? { target: targetScore, ceiling: initialPlan.max_reachable_score } : undefined);
  if (jobStepsSkipped && freeRecs.length === 0 && !(includeRepairs && (repairSteps.length || pendingEmbedSources.length))) {
    hooks.onTargetUnreachable?.(targetScore, initialPlan.max_reachable_score);
    return synthetic(initialPlan.brain_score_current, {
      target_unreachable: { target: targetScore, ceiling: initialPlan.max_reachable_score },
      ...(repairs && repairSteps.length ? { repairs: [], repairs_skipped: repairSteps } : {}),
    });
  }
  if (jobStepsSkipped) hooks.onTargetUnreachable?.(targetScore, initialPlan.max_reachable_score);
  let recs: RemediationStep[] = plannedJobSteps;
  const skippedRepairs = includeRepairs ? [] : repairSteps;
  if (!includeRepairs) repairSteps = [];
  if (recs.length === 0 && repairSteps.length === 0 && pendingEmbedSources.length === 0) {
    hooks.onNothingToDo?.(initialHealth.brain_score, targetScore);
    return {
      ...synthetic(initialHealth.brain_score),
      target_reached: initialHealth.brain_score >= targetScore,
      ...(jobStepsSkipped ? { job_steps_skipped: jobStepsSkipped } : {}),
      ...(repairs ? { repairs: [], repairs_skipped: skippedRepairs } : {}),
    };
  }

  // A4 amended: compute plan_hash off the active step ids so the checkpoint
  // binds to THIS plan. A checkpoint with a manifest keeps its original hash.
  // The brain id is part of the hash so two brains on one host never share a checkpoint file.
  const planHash = cp?.manifest ? cp.plan_hash : computePlanHash([...recs.map((r) => r.id), ...repairSteps.map((r) => r.id), ...(brainId ? [`brain:${brainId}`] : [])]);
  const originalManifest = manifest ?? { job_ids: recs.map((r) => r.id), repair_kinds: repairSteps.map((r) => r.kind) };
  let completedFromCheckpoint = new Set<string>();
  if (resumeMode) {
    if (!cp && !resumePlanHash) {
      // Legacy checkpoints (no manifest) resume only the identical job plan.
      for (const e of listRemediationCheckpoints()) {
        const candidate = loadRemediationCheckpoint(e.plan_hash);
        if (candidate && !candidate.manifest && candidate.plan_hash === planHash) { cp = candidate; break; }
      }
    }
    if (!cp || (!cp.manifest && cp.plan_hash !== planHash)) {
      hooks.onResumeMissed?.(planHash, resumePlanHash);
      // Surface as a synthetic result so the CLI shell can exit 2.
      return synthetic(initialHealth.brain_score, { target_unreachable: { target: targetScore, ceiling: initialPlan.max_reachable_score } });
    }
    completedFromCheckpoint = new Set(cp.completed.map((c) => c.id));
    repairSteps = repairSteps.filter((step) => !completedFromCheckpoint.has(step.id));
    hooks.onResumeLoaded?.(
      planHash,
      completedFromCheckpoint.size,
      recs.filter((r) => !completedFromCheckpoint.has(r.id)).length + repairSteps.length,
    );
  }

  const remainingCap = maxUsd === undefined ? undefined : Math.max(0, maxUsd - spentBefore);
  const estJobUsd = recs.reduce((sum, r) => sum + (r.est_usd_cost ?? 0), 0);
  let jobsBudgetRefused = false;
  if (remainingCap !== undefined && estJobUsd > remainingCap) {
    hooks.onBudgetRefused?.(estJobUsd, remainingCap);
    if (repairSteps.length === 0) return synthetic(initialHealth.brain_score, repairs ? { repairs: [], repairs_skipped: skippedRepairs } : {});
    jobsBudgetRefused = true;
    recs = [];
  }

  if (dryRun) {
    // Dry-run: no submission, just return the plan as a non-empty result.
    // Each rec lands in submitted[] with synthetic 'dry_run' status so the
    // shape stays consistent.
    return {
      ...synthetic(initialHealth.brain_score),
      submitted: [...repairSteps.map((r) => r.id), ...recs.map((r) => r.id)].map((id, i) => ({ step: i + 1, id, job_id: null, status: 'dry_run' })),
      ...(repairs ? { repairs: [], repairs_skipped: skippedRepairs } : {}),
    };
  }

  // Real submission path
  const submitted: StepResult[] = [];
  const abortedIds = new Set<string>();
  const attemptedIds = new Set<string>();
  const doctorRunId = crypto.randomUUID();
  const repairResults: RepairStepResult[] = [];

  const { MinionQueue } = await import('../minions/queue.ts');
  const { openStepImpact, closeStepImpact } = await import('../onboard/impact-capture.ts');
  const isPGLite = engine.kind === 'pglite';
  const queue = new MinionQueue(engine);
  const waitStep = stepWaiter(engine, queue, opts.inlineJobs === true);

  // A4 amended: install a BudgetTracker scope around the plan-step loop so
  // any gateway.chat / embed / rerank inside a Minion handler (synthesize,
  // patterns, consolidate) or a repair step auto-enforces the cap. On
  // BudgetExhausted, the onExhausted callback persists the checkpoint BEFORE
  // the throw propagates; the caller hook surfaces the actionable --resume hint.
  // Repairs run first under their own tracker; job steps then get a tracker capped at what the repairs left,
  // so in-process spend, reserved effect estimates and job spend all draw on one cumulative cap.
  const repairTracker = new BudgetTracker({ label: 'remediation.repairs', maxCostUsd: remainingCap, capSource: opts.capSource });
  let jobTracker: InstanceType<typeof BudgetTracker> | undefined;
  // Effect kinds embed outside any tracker, so their estimate is reserved up front; paid-model spend no tracker here metered is charged once its step reports it.
  let reservedUsd = 0;
  let trackerExhausted = false;
  const stepTrackers: Array<InstanceType<typeof BudgetTracker>> = [];
  const spentThisRun = () => repairTracker.totalSpent + reservedUsd + (jobTracker?.totalSpent ?? 0)
    + stepTrackers.reduce((sum, tracker) => sum + tracker.totalSpent, 0);
  const settledUsd = () => spentBefore + spentThisRun();
  const remainingUsd = () => remainingCap === undefined ? undefined : Math.max(0, remainingCap - spentThisRun());

  let exhaustionSnapshot: NonNullable<RemediationResult['budget_exhausted']> | undefined;
  const saveCheckpoint = () => {
    const completed = [
      ...[...completedFromCheckpoint].map((id) => ({ id, job: '', status: 'completed' })),
      ...submitted.filter((s) => s.status === 'completed' && !completedFromCheckpoint.has(s.id))
        .map((s) => ({ id: s.id, job: '', status: s.status, job_id: s.job_id ?? null })),
      ...repairResults.filter((r) => r.status === 'completed').map((r) => ({ id: r.id, job: 'repair', status: r.status })),
    ];
    saveRemediationCheckpoint({
      schema_version: 1 as const,
      plan_hash: planHash,
      doctor_run_id: doctorRunId,
      target_score: targetScore,
      started_at: new Date().toISOString(),
      completed,
      aborted_at: new Date().toISOString(),
      abort_reason: 'budget_exhausted' as const,
      budget_snapshot: exhaustionSnapshot
        ? { spent: exhaustionSnapshot.spent, cap: exhaustionSnapshot.cap, reason: exhaustionSnapshot.reason, model_id: exhaustionSnapshot.model_id }
        : undefined,
      ...(repairs ? {
        brain_id: brainId, max_usd: maxUsd ?? null, include_repairs: includeRepairs,
        spent_usd: settledUsd(), manifest: originalManifest,
        ...(pendingEmbedSources.length ? { pending_embed_sources: pendingEmbedSources } : {}),
      } : {}),
    });
  };
  const watch = (tracker: InstanceType<typeof BudgetTracker>) => {
    tracker.onExhausted(() => { trackerExhausted = true; });
    tracker.onExhausted(saveCheckpoint);
  };
  watch(repairTracker);

  const runRepairs = async (): Promise<void> => {
    if (!repairs) return;
    if (pendingEmbedSources.length) {
      const { embedStaleForSource } = await import('../embed-stale.ts');
      let status: RepairStepResult['status'] = 'completed';
      let embedded = 0;
      if (remainingUsd() === 0) status = 'budget_refused';
      for (const sourceId of status === 'completed' ? [...pendingEmbedSources] : []) {
        if (trackerExhausted) break;
        let result: Awaited<ReturnType<typeof embedStaleForSource>> | undefined;
        try { result = await embedStaleForSource(engine, sourceId); } catch (error) { if (!(error instanceof BudgetExhausted)) throw error; }
        embedded += result?.embedded ?? 0;
        // A source is done only when the pass finished without leaving chunks behind.
        if (!trackerExhausted && result && !result.failures && result.complete !== false && !result.remaining) {
          pendingEmbedSources = pendingEmbedSources.filter((id) => id !== sourceId);
        }
      }
      if (trackerExhausted) status = 'budget_exhausted';
      else if (status === 'completed' && pendingEmbedSources.length) status = 'stopped';
      repairResults.push({ id: 'repair:safe-chunks:embeddings', kind: 'safe-chunks', status, applied: embedded, skipped: 0, failed: 0,
        ...(status === 'budget_exhausted' || status === 'budget_refused' ? { message: 'The --max-usd budget does not cover the embeddings of re-sealed pages; resume with a higher cap to continue.' }
          : status === 'stopped' ? { message: `Some re-sealed pages still lack embeddings (${pendingEmbedSources.join(', ')}); check the embedding provider, then run gbrain embed --stale.` } : {}) });
    }
    if (repairSteps.length === 0) return;
    for (const step of repairSteps) hooks.onRepairStepStart?.(step);
    const results = await runRepairSteps(engine, repairSteps, { remote: repairs.remote, noEmbed: repairs.noEmbed, remainingUsd, registry: repairs.registry,
      charge: (usd) => { reservedUsd += usd; }, spentUsd: spentThisRun, exhausted: () => trackerExhausted,
      stepBudget: async (run) => {
        const tracker = new BudgetTracker({ label: 'remediation.repair-step', maxCostUsd: remainingUsd(), capSource: opts.capSource });
        stepTrackers.push(tracker);
        watch(tracker);
        return withBudgetTracker(tracker, run);
      },
      onStep: (step, result) => hooks.onRepairStepEnd?.(step, result) });
    repairResults.push(...results);
    // A re-seal whose embeddings the budget cut short resumes as an embedding pass over the same sources.
    if (results.some((r) => r.kind === 'safe-chunks' && r.status === 'budget_exhausted')) {
      pendingEmbedSources = [...new Set([...pendingEmbedSources, ...(await (await import('../repair/core.ts')).resolveRepairScope(engine)).source_ids])];
    }
  };

  const runLoop = async (): Promise<void> => {
    let stepCount = 0;
    const totalSteps = recs.length;
    while (recs.length > 0 && stepCount < maxJobs) {
      const step = recs[0];
      if (!step) break;
      stepCount++;

      // Resume: skip steps that the checkpoint already marked completed.
      if (completedFromCheckpoint.has(step.id)) {
        const result: StepResult = { step: stepCount, id: step.id, job_id: null, status: 'completed' };
        submitted.push(result);
        attemptedIds.add(step.id);
        hooks.onStepEnd?.(result);
        recs.shift();
        continue;
      }

      // D5: if depends_on intersects aborted, skip + cascade
      if (step.depends_on && step.depends_on.some((d: string) => abortedIds.has(d))) {
        const result: StepResult = { step: stepCount, id: step.id, job_id: null, status: 'skipped_dep_aborted' };
        submitted.push(result);
        abortedIds.add(step.id);
        attemptedIds.add(step.id);
        hooks.onStepEnd?.(result);
        recs.shift();
        continue;
      }

      hooks.onStepStart?.(stepCount, totalSteps, step);
      try {
        const impact = await openStepImpact(engine, step);
        const isProtected = !!step.protected;
        const submitWith = (key: string) =>
          queue.add(
            step.job,
            { ...step.params, doctor_run_id: doctorRunId },
            {
              queue: 'default',
              idempotency_key: key,
              max_attempts: 2,
              maxWaiting: 1,
            },
            isProtected ? { allowProtectedSubmit: true } : undefined,
          );
        let job = await submitWith(step.idempotency_key);
        let dedupedJobId: number | undefined;
        if (job.coalesced && (job.status === 'completed' || job.status === 'failed')) {
          // #3626: the content-hash key never rotates and the queue frees a
          // key only for dead/cancelled rows — a completed/failed row from a
          // PRIOR run holds it forever, so every later --remediate "ran" this
          // step as an instant no-op against the old terminal row. Rotate
          // ONCE onto this run's id so the work actually re-executes.
          // Waiting/active rows still coalesce (dedupe onto in-flight work).
          dedupedJobId = job.id;
          job = await submitWith(`${step.idempotency_key}:r:${doctorRunId}`);
        }
        const submittedResult: StepResult = {
          step: stepCount,
          id: step.id,
          job_id: job.id,
          status: 'submitted',
          ...(job.coalesced === true ? { coalesced: true } : {}),
          ...(dedupedJobId !== undefined ? { deduped_job_id: dedupedJobId } : {}),
        };
        submitted.push(submittedResult);

        const terminal = await waitStep(job.id, { pollMs: isPGLite ? 250 : 1000, timeoutMs: (step.est_seconds + 60) * 1000 });
        submittedResult.status = terminal.status;
        if (impact) await closeStepImpact(engine, impact, step, { jobId: job.id, status: terminal.status, doctorRunId });
        if (terminal.status !== 'completed') abortedIds.add(step.id);
        hooks.onStepEnd?.(submittedResult);
      } catch (e) {
        if (e instanceof BudgetExhausted) {
          exhaustionSnapshot = {
            spent: e.spent,
            cap: e.cap,
            reason: e.reason,
            model_id: e.modelId,
            plan_hash: planHash,
          };
          throw e;
        }
        const errResult: StepResult = {
          step: stepCount,
          id: step.id,
          job_id: null,
          status: `error: ${(e as Error).message.slice(0, 100)}`,
        };
        submitted.push(errResult);
        abortedIds.add(step.id);
        hooks.onStepEnd?.(errResult);
      }

      attemptedIds.add(step.id);
      recs.shift();
      // D7: scoped recheck — re-compute plan from fresh health snapshot.
      // Queue-level max_attempts handles retries within a submitted attempt.
      // A stuck health signal regenerates the same stable id, so keep ids this
      // run already attempted out of the refreshed list to avoid re-enqueueing
      // them forever.
      if (recs.length === 0 || stepCount >= maxJobs) break;
      const freshHealth = await engine.getHealth();
      // Extras carry a static status:'remediable' — a fresh health snapshot
      // never ages them out the way health-derived steps drop. Filter out
      // ids this run already processed (any terminal status), or the recheck
      // would resubmit completed extras every iteration, forever.
      const pendingExtras = extraRemediations.filter((r) => !attemptedIds.has(r.id));
      recs = (await manualOnly.withoutManualOnly(computeRecommendations(freshHealth, ctx, pendingExtras)))
        .filter((r) => r.status === 'remediable' && !attemptedIds.has(r.id) && (!manifest || manifest.job_ids.includes(r.id))
          && (!jobStepsSkipped || (r.est_usd_cost ?? 0) === 0));
    }
  };

  let budgetAbort: NonNullable<RemediationResult['budget_exhausted']> | undefined;
  try {
    try { await withBudgetTracker(repairTracker, runRepairs); }
    catch (err) { if (!(err instanceof BudgetExhausted)) throw err; }
    // Job estimates are rechecked against what the repairs left, including reserved effect estimates.
    const afterRepairs = remainingUsd();
    if (recs.length && afterRepairs !== undefined && (trackerExhausted || estJobUsd > afterRepairs)) {
      hooks.onBudgetRefused?.(estJobUsd, afterRepairs);
      jobsBudgetRefused = true;
      recs = [];
    }
    jobTracker = new BudgetTracker({ label: 'remediation.run', maxCostUsd: afterRepairs, capSource: opts.capSource });
    watch(jobTracker);
    await withBudgetTracker(jobTracker, runLoop);
  } catch (err) {
    if (err instanceof BudgetExhausted) {
      budgetAbort = exhaustionSnapshot;
    } else {
      throw err;
    }
  }
  // Tracker snapshots are per phase; report the operator's cumulative cap and settled spend.
  if (budgetAbort) budgetAbort = { ...budgetAbort, spent: settledUsd(), cap: maxUsd ?? budgetAbort.cap };
  // A paid repair step the cap refused (or that ran out mid-step) ends the
  // run as budget-exhausted even though the free steps completed.
  const refusedRepair = repairResults.find((r) => r.status === 'budget_refused' || r.status === 'budget_exhausted');
  if (!budgetAbort && (refusedRepair || jobsBudgetRefused)) {
    budgetAbort = { spent: settledUsd(), cap: maxUsd ?? 0, reason: 'max_usd', plan_hash: planHash };
    exhaustionSnapshot = budgetAbort;
    saveCheckpoint();
  }
  if (budgetAbort) hooks.onBudgetExhausted?.(planHash, budgetAbort);

  // Clear checkpoint on a clean run (no budget abort). Failed steps in the
  // submitted set don't disqualify cleanup; an uncleared health signal can
  // produce the same stable id again in a later run.
  if (!budgetAbort && pendingEmbedSources.length === 0) {
    clearRemediationCheckpoint(planHash);
  } else if (!budgetAbort) {
    // Re-sealed pages whose embeddings did not land stay resumable.
    saveCheckpoint();
  }

  const finalHealth = await engine.getHealth();
  return {
    doctor_run_id: doctorRunId,
    brain_score_initial: initialHealth.brain_score,
    brain_score_final: finalHealth.brain_score,
    brain_score_target: targetScore,
    target_reached: finalHealth.brain_score >= targetScore,
    submitted,
    aborted_count: abortedIds.size,
    budget_exhausted: budgetAbort,
    ...(jobStepsSkipped ? { job_steps_skipped: jobStepsSkipped } : {}),
    ...(repairs ? {
      repairs: repairResults, repairs_skipped: skippedRepairs,
      budget: { max_usd: maxUsd ?? null, spent_usd: settledUsd(), include_repairs: includeRepairs, plan_hash: planHash },
    } : {}),
    ...previewFailureField(previewFailures), ...manualOnly.field(),
  };
}