Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
31.4 KB
Raw Blame
/**
 * Serve-delegated sync job runner — the serve process's execution half of the
 * sync_start / sync_status / sync_abort IPC kinds (wire shapes + validation in
 * context/sync-ipc.ts; socket plumbing in context/resolve-ipc.ts; CLI half in
 * commands/sync-delegate.ts).
 *
 * One job at a time, in-memory, module-singleton. Correctness does NOT rest on
 * the single-flight guard — performSync still takes the `gbrain-sync:<source>`
 * DB row lock — the guard just answers `busy` fast and keeps the event-loop
 * load bounded. The last TERMINAL job is retained (keyed by clientToken) so a
 * client that lost the start ack can attach to its own finished run instead of
 * starting a duplicate.
 *
 *   sync_start(valid, idle, not shutting down)
 *        │
 *   ┌────▼────┐  abort()/deadline/shutdown  ┌──────────┐
 *   │ running │────────────────────────────▶│ aborting │
 *   └──┬───┬──┘                             └────┬─────┘
 *      │   │ performSync throws                  │ settles (typed partial)
 *      ▼   ▼                                     ▼
 *   ┌──────┐ ┌───────┐        done │ error (retained until next start)
 *   │ done │ │ error │
 *   └──────┘ └───────┘
 *
 * Shutdown contract: shutdownDelegatedSync() is an IDEMPOTENT SHARED promise —
 * serve.ts's beginShutdown and mcp/server.ts's shutdown both race here on
 * every signal, and the second caller must await the first, never double-run.
 * It must be awaited BEFORE engine.disconnect(): the disconnect-mode drain
 * passes allowAbort:false and runs after `_db` is early-nulled, so a job's
 * settle writes (final checkpoint flush, row-lock release) only succeed
 * against the still-live engine. The registered BackgroundWorkDrainer is a
 * BACKSTOP for disconnect paths that never called us, not the primary path.
 *
 * Embeds: delegated jobs ALWAYS run with noEmbed (the #2139 inline cost gate
 * lives in runSync, which a delegated run never passes through) and this
 * module drains them afterwards via maybeDrainDeferredEmbeds — the lock owner
 * closes its own embedding loop. NOT submitEmbedBackfill: on PGLite nothing
 * drains minion_jobs (workers refuse the engine) and a stuck `waiting` row
 * cooldown-blocks every later submit.
 *
 * #6317 (managed brains): a sync_start that carries the CLI's durable `cli`
 * writer registration is an authorization hand-off, not a lifted gate. The
 * shared-secret lane still runs under withLegacySyncDelegation, where
 * assertDurableSyncCaller refuses managed sync by design; the verified lane
 * runs under withVerifiedLocalRegistration instead, so managedSyncAuthority
 * sees the CLI's own principal and its revocation and grant checks apply to
 * every admission. A denied, revoked or stdio registration is refused at
 * sync_start with the OperationError envelope (`refusal`). On a managed brain
 * the verified job runs the managed drain (`performSync` with `drain: true`,
 * the CLI's own path) with the drain's human lines captured per job
 * (withHumanLineSink) for sync_status, where the CLI prints them verbatim.
 * The one-job-at-a-time guard, the per-job deadline, abort and shutdown
 * settle are shared by both lanes. Embeds on the managed lane follow the
 * CLI's effective flags (the managed cursor owns processing options).
 */

import { readMarkersForRun, settleRunMarkers } from './notifier/handoff.ts';
import { isAdmissionOrigin, withAdmissionOrigin } from './persistence/admission-origin.ts';
import { randomUUID } from 'node:crypto';
import { withLegacySyncDelegation } from './persistence/sync-authority.ts';
import { retiredWireNotice } from './wire-skew.ts';
import { verifyLocalWriter, withVerifiedLocalRegistration } from './persistence/identity.ts';
import { readManagedMode } from './persistence/managed-mode.ts';
import { OperationError } from './ops/contract.ts';
import { trustedCliRequired } from './ops/op-fix.ts';
import { withHumanLineSink } from './console-prefix.ts';
import type { BrainEngine } from './engine.ts';
import { registerBackgroundWorkDrainer } from './background-work.ts';
import {
  isSyncStartRegistration,
  toWireSyncResult,
  validateDelegatedSyncOptions,
  type DelegatedSyncOptions,
  type DelegatedSyncState,
  type SyncAbortResponse,
  type SyncStartRegistration,
  type SyncStartResponse,
  type SyncStatusLine,
  type SyncStatusResponse,
  type WireSyncResult,
} from './context/sync-ipc.ts';
import type { DrainNext, DrainReport } from './persistence/sync-drain.ts';

interface DelegatedSyncJob {
  id: string;
  clientToken: string;
  state: DelegatedSyncState;
  sourceId?: string;
  startedAt: number;
  finishedAt?: number;
  phase?: string;
  bankedFiles?: number;
  result?: WireSyncResult;
  jobError?: string;
  /** #6317: the job's OperationError envelope, when its failure was one. */
  jobErrorEnvelope?: Record<string, unknown>;
  /** #6317: true when the job ran the managed drain as a verified CLI writer. */
  managed: boolean;
  drain?: DrainReport;
  next?: DrainNext | null;
  /** #6317: the human lines the job printed, numbered from 1; the oldest are dropped past LINE_BUFFER_MAX. */
  lines: SyncStatusLine[];
  lineSeq: number;
  controller: AbortController;
  /** Resolves when the job reaches done/error — the shutdown settle target. */
  settled: Promise<void>;
}

/** Lines retained per job and lines returned per sync_status poll (the response must fit the IPC message cap). */
const LINE_BUFFER_MAX = 2000;
export const LINES_PER_STATUS_MAX = 400;

/** Running job, or the retained last terminal job (replaced by the next start). */
let current: DelegatedSyncJob | null = null;
let shuttingDown = false;
let shutdownPromise: Promise<void> | null = null;
let drainerRegistered = false;

interface DeferredEmbeds {
  pending: Set<string | undefined>;
  controller: AbortController;
  running?: Promise<void>;
  timer?: ReturnType<typeof setTimeout>;
  stopped: boolean;
  unregister: () => void;
  /** Consecutive passes that failed (a provider error or failed chunks); picks the retry tier. */
  failures: number;
  /** Consecutive passes deferred because a delegated sync was running. */
  busy: number;
}
/** Each datastore owns its backlog and actual provider/SQL lifetime. */
const deferredEmbeds = new Map<BrainEngine, DeferredEmbeds>();

/**
 * GBRA-133 2b.3: what the deferred-embed drain decided, oldest first (bounded), for tests and the serve log.
 * `retry` re-arms the drain after a pass left work (progress, a provider failure); `busy` defers while a delegated sync
 * runs, at most DEFERRED_EMBED_BUSY_MAX times in a row before `busy_override` drains beside it; `keyless` keeps the
 * backlog without a provider call or a timer.
 */
export type DeferredEmbedDecision = { decision: 'drained' | 'retry' | 'busy' | 'busy_override' | 'keyless' | 'no_sources'; delayMs?: number; failures?: number; ms?: number };
export const deferredEmbedDecisions: DeferredEmbedDecision[] = [];
const DEFERRED_EMBED_RETRY_TIERS = [1, 3, 12, 60];
const DEFERRED_EMBED_BUSY_MAX = 3;
let deferredEmbedBusyProbe: () => boolean = () => isDelegatedSyncRunning();

/** The retry unit: GBRAIN_DEFERRED_EMBED_RETRY_MS (default 5000; tiers 1x, 3x, 12x, 60x after failures). */
function deferredEmbedRetryMs(env: Record<string, string | undefined> = process.env): number {
  const raw = env.GBRAIN_DEFERRED_EMBED_RETRY_MS;
  const value = raw === undefined || raw === '' ? NaN : Number(raw);
  return Number.isFinite(value) && value >= 1 ? Math.floor(value) : 5_000;
}

/** Records the decision and logs it (GBRA-133: the drain is a concurrent actor beside searches, so every pass outcome is in the log). */
function decideDeferredEmbed(entry: DeferredEmbedDecision): void {
  log(`embed-drain decision=${entry.decision}${entry.delayMs !== undefined ? ` delay_ms=${entry.delayMs}` : ''}`
    + `${entry.failures !== undefined ? ` failures=${entry.failures}` : ''}${entry.ms !== undefined ? ` ms=${entry.ms}` : ''}`);
  deferredEmbedDecisions.push(entry);
  if (deferredEmbedDecisions.length > 200) deferredEmbedDecisions.splice(0, deferredEmbedDecisions.length - 200);
}

function armDeferredEmbeds(engine: BrainEngine, state: DeferredEmbeds, delayMs: number): void {
  if (state.timer || state.stopped || shuttingDown) return;
  state.timer = setTimeout(() => {
    state.timer = undefined;
    if (state.stopped) return;
    void maybeDrainDeferredEmbeds(engine);
  }, delayMs);
  state.timer.unref?.();
}

const isTerminal = (s: DelegatedSyncState): boolean => s === 'done' || s === 'error';

function log(msg: string): void {
  process.stderr.write(`[serve-sync] ${msg}\n`);
}

/**
 * Bound on the shutdown settle wait (ms). serve.ts extends its cleanup
 * deadline by exactly this much while a job is running, so the settle always
 * fits inside the deadline that would otherwise force-exit at 5s.
 */
export function delegatedSyncSettleMs(env: Record<string, string | undefined> = process.env): number {
  const raw = Number(env.GBRAIN_SERVE_SYNC_SETTLE_MS ?? '3000');
  if (!Number.isFinite(raw) || raw < 0) return 3000;
  return Math.min(raw, 60_000);
}

/** True while a delegated job occupies the event loop (sweep ticks skip). */
export function isDelegatedSyncRunning(): boolean {
  return current !== null && !isTerminal(current.state);
}

/**
 * Handle a sync_start request. `rawOptions` is the untrusted wire payload —
 * validated here (the runner is the single authority) even though the IPC
 * layer types it. `opts.registration` is the untrusted #6317 hand-off
 * payload; when present the start is asynchronous (one verification read).
 */
export function startDelegatedSync(engine: BrainEngine, rawOptions: unknown, clientToken: string, opts?: { boundSourceId?: string }): SyncStartResponse;
/**
 * `opts.logLines` (in-process callers only, never the wire): also write the job's human lines to the serve log, for a
 * run nobody polls (the resident notifier's background passes, #7243), whose phase and progress lines would otherwise
 * reach only the job's line buffer.
 */
export function startDelegatedSync(engine: BrainEngine, rawOptions: unknown, clientToken: string, opts: { boundSourceId?: string; registration?: unknown; logLines?: boolean }): SyncStartResponse | Promise<SyncStartResponse>;
export function startDelegatedSync(
  engine: BrainEngine,
  rawOptions: unknown,
  clientToken: string,
  opts: { boundSourceId?: string; registration?: unknown; logLines?: boolean } = {},
): SyncStartResponse | Promise<SyncStartResponse> {
  // #6696 (A5): the client's admission origin scopes every write of the job (validated below; a client that sends none is background).
  const origin = (rawOptions as { origin?: unknown } | null)?.origin;
  return withAdmissionOrigin(isAdmissionOrigin(origin) ? origin : 'background', () => startDelegatedSyncScoped(engine, rawOptions, clientToken, opts));
}

function startDelegatedSyncScoped(
  engine: BrainEngine,
  rawOptions: unknown,
  clientToken: string,
  opts: { boundSourceId?: string; registration?: unknown; logLines?: boolean },
): SyncStartResponse | Promise<SyncStartResponse> {
  if (typeof clientToken !== 'string' || clientToken.length === 0 || clientToken.length > 128) {
    return { ok: false, protocol: 2, error: 'invalid_options:clientToken' };
  }
  // Token attach: a retry after a lost ack finds its own job — running
  // (attach and keep polling) or retained-terminal (fetch the result) —
  // instead of erroring or duplicate-running.
  if (current && current.clientToken === clientToken) {
    return isTerminal(current.state)
      ? { ok: true, protocol: 2, jobId: current.id, completed: true }
      : { ok: true, protocol: 2, jobId: current.id };
  }
  if (shuttingDown) {
    return { ok: false, protocol: 2, error: 'shutting_down' };
  }
  if (current && !isTerminal(current.state)) {
    return { ok: false, protocol: 2, error: 'busy', jobId: current.id };
  }
  if (opts.registration !== undefined && !isSyncStartRegistration(opts.registration)) {
    return { ok: false, protocol: 2, error: 'invalid_options:registration' };
  }
  const v = validateDelegatedSyncOptions(rawOptions);
  if (!v.ok) return { ok: false, protocol: 2, error: v.error };
  const options = v.options;
  if (options.sourceId && opts.boundSourceId && options.sourceId !== opts.boundSourceId) {
    return { ok: false, protocol: 2, error: 'source_mismatch' };
  }

  const controller = new AbortController();
  const job: DelegatedSyncJob = {
    id: randomUUID(),
    clientToken,
    state: 'running',
    sourceId: options.sourceId ?? opts.boundSourceId,
    startedAt: Date.now(),
    managed: false,
    lines: [],
    lineSeq: 0,
    controller,
    settled: Promise.resolve(),
  };
  current = job;
  // An older client's retired fields (src/core/wire-skew.ts) were dropped: say so in the serve log and to that client.
  if (v.ignored) {
    const notice = retiredWireNotice(v.ignored);
    log(notice);
    job.lines.push({ seq: ++job.lineSeq, text: notice });
  }
  ensureDrainerRegistered();

  // Per-job hard deadline: 0 is the single unbounded encoding (an explicit
  // --no-hard-deadline); every other value keeps the job bounded even when
  // the polling client died.
  let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
  if (options.timeoutSeconds > 0) {
    deadlineTimer = setTimeout(() => {
      if (!isTerminal(job.state) && !job.result) {
        log(`abort job=${job.id} reason=deadline after ${options.timeoutSeconds}s`);
        job.state = 'aborting';
        controller.abort();
      }
    }, options.timeoutSeconds * 1000);
    deadlineTimer.unref?.();
  }
  const settle = (): void => {
    if (deadlineTimer) clearTimeout(deadlineTimer);
    job.finishedAt = Date.now();
  };
  const fail = (e: unknown): void => {
    job.jobError = e instanceof Error ? e.message : String(e);
    if (e instanceof OperationError) job.jobErrorEnvelope = e.toJSON();
    job.state = 'error';
    log(`error job=${job.id}: ${job.jobError}`);
  };

  if (opts.registration !== undefined) {
    return startVerifiedDelegatedSync(engine, job, options, opts.registration as SyncStartRegistration, { settle, fail }, opts.logLines === true);
  }

  job.settled = withLegacySyncDelegation(async () => {
    try {
      await (await import('./persistence/maintenance.ts')).assertUnmanagedCanonicalWriter(engine, 'shared-secret sync delegation');
      const { performSync } = await import('../commands/sync.ts');
      const sourceId = await resolveJobSource(engine, job);
      log(
        `start job=${job.id} source=${sourceId ?? 'default'} ` +
        `opts=${JSON.stringify({ ...options, timeoutSeconds: undefined })} deadline=${options.timeoutSeconds}s`,
      );
      const markers = options.dryRun ? [] : await readMarkersForRun(engine, sourceId ?? 'default');
      const r = await performSync(engine, {
        ...classicSyncOpts(options, sourceId),
        // ALWAYS deferred — see module header. Drained by maybeDrainDeferredEmbeds.
        noEmbed: true,
        signal: controller.signal,
        onProgress: (p) => {
          job.phase = p.phase;
          if (p.bankedFiles !== undefined) job.bankedFiles = p.bankedFiles;
        },
      });
      job.result = toWireSyncResult(r);
      log(
        `done job=${job.id} status=${r.status} added=${r.added} modified=${r.modified} ` +
        `deleted=${r.deleted} in=${Math.round((Date.now() - job.startedAt) / 1000)}s`,
      );
      if (!options.dryRun) await settleRunMarkers(engine, sourceId ?? 'default', markers, { pulled: !options.noPull, full: options.full === true }, { queue: false });
      // Wire noEmbed means the USER declined embeds — honor it by skipping
      // the drain too (performSync above always ran noEmbed regardless).
      if (!options.dryRun && !options.noEmbed && r.added + r.modified > 0) {
        scheduleDeferredSyncEmbeds(engine, sourceId);
      }
      job.state = 'done';
    } catch (e) {
      fail(e);
    } finally {
      settle();
    }
  });

  return { ok: true, protocol: 2, jobId: job.id };
}

async function resolveJobSource(engine: BrainEngine, job: DelegatedSyncJob): Promise<string | undefined> {
  if (job.sourceId) return job.sourceId;
  const { resolveSourceWithTier } = await import('./source-resolver.ts');
  job.sourceId = (await resolveSourceWithTier(engine, null)).source_id;
  return job.sourceId;
}

function classicSyncOpts(options: DelegatedSyncOptions, sourceId: string | undefined) {
  return {
    sourceId,
    dryRun: options.dryRun,
    full: options.full,
    noPull: options.noPull,
    noExtract: options.noExtract,
    noSchemaPack: options.noSchemaPack,
    skipFailed: options.skipFailed,
    retryFailed: options.retryFailed,
    includeGitignored: options.includeGitignored,
  };
}

/**
 * #6317: the verified lane. The registration is checked before the start is
 * acknowledged (a refusal answers sync_start itself, with the envelope); the
 * job then runs under withVerifiedLocalRegistration, which re-verifies at
 * entry, so a registration revoked between the two reads never runs either.
 * The job slot is already reserved (`current`), so a concurrent sync_start
 * answers `busy` while the verification read is in flight.
 */
async function startVerifiedDelegatedSync(
  engine: BrainEngine,
  job: DelegatedSyncJob,
  options: DelegatedSyncOptions,
  registration: SyncStartRegistration,
  hooks: { settle: () => void; fail: (e: unknown) => void },
  logLines = false,
): Promise<SyncStartResponse> {
  const refuse = (e: unknown): SyncStartResponse => {
    hooks.fail(e);
    hooks.settle();
    const envelope = e instanceof OperationError ? e.toJSON() : undefined;
    return { ok: false, protocol: 2, jobId: job.id, error: e instanceof OperationError ? e.code : 'permission_denied', ...(envelope ? { refusal: envelope } : {}) };
  };
  let verified: Awaited<ReturnType<typeof verifyLocalWriter>>;
  try { verified = await verifyLocalWriter(engine, registration); }
  catch (e) { return refuse(e); }
  if (verified.remote || verified.principal.kind !== 'local_cli') {
    return refuse(trustedCliRequired('Serve-delegated managed sync requires this host\'s trusted CLI registration; a stdio writer registration cannot run it.'));
  }
  job.managed = true;
  const push = (text: string): void => {
    if (logLines) log(`job=${job.id} ${text}`);
    job.lines.push({ seq: ++job.lineSeq, text });
    if (job.lines.length > LINE_BUFFER_MAX) job.lines.splice(0, job.lines.length - LINE_BUFFER_MAX);
  };
  job.settled = (async () => {
    try {
      await withVerifiedLocalRegistration(engine, registration, () => withHumanLineSink(push, async () => {
        const { performSync } = await import('../commands/sync.ts');
        const sourceId = await resolveJobSource(engine, job);
        const managed = await readManagedMode(engine);
        const markers = options.dryRun ? [] : await readMarkersForRun(engine, sourceId ?? 'default');
        log(
          `start job=${job.id} source=${sourceId ?? 'default'} writer=${verified.principal.id} managed=${managed} ` +
          `opts=${JSON.stringify({ ...options, timeoutSeconds: undefined })} deadline=${options.timeoutSeconds}s`,
        );
        const r = await performSync(engine, {
          ...classicSyncOpts(options, sourceId),
          // The managed drain is the CLI's own path: the cursor owns processing options, so the CLI's effective
          // flags ride through; the classic path keeps the always-deferred embed rule of the legacy lane.
          ...(managed
            ? { drain: true, noBulk: options.noBulk, lanes: options.lanes, noEmbed: options.noEmbed, embedDeferred: options.embedDeferred,
              // #6712 PR 1: the CLI's run follows a live drain-lease holder inside the serve, as it would in its own process.
              followDrainLease: true, drainHolder: { kind: 'serve' as const, id: String(job.id) } }
            : { noEmbed: true }),
          signal: job.controller.signal,
          onProgress: (p) => {
            job.phase = p.phase;
            if (p.bankedFiles !== undefined) job.bankedFiles = p.bankedFiles;
          },
        });
        job.result = toWireSyncResult(r);
        job.drain = r.drain;
        let stopFields = '';
        if (managed) {
          const { drainNext, drainStopFields } = await import('./persistence/sync-drain.ts');
          stopFields = drainStopFields(r);
          const { managedSyncResumeArgs, syncResumeCommand } = await import('./sync-reconcile.ts');
          const id = sourceId ?? 'default';
          job.next = drainNext(r, syncResumeCommand(managedSyncResumeArgs({ sourceId: id, processingOptions: options })), id);
        }
        log(
          `done job=${job.id} status=${r.status} added=${r.added} modified=${r.modified} ` +
          `deleted=${r.deleted}${r.drain ? ` drain=${r.drain.outcome}` : ''}${stopFields} in=${Math.round((Date.now() - job.startedAt) / 1000)}s`,
        );
        if (stopFields && job.next) log(`next job=${job.id}: ${job.next.command} (${job.next.why})`);
        // GBRA-133 PR 2a.3: the post-run re-check (generation-checked); whatever is still due goes to this serve's notifier.
        if (!options.dryRun) await settleRunMarkers(engine, sourceId ?? 'default', markers, { pulled: !managed && !options.noPull, full: options.full === true }, { queue: false });
        if (!managed && !options.dryRun && !options.noEmbed && r.added + r.modified > 0) {
          scheduleDeferredSyncEmbeds(engine, sourceId);
        }
        job.state = 'done';
      }));
    } catch (e) {
      hooks.fail(e);
    } finally {
      hooks.settle();
    }
  })();
  return { ok: true, protocol: 2, jobId: job.id };
}

/** Handle a sync_status poll. `afterLine` is the client's line cursor (#6317); lines before it are not resent. */
export function getDelegatedSyncStatus(jobId: string, afterLine = 0): SyncStatusResponse {
  const job = current;
  if (!job || job.id !== jobId) {
    return { ok: false, protocol: 2, error: 'unknown_job' };
  }
  const since = Number.isFinite(afterLine) ? afterLine : 0;
  const lines = job.lines.filter(line => line.seq > since).slice(0, LINES_PER_STATUS_MAX);
  return {
    ok: true,
    protocol: 2,
    state: job.state,
    sourceId: job.sourceId,
    startedAt: job.startedAt,
    elapsedMs: (job.finishedAt ?? Date.now()) - job.startedAt,
    phase: job.phase,
    bankedFiles: job.bankedFiles,
    ...(job.managed ? { managed: true, lines, lineSeq: job.lineSeq } : {}),
    ...(job.drain ? { drain: job.drain } : {}),
    ...(job.next !== undefined ? { next: job.next } : {}),
    ...(job.state === 'done' && job.result ? { result: job.result } : {}),
    ...(job.state === 'error' && job.jobError ? { jobError: job.jobError } : {}),
    ...(job.state === 'error' && job.jobErrorEnvelope ? { jobErrorEnvelope: job.jobErrorEnvelope } : {}),
  };
}

/** Handle a sync_abort request — cooperative; the client keeps polling to settle. */
export function abortDelegatedSync(jobId: string): SyncAbortResponse {
  const job = current;
  if (!job || job.id !== jobId) {
    return { ok: false, protocol: 2, error: 'unknown_job' };
  }
  if (!isTerminal(job.state) && !job.result) {
    log(`abort job=${job.id} reason=client`);
    job.state = 'aborting';
    job.controller.abort();
  }
  return { ok: true, protocol: 2, state: job.state };
}

/**
 * Idempotent shutdown settle: abort the running job and wait (bounded) for its
 * settle writes to land against the still-live engine. Both serve shutdown
 * paths call this BEFORE engine.disconnect(); the second caller awaits the
 * first run. New sync_start requests refuse with 'shutting_down' from the
 * first call onward. Never throws.
 */
export function shutdownDelegatedSync(timeoutMs?: number): Promise<void> {
  shuttingDown = true;
  if (shutdownPromise) return shutdownPromise;
  shutdownPromise = (async () => {
    for (const state of deferredEmbeds.values()) {
      state.stopped = true;
      state.controller.abort();
      if (state.timer) clearTimeout(state.timer);
    }
    const job = current;
    if (!job || isTerminal(job.state)) return;
    log(`abort job=${job.id} reason=shutdown`);
    job.state = 'aborting';
    job.controller.abort();
    const bound = timeoutMs ?? delegatedSyncSettleMs();
    await Promise.race([
      job.settled,
      new Promise<void>((r) => { const t = setTimeout(r, bound); t.unref?.(); }),
    ]);
    if (!isTerminal(job.state)) {
      log(`shutdown settle bound (${timeoutMs ?? delegatedSyncSettleMs()}ms) elapsed with job=${job.id} unsettled; proceeding (checkpoint resume covers the tail)`);
    }
  })().catch(() => { /* never throws */ });
  return shutdownPromise;
}

/**
 * BACKSTOP for disconnect paths that never awaited shutdownDelegatedSync().
 * The disconnect-mode drain runs with allowAbort:false after `_db` is nulled,
 * so this drain aborts inline and waits within the registry's bound; settle
 * writes may fail 'not connected' there — intended, resume covers it.
 */
function ensureDrainerRegistered(): void {
  if (drainerRegistered) return;
  drainerRegistered = true;
  registerBackgroundWorkDrainer({
    name: 'serve-delegated-sync',
    order: 40,
    async drain(timeoutMs: number): Promise<{ unfinished: number }> {
      const job = current;
      if (!job || isTerminal(job.state)) return { unfinished: 0 };
      job.state = 'aborting';
      job.controller.abort();
      await Promise.race([
        job.settled,
        new Promise<void>((r) => { const t = setTimeout(r, timeoutMs); t.unref?.(); }),
      ]);
      return { unfinished: isTerminal(job.state) ? 0 : 1 };
    },
  });
}

/**
 * Deferred-embed drain — the lock owner's replacement for the inline embed
 * step delegated jobs skip. Bounded per invocation by runEmbedCore's internal
 * wall clock; `pending` clears only when a run finds nothing left to embed,
 * so successive idle ticks converge on a fully-embedded brain. Keyless rule:
 * no embedding provider ⇒ leave pending (a later keyed serve drains it).
 * GBRA-133 2b.3: the drain re-arms itself until the backlog is gone: a pass
 * that left work retries after the retry unit, a failing provider backs off
 * through the tiers, and a sync backlog defers it at most
 * DEFERRED_EMBED_BUSY_MAX times before it drains beside the running sync, so
 * commit-triggered syncs (which always defer) never strand their embeddings.
 * Never throws.
 */
export async function maybeDrainDeferredEmbeds(engine: BrainEngine): Promise<void> {
  const state = deferredEmbeds.get(engine);
  if (!state || state.stopped || shuttingDown) return;
  if (state.running) return state.running;
  if (!state.pending.size) return;
  if (deferredEmbedBusyProbe()) {
    if (state.busy < DEFERRED_EMBED_BUSY_MAX) {
      state.busy++;
      decideDeferredEmbed({ decision: 'busy', delayMs: deferredEmbedRetryMs() });
      armDeferredEmbeds(engine, state, deferredEmbedRetryMs());
      return;
    }
    decideDeferredEmbed({ decision: 'busy_override' });
  }
  state.busy = 0;
  const work = (async () => {
    const { companyBrainProfile, getCompanyBrainProfile } = await import('./company-brain/profile.ts');
    if (state.pending.has(undefined)) {
      state.pending.delete(undefined);
      const sources = await engine.executeRaw<{ id: string; config: unknown }>('SELECT id,config FROM sources WHERE NOT archived');
      for (const source of sources) if (!companyBrainProfile(source.config)) state.pending.add(source.id);
    }
    for (const source of [...state.pending]) if (source && await getCompanyBrainProfile(engine, source)) state.pending.delete(source);
    if (!state.pending.size) { decideDeferredEmbed({ decision: 'no_sources' }); return; }
    const { detectCapabilities } = await import('./capability.ts');
    if (!detectCapabilities().embeddings.available) { decideDeferredEmbed({ decision: 'keyless' }); return; }
    const passStart = performance.now();
    log(`embed-drain started sources=${state.pending.size}`);
    const { runEmbedCore } = await import('../commands/embed.ts');
    let failed = false;
    // Snapshot one fair pass. A source queued again during its own provider
    // call stays pending, even when that earlier call reports no stale work.
    for (const sourceId of [...state.pending]) {
      if (state.stopped) break;
      state.pending.delete(sourceId);
      const sourceStart = performance.now();
      try {
        const r = await runEmbedCore(engine, { stale: true, sourceId, signal: state.controller.signal, quiet: true });
        log(`embed-drain embedded=${r.embedded} failures=${r.failures} source=${sourceId ?? 'all'} ms=${Math.round(performance.now() - sourceStart)}`);
        if (r.embedded !== 0 || r.failures !== 0) state.pending.add(sourceId);
        if (r.failures !== 0) failed = true;
      } catch (error) {
        state.pending.add(sourceId);
        failed = true;
        log(`embed-drain failed: ${error instanceof Error ? error.message : String(error)} source=${sourceId ?? 'all'} ms=${Math.round(performance.now() - sourceStart)}`);
      }
    }
    const ms = Math.round(performance.now() - passStart);
    state.failures = failed ? state.failures + 1 : 0;
    if (!state.pending.size) { decideDeferredEmbed({ decision: 'drained', ms }); return; }
    const delayMs = deferredEmbedRetryMs() * DEFERRED_EMBED_RETRY_TIERS[Math.max(0, Math.min(state.failures, DEFERRED_EMBED_RETRY_TIERS.length) - 1)]!;
    decideDeferredEmbed({ decision: 'retry', delayMs, failures: state.failures, ms });
    armDeferredEmbeds(engine, state, delayMs);
  })().catch(error => { log(`embed-drain failed: ${error instanceof Error ? error.message : String(error)}`); });
  state.running = work;
  try { await work; }
  finally { if (state.running === work) state.running = undefined; }
}

/** Shared post-import kick for legacy and authenticated owner delegation. */
export function scheduleDeferredSyncEmbeds(engine: BrainEngine, sourceId?: string): void {
  if (shuttingDown) return;
  let state = deferredEmbeds.get(engine);
  if (!state) {
    state = { pending: new Set(), controller: new AbortController(), stopped: false, unregister: () => {}, failures: 0, busy: 0 };
    const owned = state;
    state.unregister = engine.registerBeforeDisconnect(async () => {
      owned.stopped = true;
      owned.controller.abort();
      if (owned.timer) clearTimeout(owned.timer);
      // A provider may ignore abort. The actual promise must settle before
      // disconnect closes the datastore or releases its native owner lock.
      await owned.running?.catch(() => {});
      owned.unregister();
      if (deferredEmbeds.get(engine) === owned) deferredEmbeds.delete(engine);
    });
    deferredEmbeds.set(engine, state);
  }
  if (state.stopped) return;
  state.pending.add(sourceId);
  armDeferredEmbeds(engine, state, 5_000);
}

/** Test seam: reset every module singleton (serial tests only). */
export function __resetDelegatedSyncForTests(): void {
  current = null;
  shuttingDown = false;
  shutdownPromise = null;
  for (const state of deferredEmbeds.values()) {
    state.stopped = true; state.controller.abort(); state.unregister();
    if (state.timer) clearTimeout(state.timer);
  }
  deferredEmbeds.clear();
  deferredEmbedDecisions.length = 0;
  deferredEmbedBusyProbe = () => isDelegatedSyncRunning();
}

/** Test seam: stand in for "a delegated sync is running" (serial tests only; reset by __resetDelegatedSyncForTests). */
export function __setDeferredEmbedBusyProbeForTests(probe: () => boolean): void {
  deferredEmbedBusyProbe = probe;
}

/** Test seam: report whether the deferred-embed backlog is pending. */
export function __deferredEmbedsPendingForTests(engine?: BrainEngine): boolean {
  return engine ? !!deferredEmbeds.get(engine)?.pending.size : [...deferredEmbeds.values()].some(state => state.pending.size > 0);
}