Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

33.0 KB
Raw Blame
import { registerWorker, TriggerAction } from "iii-sdk";
import {
  hydrateProcessEnvFromFile,
  loadConfig,
  getEnvVar,
  loadEmbeddingConfig,
  loadFallbackConfig,
  loadClaudeBridgeConfig,
  loadTeamConfig,
  loadSnapshotConfig,
  isGraphExtractionEnabled,
  isAutoCompressEnabled,
  isConsolidationEnabled,
  getConsolidationIntervalMs,
  isContextInjectionEnabled,
  isDropStaleIndexEnabled,
  getVectorBackfillMax,
  isVectorBackfillAllEnabled,
  getAuditRetentionMonths,
  getStateBackend,
  isSessionSweepEnabled,
  isGraphCompactOnBootEnabled,
  getSessionSweepStaleHours,
  getFinalizeIdleMs,
} from "./config.js";
import {
  createProvider,
  createFallbackProvider,
  createEmbeddingProvider,
  createImageEmbeddingProvider,
} from "./providers/index.js";
import { StateKV } from "./state/kv.js";
import { VectorIndex } from "./state/vector-index.js";
import type { SearchLayer } from "./state/search-layer.js";
import { HybridSearch } from "./state/hybrid-search.js";
import { IndexPersistence } from "./state/index-persistence.js";
import { SHUTDOWN_FLUSH_TIMEOUT_MS, SHUTDOWN_HARD_EXIT_MS, runStartupCleanup, settleWithin } from "./shutdown.js";
import { registerPrivacyFunction, withWriteScrubbing } from "./functions/privacy.js";
import { registerObserveFunction } from "./functions/observe.js";
import { registerCaptureFunctions } from "./functions/capture.js";
import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js";
import { registerVisionSearchFunctions } from "./functions/vision-search.js";
import { registerSlotsFunctions, isSlotsEnabled, isReflectEnabled } from "./functions/slots.js";
import { registerDiskSizeManager } from "./functions/disk-size-manager.js";
import { registerCompressFunction } from "./functions/compress.js";
import {
  registerSearchFunction,
  backfillVectorBacklog,
  isBm25RebuildIncomplete,
  isMemoryIndexReady,
  rebuildKeywordIndex,
  markKeywordRebuildPending,
  getSearchIndex,
  setVectorIndex,
  setEmbeddingProvider,
  setIndexPersistence,
  setHybridRanker,
  setPendingVectorBackfillCount,
  setVectorBackfillState,
} from "./functions/search.js";
import { registerContextFunction } from "./functions/context.js";
import { registerSessionIndexMaintenanceFunction } from "./functions/session-index-maintenance.js";
import { rebuildSessionIndexIfStale } from "./state/session-index.js";
import { registerSummarizeFunction } from "./functions/summarize.js";
import { registerMigrateFunction } from "./functions/migrate.js";
import { registerFileIndexFunction } from "./functions/file-index.js";
import { registerConsolidateFunction } from "./functions/consolidate.js";
import { registerPatternsFunction } from "./functions/patterns.js";
import { registerRememberFunction } from "./functions/remember.js";
import { registerEvictFunction } from "./functions/evict.js";
import { registerRelationsFunction } from "./functions/relations.js";
import { registerTimelineFunction } from "./functions/timeline.js";
import { registerSmartSearchFunction } from "./functions/smart-search.js";
import { registerRecentSearchesSweepFunction } from "./functions/recent-searches-sweep.js";
import { registerSessionSweepFunction } from "./functions/session-sweep.js";
import { FINALIZE_TICK_MS, registerSessionFinalizeFunction } from "./functions/session-finalize.js";
import { registerProfileFunction } from "./functions/profile.js";
import { registerAutoForgetFunction } from "./functions/auto-forget.js";
import { registerExportImportFunction } from "./functions/export-import.js";
import { registerEnrichFunction } from "./functions/enrich.js";
import { registerClaudeBridgeFunction } from "./functions/claude-bridge.js";
import { registerGraphFunction } from "./functions/graph.js";
import { registerGraphIndexMigrationFunctions } from "./state/graph-index-migration.js";
import { GRAPH_COMPACT_BOOT_DELAY_MS, runGraphCompactOnBoot, setGraphCompactBootDisabled } from "./functions/graph-compact-boot.js";
import { registerGraphImportFunction } from "./functions/graph-import.js";
import { registerConsolidationPipelineFunction } from "./functions/consolidation-pipeline.js";
import { registerTeamFunction } from "./functions/team.js";
import { registerGovernanceFunction } from "./functions/governance.js";
import { registerSnapshotFunction } from "./functions/snapshot.js";
import { registerActionsFunction } from "./functions/actions.js";
import { registerFrontierFunction } from "./functions/frontier.js";
import { registerLeasesFunction } from "./functions/leases.js";
import { registerRoutinesFunction } from "./functions/routines.js";
import { registerSignalsFunction } from "./functions/signals.js";
import { registerCheckpointsFunction } from "./functions/checkpoints.js";
import { registerFlowCompressFunction } from "./functions/flow-compress.js";
import { registerMeshFunction } from "./functions/mesh.js";
import { registerBranchAwareFunction } from "./functions/branch-aware.js";
import { registerSentinelsFunction } from "./functions/sentinels.js";
import { registerSketchesFunction } from "./functions/sketches.js";
import { registerCrystallizeFunction } from "./functions/crystallize.js";
import { registerDiagnosticsFunction } from "./functions/diagnostics.js";
import { registerFacetsFunction } from "./functions/facets.js";
import { registerVerifyFunction } from "./functions/verify.js";
import { registerCascadeFunction } from "./functions/cascade.js";
import { registerLessonsFunctions } from "./functions/lessons.js";
import { registerObsidianExportFunction } from "./functions/obsidian-export.js";
import { registerReflectFunctions } from "./functions/reflect.js";
import { registerWorkingMemoryFunctions } from "./functions/working-memory.js";
import { registerSkillExtractFunctions } from "./functions/skill-extract.js";
import { registerSlidingWindowFunction } from "./functions/sliding-window.js";
import { registerQueryExpansionFunction } from "./functions/query-expansion.js";
import { registerTemporalGraphFunctions } from "./functions/temporal-graph.js";
import { registerRetentionFunctions } from "./functions/retention.js";
import { startAuditMigration } from "./functions/audit.js";
import { registerCompressFileFunction } from "./functions/compress-file.js";
import { registerReplayFunctions } from "./functions/replay.js";
import { registerApiTriggers } from "./triggers/api.js";
import { registerEventTriggers } from "./triggers/events.js";
import { registerViewerStreamTriggers } from "./triggers/viewer-streams.js";
import { registerViewerSse } from "./triggers/viewer-sse.js";
import { registerMcpEndpoints } from "./mcp/server.js";
import { getAllTools } from "./mcp/tools-registry.js";
import { startViewerServer } from "./viewer/server.js";
import { MetricsStore } from "./eval/metrics-store.js";
import { DedupMap } from "./functions/dedup.js";
import { registerHealthMonitor } from "./health/monitor.js";
import { initMetrics, OTEL_CONFIG } from "./telemetry/setup.js";
import { VERSION } from "./version.js";
import { bootLog, bootWarn } from "./logger.js";
import { runtimeMetadataPath } from "./runtime-paths.js";
import { ensureServerSecret, explicitSecret, secretFilePath } from "./secret-store.js";
import { mkdirSync, writeFileSync, unlinkSync } from "node:fs";
import { dirname } from "node:path";

function workerPidfilePath(): string {
  return runtimeMetadataPath("worker.pid");
}
function writeWorkerPidfile(): void {
  // Worker Compose owns the entire process lifecycle. Its supervised container
  // PID is authoritative; do not create a second ownership record.
  if (process.env["III_COMPOSE_FILE"]) return;
  try {
    const p = workerPidfilePath();
    mkdirSync(dirname(p), { recursive: true });
    writeFileSync(p, `${process.pid}\n`, { encoding: "utf-8" });
  } catch {
    // best-effort; stop still has the engine pidfile + port scan fallback
  }
}
function clearWorkerPidfile(): void {
  if (process.env["III_COMPOSE_FILE"]) return;
  try {
    unlinkSync(workerPidfilePath());
  } catch {}
}

function hasGetMeter(
  sdk: unknown,
): sdk is { getMeter: (name: string) => unknown } {
  return (
    typeof sdk === "object" &&
    sdk !== null &&
    "getMeter" in sdk &&
    typeof (sdk as { getMeter?: unknown }).getMeter === "function"
  );
}

let lastUnhandledLogAt = 0;
process.on("unhandledRejection", (reason) => {
  const now = Date.now();
  if (now - lastUnhandledLogAt < 60_000) return;
  lastUnhandledLogAt = now;
  const r = reason as { code?: string; function_id?: string; message?: string };
  console.warn(
    `[agentmemory] unhandledRejection (suppressed):`,
    r?.code ? `${r.code} ${r.function_id ?? ""} ${r.message ?? ""}`.trim() : reason,
  );
});

function resolveWorkerSecret(): string {
  try {
    const { secret, source } = ensureServerSecret();
    if (source === "generated") {
      bootLog(`Generated an API secret at ${secretFilePath()} (mode 0600). Local clients read it automatically.`);
    }
    return secret;
  } catch (err) {
    throw new Error(
      `agentmemory could not create its API secret at ${secretFilePath()}: ${err instanceof Error ? err.message : String(err)}. Set AGENTMEMORY_SECRET explicitly or make the directory writable.`,
    );
  }
}

async function main() {
  // Fold ~/.agentmemory/.env into process.env before anything reads config
  // or raw process.env. Only-if-unset, so real process.env still wins.
  hydrateProcessEnvFromFile();

  const config = loadConfig();
  const embeddingConfig = loadEmbeddingConfig();
  const fallbackConfig = loadFallbackConfig();

  const provider =
    fallbackConfig.providers.length > 0
      ? createFallbackProvider(config.provider, fallbackConfig)
      : createProvider(config.provider);

  const embeddingProvider = createEmbeddingProvider();
  const imageEmbeddingProvider = createImageEmbeddingProvider();

  bootLog(`Starting worker v${VERSION}...`);
  bootLog(`Engine: ${config.engineUrl}`);
  bootLog(
    `Provider: ${config.provider.provider} (${config.provider.model})`,
  );
  if (embeddingProvider) {
    bootLog(
      `Embedding provider: ${embeddingProvider.name} (${embeddingProvider.dimensions} dims)`,
    );
  } else {
    bootLog(`Embedding provider: none (BM25-only mode)`);
  }
  if (imageEmbeddingProvider) {
    bootLog(
      `Image embedding provider: ${imageEmbeddingProvider.name} (${imageEmbeddingProvider.dimensions} dims) — vision-search active`,
    );
  }
  bootLog(
    `REST API: http://localhost:${config.restPort}/agentmemory/*`,
  );

  const sdk = withWriteScrubbing(registerWorker(config.engineUrl, {
    workerName: "agentmemory",
    invocationTimeoutMs: 180000,
    otel: {
      serviceName: OTEL_CONFIG.serviceName,
      serviceVersion: OTEL_CONFIG.serviceVersion,
      metricsExportIntervalMs: OTEL_CONFIG.metricsExportIntervalMs,
    },
    // Explicit worker telemetry metadata. iii-sdk falls back to
    // auto-detection (cwd / package.json name / hostname) when this
    // is omitted, which produces inconsistent values per host —
    // `agentmemory`, `node`, `npm`, occasionally the user's home
    // directory basename. Pinning the value here gives every install
    // the same stable project identifier for downstream attribution
    // and grouping in the engine's metrics + traces output.
    telemetry: {
      project_name: "agentmemory",
      language: "node",
      framework: "iii-sdk",
    },
  }));

  writeWorkerPidfile();

  let stateBackend: "file" | "redis" = "file";
  try {
    stateBackend = getStateBackend();
  } catch {}
  const kv = new StateKV(sdk, { backend: stateBackend });
  const secret = resolveWorkerSecret();
  const metricsStore = new MetricsStore(kv);
  const dedupMap = new DedupMap();

  const vectorIndex = embeddingProvider ? new VectorIndex() : null;

  setVectorIndex(vectorIndex);
  setEmbeddingProvider(embeddingProvider);

  const meterAccessor = hasGetMeter(sdk)
    ? (sdk.getMeter.bind(sdk) as (name: string) => unknown)
    : undefined;

  initMetrics(meterAccessor as ((name: string) => import("@opentelemetry/api").Meter) | undefined);

  registerPrivacyFunction(sdk);
  registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession);
  const capture = registerCaptureFunctions(sdk, kv, { restPort: config.restPort });
  registerImageQuotaCleanup(sdk, kv);
  registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider);
  if (isSlotsEnabled()) {
    registerSlotsFunctions(sdk, kv);
  }
  registerDiskSizeManager(sdk, kv);
  registerCompressFunction(sdk, kv, provider, metricsStore);
  registerSearchFunction(sdk, kv);
  registerContextFunction(sdk, kv, config.tokenBudget);
  registerSessionIndexMaintenanceFunction(sdk, kv);
  void rebuildSessionIndexIfStale(kv)
    .then((result) => {
      if (result) {
        bootLog(
          `Session index rebuilt: ${result.projects} projects, ${result.sessions} sessions`,
        );
      }
    })
    .catch((err) => {
      console.warn(`[agentmemory] Failed to rebuild session index at boot:`, err);
    });
  registerSummarizeFunction(sdk, kv, provider, metricsStore);
  registerMigrateFunction(sdk, kv);
  registerFileIndexFunction(sdk, kv);
  registerConsolidateFunction(sdk, kv, provider);
  registerPatternsFunction(sdk, kv);
  registerRememberFunction(sdk, kv);
  registerEvictFunction(sdk, kv);
  registerSessionSweepFunction(sdk, kv);
  registerSessionFinalizeFunction(sdk, kv);

  registerRelationsFunction(sdk, kv);
  registerTimelineFunction(sdk, kv);
  registerProfileFunction(sdk, kv);
  registerAutoForgetFunction(sdk, kv);
  registerExportImportFunction(sdk, kv);
  registerEnrichFunction(sdk, kv);

  const claudeBridgeConfig = loadClaudeBridgeConfig();
  if (claudeBridgeConfig.enabled) {
    registerClaudeBridgeFunction(sdk, kv, claudeBridgeConfig);
    bootLog(
      `Claude bridge: syncing to ${claudeBridgeConfig.memoryFilePath}`,
    );
  }

  registerGraphFunction(sdk, kv, provider);
  const graphIndexMigration = registerGraphIndexMigrationFunctions(sdk, kv);
  registerGraphImportFunction(sdk, kv);
  bootLog(
    `Knowledge graph: structural extraction on (LLM relations ${isGraphExtractionEnabled() ? "enabled" : "off"})`,
  );

  registerConsolidationPipelineFunction(sdk, kv, provider);
  bootLog(`Consolidation pipeline: registered (CONSOLIDATION_ENABLED=${isConsolidationEnabled() ? "true" : "false"})`);

  if (isAutoCompressEnabled()) {
    bootLog(
      `WARNING: AGENTMEMORY_AUTO_COMPRESS=true — every PostToolUse observation will be sent to your LLM provider for compression. This spends API tokens proportional to your session tool-use frequency. Set AGENTMEMORY_AUTO_COMPRESS=false to disable.`,
    );
  } else {
    bootLog(
      `Auto-compress: OFF (default) — observations indexed via zero-LLM synthetic compression. Set AGENTMEMORY_AUTO_COMPRESS=true to opt-in to LLM-powered summaries (uses your API key).`,
    );
  }

  if (isContextInjectionEnabled()) {
    bootLog(
      `WARNING: AGENTMEMORY_INJECT_CONTEXT=true — the PreToolUse and SessionStart hooks will inject up to ~4000 chars of memory context into every tool turn. On Claude Pro this burns session tokens proportional to your tool-call frequency. Set AGENTMEMORY_INJECT_CONTEXT=false to disable.`,
    );
  } else {
    bootLog(
      `Context injection: OFF (default) — hooks capture observations but do not inject context into Claude Code's conversation. Set AGENTMEMORY_INJECT_CONTEXT=true to opt-in (warning: expect your Claude Pro allocation to drain faster).`,
    );
  }

  const teamConfig = loadTeamConfig();
  if (teamConfig) {
    registerTeamFunction(sdk, kv, teamConfig);
    bootLog(
      `Team memory: ${teamConfig.teamId} (${teamConfig.mode})`,
    );
  }

  registerGovernanceFunction(sdk, kv);

  registerActionsFunction(sdk, kv);
  registerFrontierFunction(sdk, kv);
  registerLeasesFunction(sdk, kv);
  registerRoutinesFunction(sdk, kv);
  registerSignalsFunction(sdk, kv);
  registerCheckpointsFunction(sdk, kv);
  registerMeshFunction(sdk, kv, explicitSecret() || undefined);
  registerBranchAwareFunction(sdk, kv);
  registerFlowCompressFunction(sdk, kv, provider);
  registerSentinelsFunction(sdk, kv);
  registerSketchesFunction(sdk, kv);
  registerCrystallizeFunction(sdk, kv, provider);
  registerDiagnosticsFunction(sdk, kv);
  registerFacetsFunction(sdk, kv);
  registerVerifyFunction(sdk, kv);
  registerLessonsFunctions(sdk, kv);
  registerObsidianExportFunction(sdk, kv);
  registerReflectFunctions(sdk, kv, provider);
  registerWorkingMemoryFunctions(sdk, kv, config.tokenBudget);
  registerSkillExtractFunctions(sdk, kv, provider);
  registerCascadeFunction(sdk, kv);

  registerSlidingWindowFunction(sdk, kv, provider);
  registerQueryExpansionFunction(sdk, provider);
  registerTemporalGraphFunctions(sdk, kv, provider);
  registerRetentionFunctions(sdk, kv);
  registerCompressFileFunction(sdk, kv, provider);
  registerReplayFunctions(sdk, kv);
  bootLog(
    `v0.6 advanced retrieval: sliding-window, query-expansion, temporal-graph, retention-scoring`,
  );
  bootLog(
    `Orchestration layer: actions, frontier, leases, routines, signals, checkpoints, flow-compress, mesh, branch-aware, sentinels, sketches, crystallize, diagnostics, facets`,
  );
  if (isSlotsEnabled()) {
    bootLog(
      `Slots: enabled (pinned editable memory). Reflect on Stop hook: ${isReflectEnabled() ? "on" : "off"}`,
    );
  }

  const snapshotConfig = loadSnapshotConfig();
  if (snapshotConfig.enabled) {
    registerSnapshotFunction(sdk, kv, snapshotConfig.dir);
    // The boot line promised "every <interval>s" but nothing ever fired
    // mem::snapshot-create. Drive it on a periodic timer (unref'd so it
    // never keeps the process alive), mirroring the auto-forget timer.
    // mem::snapshot-create serializes overlapping runs internally (git-lock
    // safety), so the timer can stay a simple fire-and-forget tick.
    const snapshotTimer = setInterval(() => {
      sdk
        .trigger({
          function_id: "mem::snapshot-create",
          payload: {},
          action: TriggerAction.Void(),
        })
        .catch(() => {});
    }, snapshotConfig.interval * 1000);
    snapshotTimer.unref();
    bootLog(
      `Git snapshots: ${snapshotConfig.dir} (every ${snapshotConfig.interval}s)`,
    );
  }

  const bm25Index = getSearchIndex();
  const graphWeight = parseFloat(getEnvVar("AGENTMEMORY_GRAPH_WEIGHT") || "0.3");
  const hybridSearch = new HybridSearch(
    bm25Index,
    vectorIndex,
    embeddingProvider,
    kv,
    embeddingConfig.bm25Weight,
    embeddingConfig.vectorWeight,
    graphWeight,
  );

  const hybridRanker = (query: string, limit: number, targetLayer?: SearchLayer, authorizeGraph?: Parameters<HybridSearch["search"]>[3]) =>
    hybridSearch.search(query, limit, targetLayer, authorizeGraph);
  registerSmartSearchFunction(sdk, kv, hybridRanker);
  setHybridRanker(hybridRanker);
  registerRecentSearchesSweepFunction(sdk, kv);

  markKeywordRebuildPending();
  registerApiTriggers(sdk, kv, secret, metricsStore, provider);
  registerEventTriggers(sdk, kv);
  registerViewerStreamTriggers(sdk, kv, { secret, metricsStore, provider });
  registerViewerSse(sdk);
  registerMcpEndpoints(sdk, kv, secret);

  const healthMonitor = registerHealthMonitor(sdk, kv);

  const indexPersistence = new IndexPersistence(kv, vectorIndex, { requireRecoveryPlan: true });
  setIndexPersistence(indexPersistence);

  const loaded = await indexPersistence.load().catch((err) => {
    console.warn(`[agentmemory] Failed to load persisted vector index:`, err);
    return null;
  });
  let vectorBackfillSince = await indexPersistence.prepareBackfill(loaded, isVectorBackfillAllEnabled());
  if (loaded?.vector && vectorIndex && loaded.vector.size > 0) {
    // Persisted vectors carry whatever dimension the provider had when
    // they were written. If the active provider declares a different
    // dimension — or if the on-disk index contains a mix of dimensions
    // (legacy indexes written before the live-API guard in this PR) —
    // restoring would silently corrupt search: cosineSimilarity returns
    // 0 on cross-dim pairs, so affected observations stop matching
    // anything and recall degrades without an error. Walk every stored
    // vector instead of trusting the first; refuse to load if anything
    // is off.
    const activeDim = embeddingProvider?.dimensions ?? 0;
    const { mismatches, seenDimensions } =
      activeDim > 0
        ? loaded.vector.validateDimensions(activeDim)
        : { mismatches: [], seenDimensions: new Set<number>() };

    if (mismatches.length > 0) {
      const sample = mismatches
        .slice(0, 5)
        .map((m) => `${m.obsId} (dim=${m.dim})`)
        .join(", ");
      const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", ");
      const dropStale = isDropStaleIndexEnabled();
      if (dropStale) {
        vectorBackfillSince = await indexPersistence.prepareBackfill(loaded, true);
        for (const [obsId] of loaded.vector.entries()) vectorIndex.markRemoved(obsId);
        console.warn(
          `[agentmemory] Persisted vector index has ${mismatches.length} of ` +
            `${loaded.vector.size} vectors with the wrong dimension. Active ` +
            `provider (${embeddingProvider?.name}) declares ${activeDim}; ` +
            `dimensions seen on disk: ${distinct}. ` +
            `AGENTMEMORY_DROP_STALE_INDEX=true is set — discarding the persisted ` +
            `vectors. Live observations will rebuild the index over time.`,
        );
      } else {
        throw new Error(
          `[agentmemory] Refusing to start: persisted vector index has ` +
            `${mismatches.length} of ${loaded.vector.size} vectors with the ` +
            `wrong dimension. Active provider (${embeddingProvider?.name}) ` +
            `declares ${activeDim}; dimensions seen on disk: ${distinct}. ` +
            `First mismatched obsIds: ${sample}. Loading would silently corrupt ` +
            `search (cross-dimension cosine returns 0). Choose one:\n` +
            `  - Re-embed the existing index against the new provider, then start.\n` +
            `  - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted ` +
            `vectors and rebuild from live observations.\n` +
            `  - Switch the embedding provider back to the one that wrote the index.`,
        );
      }
    } else {
      vectorIndex.restoreFrom(loaded.vector);
      bootLog(
        `Loaded persisted vector index (${vectorIndex.size} vectors)`,
      );
    }
  }

  if (vectorIndex && loaded && loaded.state !== "unavailable") {
    const replay = await indexPersistence.replayPendingLog(embeddingProvider?.dimensions ?? 0);
    if (replay.skipped > 0) vectorBackfillSince = null;
    if (indexPersistence.isRecoveryPaused()) vectorBackfillSince = undefined;
    if (replay.entries > 0) {
      bootLog(
        `Recovered ${replay.added} vectors and ${replay.removed} removals written after the last index save, without calling the embedding provider` +
          (replay.skipped > 0 ? ` (${replay.skipped} entries skipped)` : ""),
      );
    }
  }

  const auditMigration = startAuditMigration(kv).catch(() => {});

  const keywordStart = Date.now();
  try {
    const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince);
    bootLog(
      `Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`,
    );
    const pending = keyword.vectorJobs.length + keyword.fullBackfillPending;
    const backlogScanComplete = !isBm25RebuildIncomplete() && isMemoryIndexReady();
    const optedIn = isVectorBackfillAllEnabled();
    const activeIds = new Set(keyword.vectorJobs.map((job) => job.id));
    const hasVector = (id: string) => Boolean(vectorIndex?.has(id));
    const countRemaining = () => bm25Index.countMissingDocuments(hasVector, vectorBackfillSince, activeIds);
    setPendingVectorBackfillCount(indexPersistence.isRecoveryPaused() ? countRemaining() : pending);
    if (indexPersistence.isRecoveryPaused()) {
      setVectorBackfillState("paused");
    } else if (vectorBackfillSince !== undefined) {
      setVectorBackfillState(keyword.fullBackfillPending > 0 && !optedIn ? "waiting-for-opt-in" : backlogScanComplete ? "idle" : "paused");
    }
    if (keyword.fullBackfillPending > 0) {
      bootLog(optedIn
        ? `Vector backfill has ${keyword.fullBackfillPending} additional documents beyond this start's limit; restart to resume.`
        : `Vector backfill needs ${keyword.fullBackfillPending} embeddings (set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`);
    }
    if (vectorBackfillSince !== undefined && backlogScanComplete && pending === 0) {
      if (!(await indexPersistence.completeBackfill(() => countRemaining() === 0))) {
        setPendingVectorBackfillCount(countRemaining());
        setVectorBackfillState("paused");
      }
    }
    if (keyword.vectorJobs.length > 0 && !(await indexPersistence.beginBackfill())) {
      setVectorBackfillState("paused");
      activeIds.clear();
      for (const job of keyword.vectorJobs) job.revision?.release();
    } else if (keyword.vectorJobs.length > 0) {
      setVectorBackfillState("running");
      bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors, ${getVectorBackfillMax()} per batch in the background`);
      void backfillVectorBacklog(keyword.vectorJobs, {
        remainingAfter: keyword.fullBackfillPending,
        remainingCount: countRemaining,
      })
        .then(async (result) => {
          const complete = result.complete && backlogScanComplete && await indexPersistence.completeBackfill(() => countRemaining() === 0);
          const remaining = countRemaining();
          setPendingVectorBackfillCount(remaining);
          setVectorBackfillState(complete ? "idle" : "paused");
          if (result.added > 0) bootLog(`Vector index backfilled: ${result.added} entries`);
          if (!complete) bootWarn(`Vector recovery remains incomplete (${remaining} documents pending); it resumes on the next start.`);
        })
        .catch((err) => {
          setVectorBackfillState("paused");
          console.warn(`[agentmemory] Failed to backfill vectors:`, err);
        })
        .finally(() => {
          activeIds.clear();
          for (const job of keyword.vectorJobs) job.revision?.release();
        });
    }
  } catch (err) {
    setVectorBackfillState("paused");
    console.warn(`[agentmemory] Failed to rebuild the BM25 index:`, err);
  }

  // Ready / Endpoints lines are emitted via `bootLog` so they're
  // buffered in quiet mode and printed verbatim under --verbose. The
  // CLI surfaces a compact summary when it sees the worker reach
  // ready state.
  bootLog(
    `Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`,
  );
  bootLog(
    `REST API: 146 endpoints at http://localhost:${config.restPort}/agentmemory/*`,
  );
  bootLog(
    `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`,
  );

  void (async () => {
    try {
      const drained = await capture.drainLocalSpool();
      const delivered = drained.reduce((n, r) => n + r.delivered, 0);
      const duplicates = drained.reduce((n, r) => n + r.duplicates, 0);
      const remaining = drained.reduce((n, r) => n + r.remaining, 0);
      if (delivered + duplicates + remaining > 0) {
        bootLog(`Capture spool: ${delivered} recovered, ${duplicates} already stored, ${remaining} still waiting`);
      }
      const swept = await capture.sweep();
      if (swept.processed > 0) {
        bootLog(`Capture inbox: ${swept.recovered} of ${swept.processed} unfinished observations stored after restart`);
      }
      await capture.prune();
    } catch (err) {
      bootWarn(`Capture recovery at boot failed: ${err instanceof Error ? err.message : String(err)}`);
    }
    capture.start();
  })();

  const viewerServer = startViewerServer(
    config.viewerPort,
    kv,
    sdk,
    secret,
    config.restPort,
  );

  void graphIndexMigration.resumeOnBoot().catch(() => {
    bootWarn("Graph index migration could not resume. Run agentmemory graph-index status for recovery instructions.");
  });

  if (isGraphCompactOnBootEnabled()) {
    const graphCompactTimer = setTimeout(() => {
      void runGraphCompactOnBoot(kv, { log: bootLog, warn: bootWarn }).catch(() => {});
    }, GRAPH_COMPACT_BOOT_DELAY_MS);
    graphCompactTimer.unref();
  } else {
    setGraphCompactBootDisabled();
  }

  const autoForgetIntervalMs = parseInt(process.env.AUTO_FORGET_INTERVAL_MS || "3600000", 10);
  const consolidationIntervalMs = getConsolidationIntervalMs();

  if (process.env.AUTO_FORGET_ENABLED !== "false") {
    const autoForgetTimer = setInterval(async () => {
      try {
        await sdk.trigger({ function_id: "mem::auto-forget", payload: { dryRun: false } });
      } catch {}
    }, autoForgetIntervalMs);
    autoForgetTimer.unref();
    bootLog(`Auto-forget: enabled (every ${autoForgetIntervalMs / 60000}m)`);
  }

  const auditRetentionMonths = getAuditRetentionMonths();
  if (auditRetentionMonths > 0) {
    const runAuditSweep = async () => {
      try {
        await auditMigration;
        await sdk.trigger({ function_id: "mem::audit-retention-sweep", payload: {} });
      } catch {}
    };
    void runAuditSweep();
    const auditRetentionTimer = setInterval(runAuditSweep, 86400000);
    auditRetentionTimer.unref();
    bootLog(
      `Audit retention sweep: enabled (drop month scopes older than ${auditRetentionMonths}mo, every 24h)`,
    );
  }

  if (process.env.LESSON_DECAY_ENABLED !== "false") {
    const lessonDecayTimer = setInterval(async () => {
      try {
        await sdk.trigger({ function_id: "mem::lesson-decay-sweep", payload: {} });
      } catch {}
    }, 86400000);
    lessonDecayTimer.unref();
    bootLog(`Lesson decay sweep: enabled (every 24h)`);
  }

  if (process.env.INSIGHT_DECAY_ENABLED !== "false") {
    const insightDecayTimer = setInterval(async () => {
      try {
        await sdk.trigger({ function_id: "mem::insight-decay-sweep", payload: {} });
      } catch {}
    }, 86400000);
    insightDecayTimer.unref();
  }

  const recentSearchesSweepTimer = setInterval(async () => {
    try {
      await sdk.trigger({
        function_id: "mem::diagnostic::recent-searches-sweep",
        payload: {},
      });
    } catch {}
  }, 60 * 60 * 1000);
  recentSearchesSweepTimer.unref();


  if (isSessionSweepEnabled()) {
    const sessionSweepTimer = setInterval(async () => {
      try {
        await sdk.trigger({ function_id: "mem::session-sweep", payload: {} });
      } catch (err) {
        bootLog(`Session sweep failed: ${err instanceof Error ? err.message : String(err)}`);
      }
    }, 60 * 60 * 1000);
    sessionSweepTimer.unref();
    bootLog(`Session sweep: enabled (hourly, stale after ${getSessionSweepStaleHours()}h)`);
  }

  const finalizeIdleMs = getFinalizeIdleMs();
  let finalizeRunning = false;
  const runSessionFinalize = async () => {
    if (finalizeRunning) return;
    finalizeRunning = true;
    try {
      await sdk.trigger({ function_id: "mem::session-finalize", payload: {} });
    } catch (err) {
      bootLog(`Session finalize failed: ${err instanceof Error ? err.message : String(err)}`);
    } finally {
      finalizeRunning = false;
    }
  };
  const sessionFinalizeTimer = setInterval(runSessionFinalize, FINALIZE_TICK_MS);
  sessionFinalizeTimer.unref();
  void runSessionFinalize();
  bootLog(`Session finalize: ${finalizeIdleMs > 0 ? `idle sessions complete after ${Math.round(finalizeIdleMs / 1000)}s` : "idle finalization disabled"}; pending summaries recover at startup and every ${FINALIZE_TICK_MS / 1000}s`);

  if (isConsolidationEnabled()) {
    const consolidationTimer = setInterval(async () => {
      try {
        await sdk.trigger({ function_id: "mem::consolidate-pipeline", payload: {} });
      } catch {}
    }, consolidationIntervalMs);
    consolidationTimer.unref();
    bootLog(`Auto-consolidation: enabled (every ${consolidationIntervalMs / 60000}m)`);
  }

  const shutdown = async () => {
    console.log(`\n[agentmemory] Shutting down...`);
    const hardExit = setTimeout(async () => {
      console.warn(
        `[agentmemory] Shutdown still blocked after ${SHUTDOWN_HARD_EXIT_MS}ms; exiting now.`,
      );
      clearWorkerPidfile();
      process.exit(await runStartupCleanup(1).catch(() => 1));
    }, SHUTDOWN_HARD_EXIT_MS);
    hardExit.unref();
    healthMonitor.stop();
    dedupMap.stop();
    capture.stop();
    clearInterval(sessionFinalizeTimer);
    await settleWithin(graphIndexMigration.stop(), SHUTDOWN_FLUSH_TIMEOUT_MS);
    indexPersistence.stop();
    const viewerClosed = new Promise<void>((resolve) =>
      viewerServer.close(() => resolve()),
    );
    viewerServer.closeAllConnections();
    await viewerClosed;
    const flushed = await settleWithin(
      indexPersistence.save(),
      SHUTDOWN_FLUSH_TIMEOUT_MS,
    );
    if (!flushed) {
      console.warn(
        `[agentmemory] Search index flush did not finish within ${SHUTDOWN_FLUSH_TIMEOUT_MS}ms; the engine is probably gone. Observations are already in the engine's state store and the index reconciles them on the next boot.`,
      );
    }
    await settleWithin(sdk.shutdown(), SHUTDOWN_FLUSH_TIMEOUT_MS).catch((err) => {
      console.warn(`[agentmemory] SDK shutdown failed:`, err);
    });
    clearWorkerPidfile();
    process.exit(await runStartupCleanup(0).catch(() => 1));
  };
  process.on("SIGINT", shutdown);
  process.on("SIGTERM", shutdown);
}

main().catch(async (err) => {
  console.error(`[agentmemory] Fatal:`, err);
  process.exit(await runStartupCleanup(1).catch(() => 1));
});