Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

23.5 KB
Raw Blame
import { createHash, randomBytes } from "node:crypto";
import { readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import type { IIIClient } from "iii-sdk";
import type { CompressedObservation, HookPayload } from "../types.js";
import { KV, generateId } from "../state/schema.js";
import type { StateKV } from "../state/kv.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { getEnvVar, getStateBackend } from "../config.js";
import { logger } from "../logger.js";
import { isValidEventId } from "../capture/event-id.js";
import {
  EVENT_SHARD_CHARS,
  EVENT_SHARDS,
  captureEventScope as eventScope,
  type CompletedEvent,
} from "../capture/event-record.js";
import { restoreIndexEntries } from "./observe.js";
import { prunePromptLedgers } from "../capture/prompt-ledger.js";
import { scrubRecord } from "./privacy.js";
import { runtimeConfigPath } from "../cli/engine-launch.js";
import { captureDurableAfterMs, engineStateConfigPaths } from "../cli/engine-config.js";
import {
  drainSpool,
  reconcileSent,
  retainSent,
  spoolDir,
  spoolPolicy,
  spoolSummary,
  type DrainResult,
  type SendOutcome,
  type SentMark,
  type SpoolRecord,
  type SpoolSummary,
} from "../capture/spool.js";

export type InboxState = "pending" | "retrying" | "dead";

export interface InboxRecord {
  key: string;
  eventId: string;
  eventSource: "client" | "server";
  sessionId: string;
  project: string;
  hookType: string;
  observationId: string;
  status: InboxState;
  attempts: number;
  acceptedAt: string;
  updatedAt: string;
  nextAttemptAt?: string;
  deadAt?: string;
  lastError?: string;
  payload: HookPayload;
}

export type { CompletedEvent } from "../capture/event-record.js";
export { captureEventShard } from "../capture/event-record.js";

export type CaptureResult =
  | { status: "accepted"; state: "completed"; eventId: string; observationId: string; attempts: number }
  | { status: "accepted"; state: "retrying"; eventId: string; observationId: string; attempts: number; nextAttemptAt: string; error: string }
  | { status: "duplicate"; state: "completed" | InboxState; eventId: string; observationId?: string; deduplicated: true }
  | { status: "rejected"; eventId: string; error: string; retryable: boolean; observationId?: string };

export interface CapturePolicy {
  maxAttempts: number;
  retryIntervalMs: number;
  dedupRetentionHours: number;
  inboxMax: number;
  deadMax: number;
  eventsMax: number;
}

export interface CaptureStatus {
  policy: CapturePolicy;
  inbox: {
    pending: number;
    retrying: number;
    dead: number;
    oldestAcceptedAt: string | null;
    nextAttemptAt: string | null;
    lastError: string | null;
  } | null;
  sinceStart: {
    accepted: number;
    completed: number;
    duplicates: number;
    retried: number;
    recovered: number;
    deadLettered: number;
    rejected: number;
    pruned: number;
  };
  lastSweepAt: string | null;
  lastPruneAt: string | null;
  lastBootDrain: (DrainResult & { at: string }) | null;
  spool: SpoolSummary[];
}

const MAX_BACKOFF_MS = 15 * 60_000;
const ERROR_MAX_CHARS = 500;

function positiveInt(key: string, fallback: number): number {
  const raw = getEnvVar(key);
  if (!raw || !/^\d+$/.test(raw.trim())) return fallback;
  const n = parseInt(raw.trim(), 10);
  return n > 0 ? n : fallback;
}

export function capturePolicy(): CapturePolicy {
  return {
    maxAttempts: positiveInt("AGENTMEMORY_CAPTURE_MAX_ATTEMPTS", 5),
    retryIntervalMs: positiveInt("AGENTMEMORY_CAPTURE_RETRY_INTERVAL_MS", 10_000),
    dedupRetentionHours: positiveInt("AGENTMEMORY_CAPTURE_DEDUP_HOURS", 168),
    inboxMax: positiveInt("AGENTMEMORY_CAPTURE_INBOX_MAX", 10_000),
    deadMax: positiveInt("AGENTMEMORY_CAPTURE_DEAD_MAX", 1000),
    eventsMax: positiveInt("AGENTMEMORY_CAPTURE_EVENTS_MAX", 100_000),
  };
}

export function captureKey(project: string, sessionId: string, eventId: string): string {
  const hash = createHash("sha256").update(`${project}\u0000${sessionId}\u0000${eventId}`).digest("hex");
  return `cap_${hash.slice(0, 40)}`;
}

const MIN_ID_TIME = Date.UTC(2020, 0, 1);
const MAX_ID_TIME = Date.UTC(2100, 0, 1);

export function observationIdFor(key: string, timestamp: string | undefined, acceptedAt: number): string {
  const at = typeof timestamp === "string" ? Date.parse(timestamp) : NaN;
  const stamp = Number.isFinite(at) && at >= MIN_ID_TIME && at < MAX_ID_TIME ? at : acceptedAt;
  return `obs_${stamp.toString(36)}_${key.slice(4, 16)}`;
}

function backoffMs(attempts: number, policy: CapturePolicy): number {
  return Math.min(policy.retryIntervalMs * 2 ** Math.max(0, attempts - 1), MAX_BACKOFF_MS);
}

function errorText(err: unknown): string {
  return (err instanceof Error ? err.message : String(err)).slice(0, ERROR_MAX_CHARS);
}

function serverSpoolTargets(restPort: number): string[] {
  const url = `http://localhost:${restPort}`;
  const dirs = new Set<string>([spoolDir()]);
  try {
    const { AGENTMEMORY_DATA_DIR: _workerDataDir, AGENTMEMORY_CAPTURE_SPOOL_DIR: _spoolDir, ...hookEnv } = process.env;
    dirs.add(spoolDir(hookEnv));
  } catch {}
  return [...dirs].map((dir) => `${url}|${dir}`);
}

function splitTarget(target: string): { url: string; dir: string } {
  const at = target.indexOf("|");
  return { url: target.slice(0, at), dir: target.slice(at + 1) };
}

export function serverDurableAfterMs(): number {
  let backend: "file" | "redis" = "file";
  try {
    backend = getStateBackend();
  } catch {}
  const configTexts: string[] = [];
  const dataDir = getEnvVar("AGENTMEMORY_DATA_DIR");
  if (dataDir) {
    for (const path of engineStateConfigPaths(dataDir, runtimeConfigPath(dataDir))) {
      try {
        configTexts.push(readFileSync(path, "utf-8"));
      } catch {}
    }
  }
  return captureDurableAfterMs(backend, configTexts);
}

let activeController: CaptureController | null = null;

export function getCaptureController(): CaptureController | null {
  return activeController;
}

export interface CaptureController {
  sweep(): Promise<{ processed: number; recovered: number; deadLettered: number }>;
  prune(): Promise<number>;
  drainLocalSpool(): Promise<DrainResult[]>;
  status(): Promise<CaptureStatus>;
  durability(): SentMark;
  start(): void;
  stop(): void;
}

export function registerCaptureFunctions(
  sdk: IIIClient,
  kv: StateKV,
  options: { restPort?: number; durableAfterMs?: number } = {},
): CaptureController {
  const policy = capturePolicy();
  const mark: SentMark = {
    bootId: randomBytes(12).toString("hex"),
    durableAfterMs: options.durableAfterMs ?? serverDurableAfterMs(),
  };
  const counters: CaptureStatus["sinceStart"] = {
    accepted: 0,
    completed: 0,
    duplicates: 0,
    retried: 0,
    recovered: 0,
    deadLettered: 0,
    rejected: 0,
    pruned: 0,
  };
  let lastSweepAt: string | null = null;
  let lastPruneAt: string | null = null;
  let lastBootDrain: CaptureStatus["lastBootDrain"] = null;
  let sweeping: Promise<{ processed: number; recovered: number; deadLettered: number }> | null = null;
  let inboxSize: number | null = null;
  const timers: Array<ReturnType<typeof setInterval>> = [];
  const spoolTargets = options.restPort ? serverSpoolTargets(options.restPort) : [];

  async function currentInboxSize(): Promise<number> {
    if (inboxSize === null) inboxSize = (await kv.list<InboxRecord>(KV.captureInbox)).length;
    return inboxSize;
  }

  async function complete(rec: InboxRecord, observationId: string): Promise<void> {
    const done: CompletedEvent = {
      key: rec.key,
      eventId: rec.eventId,
      sessionId: rec.sessionId,
      project: rec.project,
      observationId,
      acceptedAt: rec.acceptedAt,
      completedAt: new Date().toISOString(),
      attempts: rec.attempts,
    };
    if (rec.eventSource === "client") await kv.set(eventScope(rec.key), rec.key, done);
    await kv.delete(KV.captureInbox, rec.key);
    if (inboxSize !== null) inboxSize = Math.max(0, inboxSize - 1);
    counters.completed++;
  }

  async function storedObservation(done: CompletedEvent): Promise<CompressedObservation | null | undefined> {
    if (!done.observationId) return undefined;
    try {
      return (await kv.get<CompressedObservation>(KV.observations(done.sessionId), done.observationId)) ?? null;
    } catch {
      return undefined;
    }
  }

  async function attempt(rec: InboxRecord): Promise<CaptureResult> {
    rec.attempts++;
    type ObserveResult = { observationId?: string; deduplicated?: boolean; success?: boolean; error?: string };
    let result = null as ObserveResult | null;
    let failure: string | null = null;
    try {
      result = (await sdk.trigger({
        function_id: "mem::observe",
        payload: {
          ...rec.payload,
          ...(rec.eventSource === "client" ? { eventId: rec.eventId, captureKey: rec.key } : {}),
          observationId: rec.observationId,
        },
      })) as ObserveResult | null;
    } catch (err) {
      failure = errorText(err);
    }
    const now = new Date();
    if (!failure && result && typeof result.observationId === "string") {
      if (rec.attempts > 1) counters.recovered++;
      await complete(rec, result.observationId);
      return { status: "accepted", state: "completed", eventId: rec.eventId, observationId: result.observationId, attempts: rec.attempts };
    }
    if (!failure && result?.deduplicated) {
      await complete(rec, "");
      counters.duplicates++;
      return { status: "duplicate", state: "completed", eventId: rec.eventId, deduplicated: true };
    }
    const permanent = !failure;
    const message = failure ?? (result && typeof result.error === "string" ? result.error : "mem::observe returned no observation");
    if (permanent || rec.attempts >= policy.maxAttempts) {
      const dead: InboxRecord = { ...rec, status: "dead", updatedAt: now.toISOString(), deadAt: now.toISOString(), lastError: message };
      delete dead.nextAttemptAt;
      await kv.set(KV.captureInbox, rec.key, dead);
      counters.deadLettered++;
      logger.warn("capture moved to dead letters", { eventId: rec.eventId, attempts: rec.attempts, error: message });
      return { status: "rejected", eventId: rec.eventId, error: message, retryable: false, observationId: rec.observationId };
    }
    const nextAttemptAt = new Date(now.getTime() + backoffMs(rec.attempts, policy)).toISOString();
    const retrying: InboxRecord = { ...rec, status: "retrying", updatedAt: now.toISOString(), nextAttemptAt, lastError: message };
    await kv.set(KV.captureInbox, rec.key, retrying);
    counters.retried++;
    logger.warn("capture processing failed, will retry", { eventId: rec.eventId, attempts: rec.attempts, nextAttemptAt, error: message });
    return { status: "accepted", state: "retrying", eventId: rec.eventId, observationId: rec.observationId, attempts: rec.attempts, nextAttemptAt, error: message };
  }

  async function capture(input: { payload: HookPayload; eventId?: string }): Promise<CaptureResult> {
    const payload = input.payload;
    const clientEventId = isValidEventId(input.eventId) ? input.eventId : undefined;
    const eventId = clientEventId ?? generateId("evs");
    const key = captureKey(payload.project, payload.sessionId, eventId);
    return withKeyedLock(`capture:${key}`, async () => {
      let storedObservationId: string | undefined;
      if (clientEventId) {
        const done = await kv.get<CompletedEvent>(eventScope(key), key);
        if (done?.state === "deleted") {
          counters.duplicates++;
          return { status: "duplicate", state: "completed", eventId, observationId: done.observationId, deduplicated: true };
        }
        if (done) {
          const stored = await storedObservation(done);
          if (stored !== null) {
            if (stored) await restoreIndexEntries(stored).catch(() => {});
            counters.duplicates++;
            return { status: "duplicate", state: "completed", eventId, observationId: done.observationId, deduplicated: true };
          }
          storedObservationId = done.observationId;
          logger.warn("capture event was marked stored but its observation is missing, storing it again", {
            eventId,
            observationId: done.observationId,
          });
        }
      }
      const queued = await kv.get<InboxRecord>(KV.captureInbox, key);
      if (queued) {
        counters.duplicates++;
        return { status: "duplicate", state: queued.status, eventId, observationId: queued.observationId, deduplicated: true };
      }
      if ((await currentInboxSize()) >= policy.inboxMax) {
        counters.rejected++;
        return { status: "rejected", eventId, error: `capture inbox is full (${policy.inboxMax} unprocessed events)`, retryable: true };
      }
      const acceptedAt = Date.now();
      const { eventId: _ignoredEventId, observationId: _ignoredObservationId, captureKey: _ignoredCaptureKey, ...cleanPayload } = payload;
      const rec: InboxRecord = {
        key,
        eventId,
        eventSource: clientEventId ? "client" : "server",
        sessionId: payload.sessionId,
        project: payload.project,
        hookType: payload.hookType,
        observationId: storedObservationId ?? observationIdFor(key, payload.timestamp, acceptedAt),
        status: "pending",
        attempts: 0,
        acceptedAt: new Date(acceptedAt).toISOString(),
        updatedAt: new Date(acceptedAt).toISOString(),
        payload: { ...cleanPayload, data: scrubRecord(cleanPayload.data) },
      };
      await kv.set(KV.captureInbox, key, rec);
      if (inboxSize !== null) inboxSize++;
      counters.accepted++;
      return attempt(rec);
    });
  }

  async function retryOne(key: string, force: boolean): Promise<"recovered" | "dead" | "waiting" | "gone"> {
    return withKeyedLock(`capture:${key}`, async () => {
      const rec = await kv.get<InboxRecord>(KV.captureInbox, key);
      if (!rec) return "gone";
      if (rec.status === "dead" && !force) return "waiting";
      if (!force && rec.status === "retrying" && rec.nextAttemptAt && Date.parse(rec.nextAttemptAt) > Date.now()) return "waiting";
      if (rec.eventSource === "client" && (await kv.get<CompletedEvent>(eventScope(key), key))?.state === "deleted") {
        await kv.delete(KV.captureInbox, key);
        if (inboxSize !== null) inboxSize = Math.max(0, inboxSize - 1);
        counters.duplicates++;
        return "gone";
      }
      if (force) rec.attempts = 0;
      const outcome = await attempt({ ...rec });
      if (outcome.status === "accepted" && outcome.state === "completed") return "recovered";
      if (outcome.status === "rejected") return "dead";
      return "waiting";
    });
  }

  async function runSweep(): Promise<{ processed: number; recovered: number; deadLettered: number }> {
    const records = await kv.list<InboxRecord>(KV.captureInbox);
    inboxSize = records.length;
    let processed = 0;
    let recovered = 0;
    let deadLettered = 0;
    const now = Date.now();
    const due = records
      .filter((r) => r && r.status !== "dead")
      .filter((r) => r.status === "pending" || !r.nextAttemptAt || Date.parse(r.nextAttemptAt) <= now)
      .sort((a, b) => a.acceptedAt.localeCompare(b.acceptedAt));
    for (const rec of due) {
      const outcome = await retryOne(rec.key, false).catch(() => "waiting" as const);
      if (outcome === "gone") continue;
      processed++;
      if (outcome === "recovered") recovered++;
      if (outcome === "dead") deadLettered++;
    }
    lastSweepAt = new Date().toISOString();
    return { processed, recovered, deadLettered };
  }

  function sweep() {
    if (!sweeping) {
      sweeping = runSweep().finally(() => {
        sweeping = null;
      });
    }
    return sweeping;
  }

  async function prune(): Promise<number> {
    const cutoff = Date.now() - policy.dedupRetentionHours * 3600_000;
    const perShard = Math.max(1, Math.ceil(policy.eventsMax / EVENT_SHARDS));
    let removed = 0;
    for (let i = 0; i < EVENT_SHARDS; i++) {
      const scope = KV.captureEvents(i.toString(16).padStart(EVENT_SHARD_CHARS, "0"));
      const events = await kv.list<CompletedEvent>(scope).catch(() => [] as CompletedEvent[]);
      if (events.length === 0) continue;
      const sorted = events.filter(Boolean).sort((a, b) => b.completedAt.localeCompare(a.completedAt));
      for (let j = 0; j < sorted.length; j++) {
        const ev = sorted[j]!;
        if (j >= perShard || Date.parse(ev.deletedAt ?? ev.completedAt) < cutoff) {
          await kv.delete(scope, ev.key).catch(() => {});
          removed++;
        }
      }
    }
    const inbox = await kv.list<InboxRecord>(KV.captureInbox).catch(() => [] as InboxRecord[]);
    const dead = inbox.filter((r) => r && r.status === "dead").sort((a, b) => (b.deadAt ?? "").localeCompare(a.deadAt ?? ""));
    for (let j = 0; j < dead.length; j++) {
      const rec = dead[j]!;
      if (j >= policy.deadMax || Date.parse(rec.deadAt ?? rec.updatedAt) < cutoff) {
        await kv.delete(KV.captureInbox, rec.key).catch(() => {});
        removed++;
      }
    }
    removed += await prunePromptLedgers(kv, cutoff, policy.eventsMax);
    inboxSize = null;
    counters.pruned += removed;
    lastPruneAt = new Date().toISOString();
    return removed;
  }

  function spoolSender(url: string, dir: string) {
    return async (record: SpoolRecord): Promise<SendOutcome> => {
      const body = record.body as Record<string, unknown>;
      const payload = spoolPayload(body);
      if (!payload) return "rejected";
      const outcome = await capture({ payload, eventId: record.eventId });
      if (outcome.status === "rejected") return outcome.retryable ? "retry" : "rejected";
      retainSent(url, record.eventId, body, mark, { dir });
      return outcome.status === "duplicate" ? "duplicate" : "delivered";
    };
  }

  async function settleRetained(): Promise<number> {
    let requeued = 0;
    for (const target of spoolTargets) {
      const { url, dir } = splitTarget(target);
      const settled = reconcileSent(url, mark.bootId, { dir });
      if (settled.requeued > 0) {
        await drainSpool(url, spoolSender(url, dir), { dir, policy: spoolPolicy() });
        requeued += settled.requeued;
      }
    }
    return requeued;
  }

  async function drainLocalSpool(): Promise<DrainResult[]> {
    const results: DrainResult[] = [];
    for (const target of spoolTargets) {
      const { url, dir } = splitTarget(target);
      reconcileSent(url, mark.bootId, { dir });
      results.push(await drainSpool(url, spoolSender(url, dir), { dir, policy: spoolPolicy() }));
    }
    const total = results.reduce<DrainResult>(
      (acc, r) => ({
        claimed: acc.claimed + r.claimed,
        delivered: acc.delivered + r.delivered,
        duplicates: acc.duplicates + r.duplicates,
        rejected: acc.rejected + r.rejected,
        expired: acc.expired + r.expired,
        corrupt: acc.corrupt + r.corrupt,
        remaining: acc.remaining + r.remaining,
        ...(r.error || acc.error ? { error: r.error ?? acc.error } : {}),
      }),
      { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 },
    );
    lastBootDrain = { ...total, at: new Date().toISOString() };
    return results;
  }

  async function status(): Promise<CaptureStatus> {
    let inbox: CaptureStatus["inbox"] = null;
    try {
      const records = (await kv.list<InboxRecord>(KV.captureInbox)).filter(Boolean);
      inboxSize = records.length;
      const live = records.filter((r) => r.status !== "dead");
      const next = live
        .map((r) => r.nextAttemptAt)
        .filter((v): v is string => typeof v === "string")
        .sort()[0];
      const lastFailed = records
        .filter((r) => r.lastError)
        .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))[0];
      inbox = {
        pending: records.filter((r) => r.status === "pending").length,
        retrying: records.filter((r) => r.status === "retrying").length,
        dead: records.filter((r) => r.status === "dead").length,
        oldestAcceptedAt: live.map((r) => r.acceptedAt).sort()[0] ?? null,
        nextAttemptAt: next ?? null,
        lastError: lastFailed?.lastError ?? null,
      };
    } catch {}
    const seen = new Set<string>();
    const spool: SpoolSummary[] = [];
    for (const target of spoolTargets) {
      const { url, dir } = splitTarget(target);
      const summary = spoolSummary(url, { dir });
      if (seen.has(summary.path)) continue;
      seen.add(summary.path);
      spool.push(summary);
    }
    return { policy, inbox, sinceStart: { ...counters }, lastSweepAt, lastPruneAt, lastBootDrain, spool };
  }

  sdk.registerFunction("mem::capture", async (input: { payload: HookPayload; eventId?: string }) => capture(input));

  sdk.registerFunction("mem::capture-retry", async (input: { eventId?: string; key?: string; all?: boolean } = {}) => {
    const records = (await kv.list<InboxRecord>(KV.captureInbox)).filter(Boolean);
    const targets = records.filter((r) => {
      if (input.key) return r.key === input.key;
      if (input.eventId) return r.eventId === input.eventId;
      return input.all === true;
    });
    const outcomes = { recovered: 0, dead: 0, waiting: 0 };
    for (const rec of targets) {
      const outcome = await retryOne(rec.key, true);
      if (outcome === "recovered") outcomes.recovered++;
      else if (outcome === "dead") outcomes.dead++;
      else if (outcome === "waiting") outcomes.waiting++;
    }
    return { matched: targets.length, ...outcomes };
  });

  sdk.registerFunction("mem::capture-list", async (input: { status?: InboxState; limit?: number } = {}) => {
    const limit = Math.min(Math.max(1, Number(input.limit) || 50), 500);
    const records = (await kv.list<InboxRecord>(KV.captureInbox))
      .filter((r) => r && (!input.status || r.status === input.status))
      .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
    return {
      total: records.length,
      items: records.slice(0, limit).map(({ payload, ...rest }) => ({
        ...rest,
        preview: previewPayload(payload),
      })),
    };
  });

  sdk.registerFunction("mem::capture-drain", async () => {
    const results = await drainLocalSpool();
    return { results };
  });

  const controller: CaptureController = {
    sweep,
    prune,
    drainLocalSpool,
    status,
    durability: () => ({ ...mark }),
    start() {
      if (timers.length) return;
      const sweepTimer = setInterval(() => {
        void sweep().catch(() => {});
        void settleRetained().catch(() => {});
      }, policy.retryIntervalMs);
      sweepTimer.unref();
      const pruneTimer = setInterval(() => {
        void prune().catch(() => {});
      }, 60 * 60 * 1000);
      pruneTimer.unref();
      timers.push(sweepTimer, pruneTimer);
    },
    stop() {
      for (const t of timers.splice(0)) clearInterval(t);
    },
  };
  activeController = controller;
  return controller;
}

function spoolPayload(body: Record<string, unknown>): HookPayload | null {
  const str = (v: unknown) => (typeof v === "string" && v.trim() ? v : undefined);
  const hookType = str(body["hookType"]);
  const sessionId = str(body["sessionId"]);
  const project = str(body["project"]);
  const cwd = str(body["cwd"]);
  const timestamp = str(body["timestamp"]);
  if (!hookType || !sessionId || !project || !cwd || !timestamp) return null;
  return { hookType: hookType as HookPayload["hookType"], sessionId, project, cwd, timestamp, data: body["data"] };
}

function previewPayload(payload: HookPayload | undefined): string {
  if (!payload) return "";
  const d = payload.data && typeof payload.data === "object" ? (payload.data as Record<string, unknown>) : {};
  const label = typeof d["tool_name"] === "string" ? d["tool_name"] : payload.hookType;
  let text = "";
  try {
    text = JSON.stringify(d["tool_input"] ?? d["prompt"] ?? payload.data ?? "");
  } catch {}
  return `${label}: ${text}`.slice(0, 160);
}