Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

15.4 KB
Raw Blame
import { stat, readFile } from "node:fs/promises";
import { join } from "node:path";
import type { AuditEntry, AuditMigrationState, AuditQueryResult } from "../types.js";
import { KV, generateId } from "../state/schema.js";
import type { StateKV } from "../state/kv.js";
import { logger } from "../logger.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { getEnvVar, getAuditMigrateMaxBytes } from "../config.js";
import { runtimeConfigPath } from "../cli/engine-launch.js";
import { ENGINE_DEFAULT_SAVE_INTERVAL_MS, engineSaveIntervalMs, engineStateConfigPaths } from "../cli/engine-config.js";

const AUDIT_MIGRATE_FUNCTION_ID = "mem::audit-migrate";
const AUDIT_MIGRATION_COPY_CONCURRENCY = 32;
const AUDIT_MIGRATION_DELETE_CONCURRENCY = 8;
const AUDIT_MONTHS_LOCK = KV.auditMonths;
const AUDIT_MIGRATION_LOCK = "mem:audit:migration-lock";
export const AUDIT_MIGRATION_STATE_KEY = "migration";
const AUDIT_MIGRATION_DELETE_WAIT_INTERVALS = 2;
const STATE_STORE_DIR_NAME = "state_store.db";

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function encodeStateIndexSegment(index: string): string {
  let out = "";
  for (const byte of Buffer.from(index, "utf8")) {
    const ch = String.fromCharCode(byte);
    out += /^[A-Za-z0-9\-_.]$/.test(ch)
      ? ch
      : `%${byte.toString(16).toUpperCase().padStart(2, "0")}`;
  }
  return out;
}

function legacyAuditStateFilePath(dataDir: string): string {
  return join(dataDir, STATE_STORE_DIR_NAME, `${encodeStateIndexSegment(KV.audit)}.bin`);
}

export type LegacyAuditProbe =
  | { safe: true; empty: true }
  | { safe: true; empty: false; sizeBytes: number }
  | { safe: false; reason: "too-large"; sizeBytes: number }
  | { safe: false; reason: "unreadable" };

export async function probeLegacyAuditFile(): Promise<LegacyAuditProbe> {
  const dataDir = getEnvVar("AGENTMEMORY_DATA_DIR");
  if (!dataDir) return { safe: false, reason: "unreadable" };

  const storeDir = join(dataDir, STATE_STORE_DIR_NAME);
  try {
    await stat(storeDir);
  } catch {
    return { safe: false, reason: "unreadable" };
  }

  let fileStat;
  try {
    fileStat = await stat(legacyAuditStateFilePath(dataDir));
  } catch (err) {
    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
      return { safe: true, empty: true };
    }
    return { safe: false, reason: "unreadable" };
  }

  if (fileStat.size > getAuditMigrateMaxBytes()) {
    return { safe: false, reason: "too-large", sizeBytes: fileStat.size };
  }
  return { safe: true, empty: false, sizeBytes: fileStat.size };
}

export async function probeLegacyAuditScope(kv: StateKV): Promise<LegacyAuditProbe> {
  let rows: AuditEntry[];
  try {
    rows = await kv.list<AuditEntry>(KV.audit);
  } catch {
    return { safe: false, reason: "unreadable" };
  }
  if (!Array.isArray(rows) || rows.length === 0) return { safe: true, empty: true };
  const sizeBytes = Buffer.byteLength(JSON.stringify(rows), "utf8");
  if (sizeBytes > getAuditMigrateMaxBytes()) {
    return { safe: false, reason: "too-large", sizeBytes };
  }
  return { safe: true, empty: false, sizeBytes };
}

async function auditMigrationDeleteDelayMs(): Promise<number> {
  const dataDir = getEnvVar("AGENTMEMORY_DATA_DIR");
  if (!dataDir) return ENGINE_DEFAULT_SAVE_INTERVAL_MS * AUDIT_MIGRATION_DELETE_WAIT_INTERVALS;
  const configTexts: string[] = [];
  for (const path of engineStateConfigPaths(dataDir, runtimeConfigPath(dataDir))) {
    try {
      configTexts.push(await readFile(path, "utf-8"));
    } catch {}
  }
  return engineSaveIntervalMs(configTexts) * AUDIT_MIGRATION_DELETE_WAIT_INTERVALS;
}

async function readAuditMigrationState(kv: StateKV): Promise<AuditMigrationState | null> {
  return kv.get<AuditMigrationState>(KV.auditMonths, AUDIT_MIGRATION_STATE_KEY);
}

async function writeAuditMigrationState(
  kv: StateKV,
  state: AuditMigrationState,
): Promise<void> {
  await kv.set(KV.auditMonths, AUDIT_MIGRATION_STATE_KEY, state);
}

export function auditMonthOf(timestamp: string): string {
  const parsed = new Date(timestamp);
  const iso = Number.isNaN(parsed.getTime())
    ? new Date().toISOString()
    : parsed.toISOString();
  return iso.slice(0, 7);
}

function monthStartMs(month: string): number {
  const [year, mon] = month.split("-").map(Number);
  return Date.UTC(year, mon - 1, 1);
}

function monthEndMs(month: string): number {
  const [year, mon] = month.split("-").map(Number);
  return Date.UTC(year, mon, 1) - 1;
}

function monthNMonthsAgo(count: number): string {
  const now = new Date();
  const cutoff = new Date(
    Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - count, 1),
  );
  return cutoff.toISOString().slice(0, 7);
}

async function readAuditMonthIndex(kv: StateKV): Promise<string[]> {
  const index = await kv.get<{ months: string[] }>(KV.auditMonths, "index");
  return index?.months ?? [];
}

async function markAuditMonth(kv: StateKV, month: string): Promise<void> {
  await withKeyedLock(AUDIT_MONTHS_LOCK, async () => {
    const months = await readAuditMonthIndex(kv);
    if (months.includes(month)) return;
    const next = [...months, month].sort();
    await kv.set(KV.auditMonths, "index", { months: next });
  });
}

export async function listAuditMonthsDesc(kv: StateKV): Promise<string[]> {
  const months = await readAuditMonthIndex(kv);
  return [...months].sort().reverse();
}

let auditRecordedListener: ((entry: AuditEntry) => void) | null = null;

export function setAuditRecordedListener(listener: ((entry: AuditEntry) => void) | null): void {
  auditRecordedListener = listener;
}

export async function recordAudit(
  kv: StateKV,
  operation: AuditEntry["operation"],
  functionId: string,
  targetIds: string[],
  details: Record<string, unknown> = {},
  qualityScore?: number,
  userId?: string,
): Promise<AuditEntry> {
  const entry: AuditEntry = {
    id: generateId("aud"),
    timestamp: new Date().toISOString(),
    operation,
    userId,
    functionId,
    targetIds,
    details,
    qualityScore,
  };
  const month = auditMonthOf(entry.timestamp);
  await kv.set(KV.auditMonth(month), entry.id, entry);
  await markAuditMonth(kv, month);
  try {
    auditRecordedListener?.(entry);
  } catch {}
  return entry;
}

export async function safeAudit(
  kv: StateKV,
  operation: AuditEntry["operation"],
  functionId: string,
  targetIds: string[],
  details: Record<string, unknown> = {},
  qualityScore?: number,
  userId?: string,
): Promise<void> {
  try {
    await recordAudit(kv, operation, functionId, targetIds, details, qualityScore, userId);
  } catch (err) {
    try {
      logger.warn("audit write failed", {
        functionId,
        operation,
        targetIds,
        error: err instanceof Error ? err.message : String(err),
      });
    } catch {}
  }
}

export async function queryAudit(
  kv: StateKV,
  filter?: {
    operation?: AuditEntry["operation"];
    dateFrom?: string;
    dateTo?: string;
    query?: string;
    limit?: number;
  },
): Promise<AuditQueryResult> {
  const limit = filter?.limit || 100;

  let fromMs: number | undefined;
  if (filter?.dateFrom) {
    fromMs = new Date(filter.dateFrom).getTime();
    if (Number.isNaN(fromMs)) {
      throw new Error(`Invalid dateFrom: ${filter.dateFrom}`);
    }
  }
  const query = filter?.query?.trim().toLowerCase();

  let toMs: number | undefined;
  if (filter?.dateTo) {
    toMs = new Date(filter.dateTo).getTime();
    if (Number.isNaN(toMs)) {
      throw new Error(`Invalid dateTo: ${filter.dateTo}`);
    }
  }

  const matches = (entry: AuditEntry): boolean => {
    if (filter?.operation && entry.operation !== filter.operation) return false;
    const t = new Date(entry.timestamp).getTime();
    if (fromMs !== undefined && t < fromMs) return false;
    if (toMs !== undefined && t > toMs) return false;
    if (
      query &&
      ![entry.functionId, ...(entry.targetIds || [])].some((v) =>
        String(v || "").toLowerCase().includes(query),
      )
    ) {
      return false;
    }
    return true;
  };

  const seenIds = new Set<string>();
  const collected: AuditEntry[] = [];
  const addIfNew = (row: AuditEntry): void => {
    if (seenIds.has(row.id)) return;
    seenIds.add(row.id);
    if (matches(row)) collected.push(row);
  };

  const months = await listAuditMonthsDesc(kv);
  for (const month of months) {
    if (collected.length >= limit) break;
    if (toMs !== undefined && monthStartMs(month) > toMs) continue;
    if (fromMs !== undefined && monthEndMs(month) < fromMs) break;
    const rows = await kv.list<AuditEntry>(KV.auditMonth(month));
    for (const row of rows) addIfNew(row);
  }

  let legacyFrozen = false;
  let legacyFrozenBytes: number | undefined;

  if (collected.length < limit) {
    const migrationState = await readAuditMigrationState(kv);
    if (migrationState?.safeToListLegacy) {
      const legacyRows = await kv.list<AuditEntry>(KV.audit);
      for (const row of legacyRows) addIfNew(row);
    } else if (migrationState?.status === "too-large" || migrationState?.status === "unreadable") {
      legacyFrozen = true;
      legacyFrozenBytes = migrationState.legacySizeBytes;
    }
  }

  collected.sort(
    (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(),
  );
  return {
    entries: collected.slice(0, limit),
    legacyFrozen,
    legacyFrozenBytes,
  };
}

async function inParallel<T>(
  items: T[],
  concurrency: number,
  work: (item: T) => Promise<void>,
): Promise<void> {
  let next = 0;
  const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
    while (next < items.length) {
      const item = items[next++];
      await work(item);
    }
  });
  await Promise.all(runners);
}

export interface AuditMigrationDeps {
  probe?: () => Promise<LegacyAuditProbe>;
  deleteDelayMs?: () => Promise<number>;
}

export async function startAuditMigration(
  kv: StateKV,
  deps: AuditMigrationDeps = {},
): Promise<void> {
  const onRedis = kv.backend === "redis";
  const runProbe =
    deps.probe ?? (onRedis ? () => probeLegacyAuditScope(kv) : probeLegacyAuditFile);
  const runDeleteDelayMs =
    deps.deleteDelayMs ?? (onRedis ? async () => 0 : auditMigrationDeleteDelayMs);

  const marker = await readAuditMigrationState(kv);
  if (marker?.status === "done") return;

  const probe = await runProbe();

  if (probe.safe && probe.empty) {
    await writeAuditMigrationState(kv, {
      status: "done",
      safeToListLegacy: true,
      legacySizeBytes: 0,
      migrated: marker?.migrated ?? 0,
      purged: marker?.purged ?? 0,
      summaryWritten: marker?.summaryWritten ?? true,
      checkedAt: new Date().toISOString(),
    });
    return;
  }

  if (!probe.safe) {
    await writeAuditMigrationState(kv, {
      status: probe.reason,
      safeToListLegacy: false,
      legacySizeBytes: probe.reason === "too-large" ? probe.sizeBytes : marker?.legacySizeBytes,
      migrated: marker?.migrated ?? 0,
      purged: marker?.purged ?? 0,
      summaryWritten: marker?.summaryWritten ?? false,
      checkedAt: new Date().toISOString(),
    });
    logger.warn("legacy audit log left frozen; migration only reads it locally and under a size cap", {
      reason: probe.reason,
      sizeBytes: probe.reason === "too-large" ? probe.sizeBytes : undefined,
    });
    return;
  }

  let migrated = marker?.migrated ?? 0;
  let purged = marker?.purged ?? 0;

  if (marker?.status !== "copied") {
    let legacy: AuditEntry[];
    try {
      legacy = await kv.list<AuditEntry>(KV.audit);
    } catch (err) {
      logger.warn("audit migration could not read the legacy audit log", {
        error: err instanceof Error ? err.message : String(err),
      });
      return;
    }
    if (legacy.length === 0) {
      await writeAuditMigrationState(kv, {
        status: "done",
        safeToListLegacy: true,
        legacySizeBytes: 0,
        migrated,
        purged,
        summaryWritten: marker?.summaryWritten ?? true,
        checkedAt: new Date().toISOString(),
      });
      return;
    }

    const seen = new Set<string>();
    const deduped = legacy.filter((entry) => {
      if (seen.has(entry.id)) return false;
      seen.add(entry.id);
      return true;
    });
    const kept = deduped.filter((entry) => entry.operation !== "index_persist");
    purged = deduped.length - kept.length;
    migrated = kept.length;

    try {
      const months = new Set<string>();
      await inParallel(kept, AUDIT_MIGRATION_COPY_CONCURRENCY, async (entry) => {
        const month = auditMonthOf(entry.timestamp);
        await kv.set(KV.auditMonth(month), entry.id, entry);
        months.add(month);
      });
      for (const month of months) await markAuditMonth(kv, month);
    } catch (err) {
      logger.warn(
        "audit migration stopped before finishing the copy pass; it resumes on the next boot",
        { error: err instanceof Error ? err.message : String(err) },
      );
      return;
    }

    await writeAuditMigrationState(kv, {
      status: "copied",
      safeToListLegacy: false,
      legacySizeBytes: probe.sizeBytes,
      migrated,
      purged,
      summaryWritten: false,
      checkedAt: new Date().toISOString(),
    });
  }

  await sleep(await runDeleteDelayMs());

  let remaining: AuditEntry[];
  try {
    remaining = await kv.list<AuditEntry>(KV.audit);
  } catch (err) {
    logger.warn(
      "audit migration could not re-read the legacy audit log before deleting; it resumes on the next boot",
      { error: err instanceof Error ? err.message : String(err) },
    );
    return;
  }

  try {
    await inParallel(
      remaining.slice().reverse(),
      AUDIT_MIGRATION_DELETE_CONCURRENCY,
      async (entry) => {
        await kv.delete(KV.audit, entry.id);
      },
    );
  } catch (err) {
    logger.warn(
      "audit migration stopped before finishing the delete pass; it resumes on the next boot",
      { error: err instanceof Error ? err.message : String(err) },
    );
    return;
  }

  await withKeyedLock(AUDIT_MIGRATION_LOCK, async () => {
    const current = await readAuditMigrationState(kv);
    if (current?.summaryWritten) return;
    if (purged > 0) {
      await safeAudit(kv, "audit_migrate", AUDIT_MIGRATE_FUNCTION_ID, [], {
        reason: "index_persist rows predate opt-in auditing",
        purged,
        migrated,
      });
    }
    await writeAuditMigrationState(kv, {
      status: "done",
      safeToListLegacy: true,
      legacySizeBytes: 0,
      migrated,
      purged,
      summaryWritten: true,
      checkedAt: new Date().toISOString(),
    });
  });

  logger.info("audit log moved to monthly scopes", { migrated, purged });
}

export async function runAuditRetentionSweep(
  kv: StateKV,
  retentionMonths: number,
): Promise<{ droppedMonths: string[]; droppedRows: number }> {
  if (!Number.isFinite(retentionMonths) || retentionMonths <= 0) {
    return { droppedMonths: [], droppedRows: 0 };
  }

  const months = await readAuditMonthIndex(kv);
  const cutoff = monthNMonthsAgo(retentionMonths);
  const toDrop = months.filter((month) => month < cutoff).sort();
  if (toDrop.length === 0) {
    return { droppedMonths: [], droppedRows: 0 };
  }

  let droppedRows = 0;
  for (const month of toDrop) {
    const rows = await kv.list<AuditEntry>(KV.auditMonth(month));
    for (const row of rows.slice().reverse()) {
      await kv.delete(KV.auditMonth(month), row.id);
      droppedRows++;
    }
  }

  await withKeyedLock(AUDIT_MONTHS_LOCK, async () => {
    const current = await readAuditMonthIndex(kv);
    const remaining = current.filter((month) => !toDrop.includes(month)).sort();
    await kv.set(KV.auditMonths, "index", { months: remaining });
  });

  return { droppedMonths: toDrop, droppedRows };
}