Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

19.3 KB
Raw Blame
import { homedir } from "node:os";
import { lstat, readFile, readdir } from "node:fs/promises";
import { resolve, join } from "node:path";
import { allowedFileRoots, confinePath } from "./path-guard.js";
import { scrubRecord } from "./privacy.js";
import type { IIIClient } from "iii-sdk";
import type {
  CompressedObservation,
  Crystal,
  Lesson,
  RawObservation,
  Session,
} from "../types.js";
import { importOrigin } from "../types.js";
import type { StateKV } from "../state/kv.js";
import { KV, generateId, fingerprintId } from "../state/schema.js";
import { addSessionToProjectIndex } from "../state/session-index.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { parseJsonlText } from "../replay/jsonl-parser.js";
import { checkPayloadFrameSize } from "../state/frame-guard.js";
import { resetLessonIndex } from "./lessons.js";
import { projectTimeline, type Timeline } from "../replay/timeline.js";
import { safeAudit } from "./audit.js";
import { buildSyntheticCompression } from "./compress-synthetic.js";
import { indexRecords } from "./search.js";
import { logger } from "../logger.js";
import { rawFromObservationSource } from "./observation-source.js";
import { budgetImportedObservationSources } from "./observation-source-budget.js";

export const MAX_FILES_DEFAULT = 200;
export const MAX_FILES_UPPER_BOUND = 1000;

const SENSITIVE_PATH_PATTERNS: RegExp[] = [
  /(^|[\\/_.-])secret([\\/_.-]|s?$)/i,
  /(^|[\\/_.-])credentials?([\\/_.-]|$)/i,
  /(^|[\\/_.-])private[_-]?key([\\/_.-]|$)/i,
  /(^|[\\/])\.env(\.[\w-]+)?$/i,
  /(^|[\\/_.-])id_rsa([\\/_.-]|$)/i,
  /(^|[\\/])auth[_-]?token([\\/_.-]|$)/i,
  /(^|[\\/])bearer[_-]?token([\\/_.-]|$)/i,
  /(^|[\\/])access[_-]?token([\\/_.-]|$)/i,
  /(^|[\\/])api[_-]?token([\\/_.-]|$)/i,
];

export function isSensitive(path: string): boolean {
  return SENSITIVE_PATH_PATTERNS.some((re) => re.test(path));
}

async function isSymlink(path: string): Promise<boolean> {
  try {
    const st = await lstat(path);
    return st.isSymbolicLink();
  } catch {
    return false;
  }
}

function splitSyntheticNarrative(narrative: string): { input: unknown; output: string } | null {
  if (!narrative.startsWith("{")) return null;
  let at = narrative.indexOf("} | ");
  while (at >= 0) {
    try {
      return { input: JSON.parse(narrative.slice(0, at + 1)), output: narrative.slice(at + 4) };
    } catch {}
    at = narrative.indexOf("} | ", at + 1);
  }
  try {
    return { input: JSON.parse(narrative), output: "" };
  } catch {
    return null;
  }
}

export function rawFromCompressed(obs: CompressedObservation): RawObservation {
  const retained = rawFromObservationSource(obs);
  if (retained) return retained;
  const isPrompt = obs.type === "conversation";
  const narrative = obs.narrative || "";
  const split = isPrompt ? null : splitSyntheticNarrative(narrative);
  return {
    id: obs.id,
    sessionId: obs.sessionId,
    timestamp: obs.timestamp,
    hookType: isPrompt ? "prompt_submit" : obs.type === "error" ? "post_tool_failure" : "post_tool_use",
    toolName: isPrompt ? undefined : obs.title || undefined,
    toolInput: isPrompt ? undefined : split ? split.input : obs.subtitle || undefined,
    toolOutput: isPrompt ? undefined : (split ? split.output : narrative) || undefined,
    userPrompt: isPrompt ? narrative : undefined,
    assistantResponse: undefined,
    raw: { title: obs.title, narrative: obs.narrative, facts: obs.facts },
  };
}

const LESSON_PATTERNS: RegExp[] = [
  /\b(always|never|don'?t|do not|make sure|remember to|note:|caveat:|warning:)\b[^.\n]{10,200}[.!\n]/gi,
  /\b(prefer|avoid)\s[^.\n]{10,200}[.!\n]/gi,
];

async function deriveCrystalAndLessons(
  kv: StateKV,
  sessionId: string,
  project: string,
  rawObs: RawObservation[],
  compressed: CompressedObservation[],
  firstPrompt: string | undefined,
): Promise<void> {
  if (rawObs.length === 0) return;
  const createdAt = new Date().toISOString();

  const files = new Set<string>();
  const tools = new Set<string>();
  for (const c of compressed) {
    for (const f of c.files || []) files.add(f);
    if (c.type && c.type !== "conversation" && c.title) tools.add(c.title);
  }

  const assistantTexts: string[] = [];
  const userPrompts: string[] = [];
  for (const r of rawObs) {
    if (typeof r.assistantResponse === "string" && r.assistantResponse.trim()) {
      assistantTexts.push(r.assistantResponse);
    }
    if (typeof r.userPrompt === "string" && r.userPrompt.trim()) {
      userPrompts.push(r.userPrompt);
    }
  }

  const lessonMatches = new Map<string, string>();
  for (const text of assistantTexts.concat(userPrompts).slice(0, 200)) {
    for (const pat of LESSON_PATTERNS) {
      pat.lastIndex = 0;
      let m: RegExpExecArray | null;
      while ((m = pat.exec(text)) !== null && lessonMatches.size < 40) {
        const snippet = m[0].replace(/\s+/g, " ").trim();
        if (snippet.length >= 20 && snippet.length <= 220) {
          const key = snippet.toLowerCase();
          if (!lessonMatches.has(key)) lessonMatches.set(key, snippet);
        }
      }
    }
  }

  const lessonEntries = Array.from(lessonMatches.values()).slice(0, 20);
  const lessonIds: string[] = [];
  for (const content of lessonEntries) {
    // Content-addressed ID so re-importing the same JSONL does not
    // duplicate lessons. fingerprintId hashes the normalized content,
    // giving a stable lesson_xxx for identical text.
    const lessonId = fingerprintId("lesson", content.trim().toLowerCase());
    try {
      const existing = await kv.get<Lesson>(KV.lessons, lessonId);
      if (existing) {
        const existingSources = existing.sourceIds || [];
        const mergedSources = existingSources.includes(sessionId)
          ? existingSources
          : [...existingSources, sessionId];
        const existingTags = existing.tags || [];
        const mergedTags = existingTags.includes("auto-import")
          ? existingTags
          : [...existingTags, "auto-import"];
        const merged: Lesson = {
          ...existing,
          sourceIds: mergedSources,
          tags: mergedTags,
          reinforcements: (existing.reinforcements || 0) + 1,
          updatedAt: createdAt,
          lastReinforcedAt: createdAt,
        };
        await kv.set(KV.lessons, lessonId, merged);
      } else {
        const lesson: Lesson = {
          id: lessonId,
          content,
          context: firstPrompt || project,
          confidence: 0.4,
          reinforcements: 0,
          source: "consolidation",
          sourceIds: [sessionId],
          project,
          tags: ["auto-import"],
          createdAt,
          updatedAt: createdAt,
          decayRate: 0.05,
        };
        await kv.set(KV.lessons, lessonId, lesson);
      }
      lessonIds.push(lessonId);
    } catch {}
  }
  if (lessonIds.length > 0) resetLessonIndex();

  // Content-addressed on sessionId so re-importing the same session
  // upserts the crystal in place instead of creating a new one.
  const crystalId = fingerprintId("crystal", sessionId);
  const narrativePreview = firstPrompt
    ? firstPrompt.slice(0, 300)
    : compressed
        .slice(0, 5)
        .map((c) => c.narrative || c.title)
        .filter(Boolean)
        .join(" · ")
        .slice(0, 300);

  try {
    const existingCrystal = await kv.get<Crystal>(KV.crystals, crystalId);
    const crystal: Crystal = {
      id: crystalId,
      narrative: narrativePreview || `Session ${sessionId.slice(0, 12)} (${rawObs.length} observations)`,
      keyOutcomes: Array.from(tools).slice(0, 8),
      filesAffected: Array.from(files).slice(0, 20),
      lessons: lessonIds,
      sourceActionIds: existingCrystal?.sourceActionIds ?? [],
      sessionId,
      project,
      createdAt: existingCrystal?.createdAt ?? createdAt,
    };
    await kv.set(KV.crystals, crystalId, crystal);
  } catch {}
}

function isRawShape(o: unknown): o is RawObservation {
  if (!o || typeof o !== "object") return false;
  const r = o as Record<string, unknown>;
  return typeof r.hookType === "string";
}

async function loadObservations(
  kv: StateKV,
  sessionId: string,
): Promise<{ observations: RawObservation[]; sourceRetention: SourceRetention[] }> {
  const rows = await kv.list<RawObservation | CompressedObservation>(
    KV.observations(sessionId),
  );
  return {
    observations: rows.map((r) => (isRawShape(r) ? r : rawFromCompressed(r as CompressedObservation))),
    sourceRetention: rows.map((row) => {
      const source = !isRawShape(row) ? row.source : undefined;
      return {
        observationId: row.id,
        retained: isRawShape(row) || source !== undefined,
        ...(source ? { truncated: source.truncated, originalBytes: source.originalBytes } : {}),
      };
    }),
  };
}

interface SourceRetention {
  observationId: string;
  retained: boolean;
  truncated?: boolean;
  originalBytes?: number;
}

async function findJsonlFiles(
  root: string,
  limit = 200,
): Promise<{
  files: string[];
  truncated: boolean;
  discovered: number;
  traversalCapped: boolean;
}> {
  const out: string[] = [];
  let discovered = 0;
  let walked = 0;
  // Hard bound on entries visited (regardless of extension) so trees
  // dominated by non-jsonl files (node_modules, lockfiles, etc.) cannot
  // lock the 30s function timeout. `discovered` may underrepresent the
  // true count when traversalCapped fires — callers should surface that
  // distinction to the user.
  const traversalCap = Math.max(limit * 50, 50_000);
  async function walk(dir: string) {
    if (walked >= traversalCap) return;
    let names: string[];
    try {
      names = await readdir(dir);
    } catch {
      return;
    }
    for (const name of names) {
      if (walked >= traversalCap) return;
      walked++;
      const full = join(dir, name);
      let st;
      try {
        st = await lstat(full);
      } catch {
        continue;
      }
      if (st.isSymbolicLink()) continue;
      if (st.isDirectory()) {
        await walk(full);
      } else if (st.isFile() && name.endsWith(".jsonl")) {
        discovered++;
        if (out.length < limit) out.push(full);
      }
    }
  }
  await walk(root);
  const traversalCapped = walked >= traversalCap;
  return {
    files: out,
    truncated: discovered > out.length || traversalCapped,
    discovered,
    traversalCapped,
  };
}

export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void {
  sdk.registerFunction(
    "mem::replay::load",
    async (data: { sessionId: string }): Promise<
      | { success: true; timeline: Timeline; session: Session | null; sourceRetention: SourceRetention[] }
      | { success: false; error: string }
    > => {
      if (!data?.sessionId || typeof data.sessionId !== "string") {
        return { success: false, error: "sessionId is required" };
      }
      const session = await kv.get<Session>(KV.sessions, data.sessionId);
      const { observations, sourceRetention } = await loadObservations(kv, data.sessionId);
      const timeline = projectTimeline(observations);
      const result = { success: true as const, timeline, session, sourceRetention };
      return checkPayloadFrameSize(result, "this session is too large to replay in one response") ?? result;
    },
  );

  sdk.registerFunction(
    "mem::replay::sessions",
    async (): Promise<{ success: true; sessions: Session[] }> => {
      const sessions = await kv.list<Session>(KV.sessions);
      sessions.sort((a, b) => (b.startedAt || "").localeCompare(a.startedAt || ""));
      return { success: true, sessions };
    },
  );

  sdk.registerFunction(
    "mem::replay::import-jsonl",
    async (
      data: { path?: string; maxFiles?: number } = {},
    ): Promise<
      | {
          success: true;
          imported: number;
          sessionIds: string[];
          observations: number;
          discovered: number;
          truncated: boolean;
          traversalCapped: boolean;
          maxFiles: number;
          maxFilesUpperBound: number;
          sourceTruncated: number;
          sourceOmitted: number;
        }
      | { success: false; error: string; imported?: number; sessionIds?: string[]; observations?: number }
    > => {
      const defaultRoot = join(homedir(), ".claude", "projects");
      const rawPath = data.path || defaultRoot;
      if (typeof rawPath !== "string" || rawPath.length === 0) {
        return { success: false, error: "path must be a non-empty string" };
      }
      const expanded = rawPath.startsWith("~")
        ? join(homedir(), rawPath.slice(1))
        : rawPath;
      const requested = resolve(expanded);
      if (isSensitive(requested)) {
        return { success: false, error: "refusing to process sensitive-looking path" };
      }
      if (await isSymlink(requested)) {
        return { success: false, error: "symlinks are not supported" };
      }
      const confined = await confinePath(requested, allowedFileRoots([defaultRoot]));
      if (!confined.ok) {
        return { success: false, error: confined.error };
      }
      const abs = confined.path;

      let stat;
      try {
        stat = await lstat(abs);
      } catch {
        return { success: false, error: "path not found" };
      }

      // Valid integer requests are clamped to MAX_FILES_UPPER_BOUND so
      // callers see a stable maxFiles in the response. Non-integer or
      // <= 0 falls back to the safe default. The HTTP layer rejects
      // out-of-range up front; this is the SDK-callable safety net.
      const maxFiles =
        Number.isInteger(data.maxFiles) && (data.maxFiles as number) > 0
          ? Math.min(data.maxFiles as number, MAX_FILES_UPPER_BOUND)
          : MAX_FILES_DEFAULT;
      let files: string[] = [];
      let truncated = false;
      let discovered = 0;
      let traversalCapped = false;
      if (stat.isDirectory()) {
        const found = await findJsonlFiles(abs, maxFiles);
        files = found.files;
        truncated = found.truncated;
        discovered = found.discovered;
        traversalCapped = found.traversalCapped;
      } else if (stat.isFile() && abs.endsWith(".jsonl")) {
        files = [abs];
        discovered = 1;
      } else {
        return { success: false, error: "path must be a .jsonl file or directory" };
      }

      if (files.length === 0) {
        return {
          success: true,
          imported: 0,
          sessionIds: [],
          observations: 0,
          discovered,
          truncated,
          traversalCapped,
          maxFiles,
          maxFilesUpperBound: MAX_FILES_UPPER_BOUND,
          sourceTruncated: 0,
          sourceOmitted: 0,
        };
      }

      const sessionIds: string[] = [];
      let observationCount = 0;
      let sourceTruncated = 0;
      let sourceOmitted = 0;

      for (const file of files) {
        if (isSensitive(file)) continue;
        if (await isSymlink(file)) continue;
        let text: string;
        try {
          text = await readFile(file, "utf-8");
        } catch (err) {
          logger.warn("replay: failed to read jsonl", {
            file,
            error: err instanceof Error ? err.message : String(err),
          });
          continue;
        }

        const parsed = parseJsonlText(text, generateId("sess"));
        if (parsed.observations.length === 0) continue;
        parsed.observations = parsed.observations.map((obs) => scrubRecord(obs));

        const firstPromptObs = parsed.observations.find(
          (o) => typeof o.userPrompt === "string" && o.userPrompt.trim().length > 0,
        );
        const firstPrompt = firstPromptObs?.userPrompt
          ? firstPromptObs.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200)
          : undefined;

        const incoming = parsed.observations.map((obs) => {
          const synthetic = buildSyntheticCompression(obs);
          synthetic.origin = importOrigin(synthetic.origin, synthetic.timestamp, "jsonl");
          return synthetic;
        });
        const committed = await withKeyedLock(`obs:${parsed.sessionId}`, async () => {
          const existingRows = await kv.list<CompressedObservation>(KV.observations(parsed.sessionId));
          const planned = budgetImportedObservationSources(incoming, existingRows);
          if (!planned.success) return planned;
          const existing = await kv.get<Session>(KV.sessions, parsed.sessionId);
          const sessionRow: Session = existing ?? {
            id: parsed.sessionId,
            project: parsed.project,
            cwd: parsed.cwd,
            startedAt: parsed.startedAt,
            endedAt: parsed.endedAt,
            status: "completed",
            observationCount: 0,
            tags: ["jsonl-import"],
            firstPrompt,
          };
          sessionRow.id ||= parsed.sessionId;
          const knownIds = new Set(existingRows.map((row) => row.id));
          const added = planned.observations.filter((obs) => !knownIds.has(obs.id)).length;
          sessionRow.observationCount = (sessionRow.observationCount || 0) + added;
          if (parsed.endedAt > (sessionRow.endedAt || "")) sessionRow.endedAt = parsed.endedAt;
          if (sessionRow.status === "active") sessionRow.status = "completed";
          if (!sessionRow.tags?.includes("jsonl-import")) sessionRow.tags = [...(sessionRow.tags || []), "jsonl-import"];
          if (!sessionRow.firstPrompt && firstPrompt) sessionRow.firstPrompt = firstPrompt;
          await kv.set(KV.sessions, parsed.sessionId, sessionRow);
          for (let i = 0; i < planned.observations.length; i += 20) {
            const batch = planned.observations.slice(i, i + 20);
            const results = await Promise.allSettled(batch.map((obs) => kv.set(KV.observations(parsed.sessionId), obs.id, obs)));
            const failure = results.find((result) => result.status === "rejected");
            if (failure) throw failure.reason;
          }
          return { ...planned, sessionRow, added };
        });
        if (!committed.success) return { ...committed, imported: sessionIds.length, sessionIds, observations: observationCount };
        const { sessionRow, observations: compressed } = committed;
        sourceTruncated += committed.sourceTruncated;
        sourceOmitted += committed.sourceOmitted;
        await addSessionToProjectIndex(kv, sessionRow.project, {
          id: sessionRow.id,
          startedAt: sessionRow.startedAt,
          ...(sessionRow.agentId ? { agentId: sessionRow.agentId } : {}),
        }).catch(() => {});

        // BM25 + vector in one path so jsonl-imported observations are
        // reachable by semantic search, not just keyword.
        try {
          await indexRecords(compressed, []);
        } catch (err) {
          logger.warn("Import indexing failed; restart rebuild will recover", {
            error: err instanceof Error ? err.message : String(err),
          });
        }
        observationCount += committed.added;
        sessionIds.push(parsed.sessionId);

        await deriveCrystalAndLessons(
          kv,
          parsed.sessionId,
          parsed.project,
          parsed.observations,
          compressed,
          firstPrompt,
        );
      }

      await safeAudit(kv, "import", "mem::replay::import-jsonl", sessionIds, {
        source: "jsonl",
        path: abs,
        files: files.length,
        observations: observationCount,
      });

      return {
        success: true,
        imported: files.length,
        sessionIds,
        observations: observationCount,
        discovered,
        truncated,
        traversalCapped,
        maxFiles,
        maxFilesUpperBound: MAX_FILES_UPPER_BOUND,
        sourceTruncated,
        sourceOmitted,
      };
    },
  );
}