Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix: env hydration, indexing, consolidation lifecycle, connector activation, hardening (#1136)

Rohit Ghumare6cc9b9f0fe52
diff --git a/AGENTS.md b/AGENTS.md
index ead439f1..762b9b74 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -117,7 +117,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import).
## Current Stats (v0.9.28)
- 53 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all)
-- 128 REST endpoints
+- 129 REST endpoints
- 6 MCP resources, 3 MCP prompts
- 12 hooks, 15 skills
- 260+ iii functions
diff --git a/README.md b/README.md
index 48f4045c..4c6261f9 100644
--- a/README.md
+++ b/README.md
@@ -1498,7 +1498,7 @@ Create `~/.agentmemory/.env`:
<h2 id="api"><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/section-api.svg"><img src="assets/tags/section-api.svg" alt="API" height="32" /></picture></h2>
-128 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer <secret>` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
+129 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer <secret>` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
<details>
<summary>Key endpoints</summary>
diff --git a/docs/recipes/pairings.md b/docs/recipes/pairings.md
index 093abf85..3337fe13 100644
--- a/docs/recipes/pairings.md
+++ b/docs/recipes/pairings.md
@@ -80,6 +80,15 @@ Recipe with agentmemory:
This is the broadest sweep across artifacts that live alongside the code. agentmemory then captures everything the agent does while exploring that graph — the questions you asked, the conclusions, the decisions — so the next session opens with both the graph and the conversation history available.
+agentmemory can also import the graph directly. With `GRAPH_EXTRACTION_ENABLED=true`:
+
+```bash
+curl -X POST localhost:3111/agentmemory/graph/import-graphify \
+ -H 'content-type: application/json' -d '{"cwd": "'"$PWD"'"}'
+```
+
+This merges Graphify's structural entities and relationships (with their EXTRACTED/INFERRED confidence carried over as edge weight) into agentmemory's knowledge graph, so graph retrieval and context injection see codebase structure the developer never touched in a session. Re-importing after `graphify update .` merges instead of duplicating.
+
## How the four projects line up
Four planes, four consumers, four update models. None of them try to do what the others do.
diff --git a/integrations/filesystem-watcher/watcher.mjs b/integrations/filesystem-watcher/watcher.mjs
index d68b2b8d..a73d178f 100644
--- a/integrations/filesystem-watcher/watcher.mjs
+++ b/integrations/filesystem-watcher/watcher.mjs
@@ -248,6 +248,18 @@ export class FilesystemWatcher {
const failures = [];
for (const root of this.roots) {
try {
+ // Validate the root before handing it to fs.watch: on Linux with
+ // Node 24+, fs.watch on a nonexistent path no longer throws
+ // synchronously, so a missing root would otherwise count as
+ // "attached" and be watched-in-name-only. An explicit stat keeps
+ // the failure deterministic across Node versions and platforms.
+ const st = statSync(root, { throwIfNoEntry: false });
+ if (!st) {
+ throw new Error("no such directory");
+ }
+ if (!st.isDirectory()) {
+ throw new Error("not a directory");
+ }
const handle = watch(
root,
{ recursive: true, persistent: true },
diff --git a/plugin/scripts/session-end.mjs b/plugin/scripts/session-end.mjs
index 811c1c26..f19d80ad 100755
--- a/plugin/scripts/session-end.mjs
+++ b/plugin/scripts/session-end.mjs
@@ -30,23 +30,6 @@ async function main() {
body: JSON.stringify({ sessionId }),
signal: AbortSignal.timeout(3e4)
}).catch(() => {});
- if (process.env["CONSOLIDATION_ENABLED"] === "true") {
- fetch(`${REST_URL}/agentmemory/crystals/auto`, {
- method: "POST",
- headers: authHeaders(),
- body: JSON.stringify({ olderThanDays: 0 }),
- signal: AbortSignal.timeout(6e4)
- }).catch(() => {});
- fetch(`${REST_URL}/agentmemory/consolidate-pipeline`, {
- method: "POST",
- headers: authHeaders(),
- body: JSON.stringify({
- tier: "all",
- force: true
- }),
- signal: AbortSignal.timeout(12e4)
- }).catch(() => {});
- }
if (process.env["CLAUDE_MEMORY_BRIDGE"] === "true") fetch(`${REST_URL}/agentmemory/claude-bridge/sync`, {
method: "POST",
headers: authHeaders(),
diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md
index e56e8743..cbd485d3 100644
--- a/plugin/skills/agentmemory-config/REFERENCE.md
+++ b/plugin/skills/agentmemory-config/REFERENCE.md
@@ -3,12 +3,13 @@
Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded.
<!-- AUTOGEN:env START - generated by scripts/skills/generate.ts, do not edit by hand -->
-Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 35 recognized variables:
+Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 36 recognized variables:
- `AGENTMEMORY_AGENT_SCOPE`
- `AGENTMEMORY_ALLOW_AGENT_SDK`
- `AGENTMEMORY_AUTO_COMPRESS`
- `AGENTMEMORY_COMMIT_SHA`
+- `AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS`
- `AGENTMEMORY_COPILOT_MCP_BLOCK`
- `AGENTMEMORY_CWD`
- `AGENTMEMORY_DATA_DIR`
diff --git a/plugin/skills/agentmemory-rest-api/REFERENCE.md b/plugin/skills/agentmemory-rest-api/REFERENCE.md
index d36171de..77e0a5e4 100644
--- a/plugin/skills/agentmemory-rest-api/REFERENCE.md
+++ b/plugin/skills/agentmemory-rest-api/REFERENCE.md
@@ -5,7 +5,7 @@ Generated from `src/triggers/api.ts`. Do not edit the block below by hand; run `
<!-- AUTOGEN:rest START - generated by scripts/skills/generate.ts, do not edit by hand -->
The REST API is the primary surface. All paths are under `http://localhost:3111` (override with `--port`). When `AGENTMEMORY_SECRET` is set, send `Authorization: Bearer $AGENTMEMORY_SECRET`; localhost is otherwise open.
-117 registered endpoints:
+118 registered endpoints:
| Method | Path |
| --- | --- |
@@ -52,6 +52,7 @@ The REST API is the primary surface. All paths are under `http://localhost:3111`
| DELETE | `/agentmemory/governance/memories` |
| POST | `/agentmemory/graph/build` |
| POST | `/agentmemory/graph/extract` |
+| POST | `/agentmemory/graph/import-graphify` |
| POST | `/agentmemory/graph/query` |
| POST | `/agentmemory/graph/reset` |
| POST | `/agentmemory/graph/snapshot-rebuild` |
diff --git a/src/cli.ts b/src/cli.ts
index d763eada..2ae20f08 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -58,6 +58,7 @@ import { renderSplash } from "./cli/splash.js";
import { isFirstRun, readPrefs, resetPrefs, writePrefs } from "./cli/preferences.js";
import { runOnboarding } from "./cli/onboarding.js";
import { setBootVerbose } from "./logger.js";
+import { hydrateProcessEnvFromFile } from "./config.js";
import { VERSION } from "./version.js";
import { getAllTools, ESSENTIAL_TOOLS } from "./mcp/tools-registry.js";
import { knownAgents } from "./cli/connect/index.js";
@@ -83,6 +84,12 @@ setBootVerbose(IS_VERBOSE);
const IS_RESET = args.includes("--reset");
+// Fold ~/.agentmemory/.env into process.env before any port/URL read
+// (getRestPort/getBaseUrl/getStreamPort/getEnginePort) or the --port /
+// --instance / --tools handlers below. Only-if-unset, so a real
+// process.env value — including one just set by a CLI flag — still wins.
+hydrateProcessEnvFromFile();
+
// --version / -V early exit. Print VERSION + exit before any side effects
// (engine boot, env load, dir mkdir). `-v` is taken by --verbose so we
// reserve `-V` (capital) for version per POSIX convention.
diff --git a/src/cli/connect/guidelines.ts b/src/cli/connect/guidelines.ts
new file mode 100644
index 00000000..d1b95593
--- /dev/null
+++ b/src/cli/connect/guidelines.ts
@@ -0,0 +1,196 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+
+// Activating memory for agents that have no auto-capture hooks: write a short
+// "use the memory tools" guideline into each agent's native rules mechanism so
+// the agent proactively calls memory_recall / memory_save. Paths + formats are
+// verified against each agent's OFFICIAL docs (cited per entry below). Claude
+// Code / Codex are omitted here: they already auto-capture via lifecycle hooks.
+
+const START = "<!-- agentmemory:start -->";
+const END = "<!-- agentmemory:end -->";
+
+const GUIDELINE_BODY = `## Agent memory (agentmemory)
+
+You have persistent long-term memory via the agentmemory MCP server. Tools: \`memory_recall\`, \`memory_smart_search\`, \`memory_save\`, \`memory_sessions\`.
+
+- At the START of a task, call \`memory_recall\` (or \`memory_smart_search\`) with the task context to load relevant past decisions, fixes, and preferences before asking the user to repeat anything.
+- When you learn something durable (a decision, a fix, a gotcha, a user preference, a project convention), call \`memory_save\` to persist it.
+- Prefer recalling over re-deriving, and save concise reusable facts rather than transcripts.`;
+
+// "block" -> upsert a marked block inside a shared instructions file
+// "mdc" -> Cursor project rule: dedicated .mdc with alwaysApply frontmatter
+// "steering" -> Kiro steering file: dedicated .md with inclusion:always frontmatter
+// "rule" -> dedicated always-on markdown rule file (own file, safe to own)
+type GuidelineFormat = "block" | "mdc" | "steering" | "rule";
+
+type GuidelineTarget = {
+ // Preferred user-global rules file (absolute). Omitted when the agent has no
+ // clean global rules FILE (UI-only or project-scoped), forcing the project
+ // fallback. Global is preferred so memory activates across all repos.
+ globalPath?: string;
+ // Project fallback, relative to cwd.
+ projectPath: string;
+ format: GuidelineFormat;
+ scope: "global" | "project";
+ source: string; // official doc URL the path/format was verified against
+};
+
+export function guidelineTargets(
+ home: string = homedir(),
+): Record<string, GuidelineTarget> {
+ return {
+ // No global rules FILE (User Rules are UI-only) -> project .cursor/rules/*.mdc
+ cursor: {
+ projectPath: join(".cursor", "rules", "agentmemory.mdc"),
+ format: "mdc",
+ scope: "project",
+ source: "https://cursor.com/docs/rules",
+ },
+ // Cline's global rules dir is a non-standard ~/Documents path; the dedicated
+ // project rule (no frontmatter = always active) is the verified stable form.
+ cline: {
+ projectPath: join(".clinerules", "agentmemory.md"),
+ format: "rule",
+ scope: "project",
+ source: "https://docs.cline.bot/customization/cline-rules",
+ },
+ // Continue does not auto-read AGENTS.md; rules live in .continue/rules/*.md.
+ continue: {
+ projectPath: join(".continue", "rules", "00-agentmemory.md"),
+ format: "rule",
+ scope: "project",
+ source: "https://docs.continue.dev/customize/deep-dives/rules",
+ },
+ zed: {
+ globalPath: join(home, ".config", "zed", "AGENTS.md"),
+ projectPath: "AGENTS.md",
+ format: "block",
+ scope: "global",
+ source: "https://zed.dev/docs/ai/instructions",
+ },
+ // Warp global Rules are UI-managed (no file path) -> project AGENTS.md.
+ warp: {
+ projectPath: "AGENTS.md",
+ format: "block",
+ scope: "project",
+ source: "https://docs.warp.dev/agent-platform/capabilities/rules",
+ },
+ kiro: {
+ globalPath: join(home, ".kiro", "steering", "agentmemory.md"),
+ projectPath: join(".kiro", "steering", "agentmemory.md"),
+ format: "steering",
+ scope: "global",
+ source: "https://kiro.dev/docs/steering",
+ },
+ "gemini-cli": {
+ globalPath: join(home, ".gemini", "GEMINI.md"),
+ projectPath: "GEMINI.md",
+ format: "block",
+ scope: "global",
+ source: "https://github.com/google-gemini/gemini-cli/blob/main/docs/cli/gemini-md.md",
+ },
+ qwen: {
+ globalPath: join(home, ".qwen", "QWEN.md"),
+ projectPath: "QWEN.md",
+ format: "block",
+ scope: "global",
+ source: "https://github.com/QwenLM/qwen-code/blob/main/docs/users/features/memory.md",
+ },
+ opencode: {
+ globalPath: join(home, ".config", "opencode", "AGENTS.md"),
+ projectPath: "AGENTS.md",
+ format: "block",
+ scope: "global",
+ source: "https://opencode.ai/docs/rules",
+ },
+ droid: {
+ globalPath: join(home, ".factory", "AGENTS.md"),
+ projectPath: "AGENTS.md",
+ format: "block",
+ scope: "global",
+ source: "https://docs.factory.ai/cli/configuration/agents-md",
+ },
+ // Antigravity does NOT read AGENTS.md; global rules live in ~/.gemini/GEMINI.md.
+ antigravity: {
+ globalPath: join(home, ".gemini", "GEMINI.md"),
+ projectPath: join(".agents", "rules", "agentmemory.md"),
+ format: "block",
+ scope: "global",
+ source: "https://antigravity.google/docs/rules-workflows",
+ },
+ "copilot-cli": {
+ globalPath: join(home, ".copilot", "copilot-instructions.md"),
+ projectPath: join(".github", "copilot-instructions.md"),
+ format: "block",
+ scope: "global",
+ source:
+ "https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-custom-instructions",
+ },
+ };
+}
+
+function renderDedicated(format: GuidelineFormat): string {
+ if (format === "mdc") {
+ return `---\ndescription: agentmemory long-term memory usage\nalwaysApply: true\n---\n\n${GUIDELINE_BODY}\n`;
+ }
+ if (format === "steering") {
+ return `---\ninclusion: always\n---\n\n${GUIDELINE_BODY}\n`;
+ }
+ // plain dedicated rule (Cline / Continue)
+ return `${GUIDELINE_BODY}\n`;
+}
+
+// Insert or replace the marked block in a shared instructions file, preserving
+// any surrounding user content. Idempotent: re-running updates only our block.
+function upsertBlock(existing: string): string {
+ const block = `${START}\n${GUIDELINE_BODY}\n${END}`;
+ const startIdx = existing.indexOf(START);
+ const endIdx = existing.indexOf(END);
+ if (startIdx !== -1 && endIdx !== -1 && endIdx > startIdx) {
+ return (
+ existing.slice(0, startIdx) + block + existing.slice(endIdx + END.length)
+ );
+ }
+ // Malformed marker state: a lone or reversed marker. Appending here would
+ // let a later run pair the orphan marker with the appended block's twin and
+ // cut the user's content between them. Leave the file untouched.
+ if (startIdx !== -1 || endIdx !== -1) {
+ return existing;
+ }
+ const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n";
+ const lead = existing.length === 0 ? "" : "\n";
+ return `${existing}${sep}${lead}${block}\n`;
+}
+
+export type GuidelineResult =
+ | { kind: "written"; path: string; scope: string; source: string }
+ | { kind: "unchanged"; path: string; scope: string }
+ | { kind: "would-write"; path: string; scope: string }
+ | { kind: "no-target" };
+
+export function writeGuideline(
+ agentName: string,
+ opts: { cwd: string; home?: string; dryRun?: boolean } = { cwd: process.cwd() },
+): GuidelineResult {
+ const home = opts.home ?? homedir();
+ const target = guidelineTargets(home)[agentName];
+ if (!target) return { kind: "no-target" };
+
+ const path = target.globalPath ?? join(opts.cwd, target.projectPath);
+ const dedicated = target.format !== "block";
+
+ const existing = existsSync(path) ? readFileSync(path, "utf8") : "";
+ const next = dedicated ? renderDedicated(target.format) : upsertBlock(existing);
+
+ if (existing === next) {
+ return { kind: "unchanged", path, scope: target.scope };
+ }
+ if (opts.dryRun) {
+ return { kind: "would-write", path, scope: target.scope };
+ }
+ mkdirSync(dirname(path), { recursive: true });
+ writeFileSync(path, next, "utf8");
+ return { kind: "written", path, scope: target.scope, source: target.source };
+}
diff --git a/src/cli/connect/index.ts b/src/cli/connect/index.ts
index 0d9412ca..0014e7d4 100644
--- a/src/cli/connect/index.ts
+++ b/src/cli/connect/index.ts
@@ -2,6 +2,7 @@ import { platform } from "node:os";
import * as p from "@clack/prompts";
import pc from "picocolors";
import type { ConnectAdapter, ConnectOptions, ConnectResult } from "./types.js";
+import { writeGuideline } from "./guidelines.js";
import { adapter as antigravity } from "./antigravity.js";
import { adapter as claudeCode } from "./claude-code.js";
import { adapter as cline } from "./cline.js";
@@ -56,6 +57,7 @@ function parseFlags(args: string[]): {
force: boolean;
all: boolean;
withHooks: boolean;
+ guidelines: boolean;
positional: string[];
} {
const positional: string[] = [];
@@ -63,14 +65,16 @@ function parseFlags(args: string[]): {
let force = false;
let all = false;
let withHooks = false;
+ let guidelines = true; // memory-usage guideline is written by default
for (const a of args) {
if (a === "--dry-run") dryRun = true;
else if (a === "--force") force = true;
else if (a === "--all") all = true;
else if (a === "--with-hooks") withHooks = true;
+ else if (a === "--no-guidelines") guidelines = false;
else if (!a.startsWith("-")) positional.push(a);
}
- return { dryRun, force, all, withHooks, positional };
+ return { dryRun, force, all, withHooks, guidelines, positional };
}
export async function runAdapter(
@@ -88,7 +92,33 @@ export async function runAdapter(
p.log.message(adapter.protocolNote);
}
try {
- return await adapter.install(opts);
+ const result = await adapter.install(opts);
+ // After MCP/hooks are wired, activate memory for hook-less agents by
+ // writing a memory-usage guideline into their native rules file. Best
+ // effort: never fail the connect over the guideline.
+ if (
+ opts.guidelines !== false &&
+ (result.kind === "installed" || result.kind === "already-wired")
+ ) {
+ try {
+ const g = writeGuideline(adapter.name, {
+ cwd: process.cwd(),
+ dryRun: opts.dryRun,
+ });
+ if (g.kind === "written") {
+ p.log.message(
+ ` ${pc.dim("guideline")} ${g.scope} → ${g.path} (memory auto-use)`,
+ );
+ } else if (g.kind === "would-write") {
+ p.log.message(` ${pc.dim("[dry-run] guideline")} → ${g.path}`);
+ }
+ } catch (gerr) {
+ p.log.warn(
+ `${adapter.displayName}: guideline not written (${gerr instanceof Error ? gerr.message : String(gerr)})`,
+ );
+ }
+ }
+ return result;
} catch (err) {
p.log.error(
`${adapter.displayName}: ${err instanceof Error ? err.message : String(err)}`,
@@ -98,7 +128,8 @@ export async function runAdapter(
}
export async function runConnect(args: string[]): Promise<void> {
- const { dryRun, force, all, withHooks, positional } = parseFlags(args);
+ const { dryRun, force, all, withHooks, guidelines, positional } =
+ parseFlags(args);
const allowWindowsAdapter =
positional.length === 1 && positional[0]?.toLowerCase() === "copilot-cli";
if (platform() === "win32" && !allowWindowsAdapter) {
@@ -110,7 +141,7 @@ export async function runConnect(args: string[]): Promise<void> {
return;
}
- const opts: ConnectOptions = { dryRun, force, withHooks };
+ const opts: ConnectOptions = { dryRun, force, withHooks, guidelines };
p.intro("agentmemory connect");
diff --git a/src/cli/connect/types.ts b/src/cli/connect/types.ts
index 169dde13..bae29622 100644
--- a/src/cli/connect/types.ts
+++ b/src/cli/connect/types.ts
@@ -8,6 +8,13 @@ export type ConnectOptions = {
* hooks from dispatching on Codex Desktop. No-op for other adapters.
*/
withHooks?: boolean;
+ /**
+ * When true (default), after wiring the agent's MCP/hooks, also write a
+ * memory-usage guideline into the agent's native rules file so hook-less
+ * agents proactively call memory_recall / memory_save. Disabled with
+ * `--no-guidelines`. No-op for agents without a guideline target.
+ */
+ guidelines?: boolean;
};
export type ConnectAdapter = {
diff --git a/src/config.ts b/src/config.ts
index 3348ac22..e2b52061 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -22,8 +22,20 @@ const ENV_FILE = join(DATA_DIR, ".env");
let warnPremiumModelShown = false;
+// Parsed ~/.agentmemory/.env, memoized for the process lifetime. getMergedEnv()
+// runs on every config getter (~20 of them), so without this cache a single
+// request would readFileSync + reparse the file dozens of times. The file is
+// boot-static, so read it from disk once and reuse the result. Tests that
+// mutate the file between cases reset the module (clearing this via reload) or
+// call __resetEnvFileCache().
+let envFileCache: Record<string, string> | undefined;
+
function loadEnvFile(): Record<string, string> {
- if (!existsSync(ENV_FILE)) return {};
+ if (envFileCache) return envFileCache;
+ if (!existsSync(ENV_FILE)) {
+ envFileCache = {};
+ return envFileCache;
+ }
const content = readFileSync(ENV_FILE, "utf-8");
const vars: Record<string, string> = {};
for (const line of content.split("\n")) {
@@ -43,13 +55,34 @@ function loadEnvFile(): Record<string, string> {
}
vars[key] = val;
}
- return vars;
+ envFileCache = vars;
+ return envFileCache;
+}
+
+// Test hook: clears the memoized .env so the next loadEnvFile() re-reads disk
+// within the same module instance. vi.resetModules() reloads this module and
+// resets the cache on its own; this exists for tests that mutate the file
+// without a module reload.
+export function __resetEnvFileCache(): void {
+ envFileCache = undefined;
}
function hasRealValue(v: string | undefined): v is string {
return typeof v === "string" && v.trim().length > 0;
}
+// Hydrate ~/.agentmemory/.env into process.env at boot. loadEnvFile() is
+// otherwise only consumed via getMergedEnv(), which the many modules that
+// read raw process.env["X"] never call — so .env-only values were silently
+// ignored by them. Copy the file's vars into process.env, but only when the
+// key is currently unset so a real process.env value still wins (this
+// preserves the {...fileEnv, ...process.env} precedence getMergedEnv uses).
+export function hydrateProcessEnvFromFile(): void {
+ for (const [k, v] of Object.entries(loadEnvFile())) {
+ if (process.env[k] === undefined) process.env[k] = v;
+ }
+}
+
function detectProvider(env: Record<string, string>): ProviderConfig {
const maxTokens = parseInt(env["MAX_TOKENS"] || "4096", 10);
@@ -312,15 +345,30 @@ export function isAgentScopeIsolated(): boolean {
return loadAgentScope()?.mode === "isolated";
}
+// Floor for the git-snapshot timer. A zero/negative SNAPSHOT_INTERVAL would
+// make setInterval fire on roughly every event-loop tick, saturating the
+// worker with back-to-back full-state snapshots + git commits. Anything below
+// this floor is treated as a misconfiguration and falls back to the default.
+const SNAPSHOT_INTERVAL_DEFAULT_SECONDS = 3600;
+const MIN_SNAPSHOT_INTERVAL_SECONDS = 1;
+
export function loadSnapshotConfig(): {
enabled: boolean;
interval: number;
dir: string;
} {
const env = getMergedEnv();
+ const rawInterval = safeParseInt(
+ env["SNAPSHOT_INTERVAL"],
+ SNAPSHOT_INTERVAL_DEFAULT_SECONDS,
+ );
+ const interval =
+ rawInterval >= MIN_SNAPSHOT_INTERVAL_SECONDS
+ ? rawInterval
+ : SNAPSHOT_INTERVAL_DEFAULT_SECONDS;
return {
enabled: env["SNAPSHOT_ENABLED"] === "true",
- interval: safeParseInt(env["SNAPSHOT_INTERVAL"], 3600),
+ interval,
dir: env["SNAPSHOT_DIR"] || join(homedir(), ".agentmemory", "snapshots"),
};
}
@@ -400,6 +448,21 @@ export function getConsolidationDecayDays(): number {
return safeParseInt(getMergedEnv()["CONSOLIDATION_DECAY_DAYS"], 30);
}
+// Cooldown between corpus consolidations triggered by session stop. The Stop
+// hook fires per agent turn and posts /session/end, so without this every turn
+// would kick a full LLM semantic-merge + reflect + crystallize. Debounced to at
+// most once per window. Set to 0 to disable the debounce (consolidate on every
+// stop). Default 5 minutes.
+const CONSOLIDATION_COOLDOWN_DEFAULT_MS = 300000;
+
+export function getConsolidationCooldownMs(): number {
+ const raw = safeParseInt(
+ getMergedEnv()["AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS"],
+ CONSOLIDATION_COOLDOWN_DEFAULT_MS,
+ );
+ return raw >= 0 ? raw : CONSOLIDATION_COOLDOWN_DEFAULT_MS;
+}
+
export function isStandaloneMcp(): boolean {
return getMergedEnv()["STANDALONE_MCP"] === "true";
}
diff --git a/src/functions/evict.ts b/src/functions/evict.ts
index 626f4ff1..72260695 100644
--- a/src/functions/evict.ts
+++ b/src/functions/evict.ts
@@ -8,6 +8,7 @@ import type {
} from "../types.js";
import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
+import { isConsolidationEnabled } from "../config.js";
import { recordAudit } from "./audit.js";
import { deleteAccessLog } from "./access-tracker.js";
import { logger } from "../logger.js";
@@ -60,7 +61,9 @@ async function recoverStaleSession(
try {
const result = await sdk.trigger({
function_id: "event::session::stopped",
- payload: { sessionId },
+ // Suppress the per-session consolidation fan-out: eviction runs a
+ // single corpus-wide consolidation pass after all recoveries instead.
+ payload: { sessionId, skipConsolidation: true },
});
if (!isValidRecoveryResult(result)) {
logger.warn("Stale session recovery failed", {
@@ -80,10 +83,20 @@ async function recoverStaleSession(
}
async function runRecoveredSessionConsolidation(sdk: ISdk): Promise<void> {
+ // Same gate as the session-stop path: keyless installs must not fire
+ // no-op LLM consolidation from an eviction sweep either.
+ if (!isConsolidationEnabled()) return;
try {
await sdk.trigger({
function_id: "mem::consolidate-pipeline",
- payload: { tier: "all" },
+ payload: { tier: "all", force: true },
+ });
+ // One crystallization pass for the batch (the per-session fan-out was
+ // suppressed with skipConsolidation), keeping recovered sessions
+ // consistent with normally-stopped ones without the N-fold amplification.
+ await sdk.trigger({
+ function_id: "mem::auto-crystallize",
+ payload: { olderThanDays: 0 },
});
} catch (err) {
logger.warn("Recovered session consolidation failed", {
diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts
index 8b3bc7cc..2e30e070 100644
--- a/src/functions/export-import.ts
+++ b/src/functions/export-import.ts
@@ -29,8 +29,32 @@ import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { VERSION } from "../version.js";
import { recordAudit } from "./audit.js";
+import { indexRecords } from "./search.js";
import { logger } from "../logger.js";
+// Bounded-concurrency chunk size for the import delete/write loops. A
+// "replace" or "merge" of a large export (up to MAX_TOTAL_OBSERVATIONS,
+// ~500k) would otherwise issue hundreds of thousands of sequential state
+// round-trips and blow the 180s function timeout, leaving partial state.
+// 20 keeps per-chunk fan-out low enough not to overwhelm the state
+// backend while collapsing wallclock by ~20x versus the serial path.
+const IMPORT_CHUNK_SIZE = 20;
+
+// Run `fn` over `items` in fixed-size chunks, awaiting each chunk before
+// starting the next. Preserves ordering guarantees across chunks (chunk N
+// fully settles before chunk N+1 begins) while parallelizing within a
+// chunk. Errors propagate — a failing item rejects the whole import, same
+// as the original serial loops.
+async function runChunked<T>(
+ items: readonly T[],
+ fn: (item: T) => Promise<void>,
+): Promise<void> {
+ for (let i = 0; i < items.length; i += IMPORT_CHUNK_SIZE) {
+ const chunk = items.slice(i, i + IMPORT_CHUNK_SIZE);
+ await Promise.all(chunk.map(fn));
+ }
+}
+
export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
sdk.registerFunction("mem::export",
async (data?: { maxSessions?: number; offset?: number }) => {
@@ -265,114 +289,145 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
if (strategy === "replace") {
const existing = await kv.list<Session>(KV.sessions);
- for (const session of existing) {
+ // Collect observation deletes across all sessions, then run them in
+ // one bounded pass: a runChunked nested inside a runChunked callback
+ // multiplies in-flight deletes to chunk-size squared.
+ const obsDeletes: Array<{ sessionId: string; obsId: string }> = [];
+ await runChunked(existing, async (session) => {
await kv.delete(KV.sessions, session.id);
const obs = await kv
.list<CompressedObservation>(KV.observations(session.id))
.catch(() => []);
for (const o of obs) {
- await kv.delete(KV.observations(session.id), o.id);
+ obsDeletes.push({ sessionId: session.id, obsId: o.id });
}
- }
- const existingMem = await kv.list<Memory>(KV.memories);
- for (const m of existingMem) {
- await kv.delete(KV.memories, m.id);
- }
- const existingSummaries = await kv.list<SessionSummary>(KV.summaries);
- for (const s of existingSummaries) {
- await kv.delete(KV.summaries, s.sessionId);
- }
- for (const a of await kv.list<Action>(KV.actions).catch(() => [])) {
- await kv.delete(KV.actions, a.id);
- }
- for (const e of await kv.list<ActionEdge>(KV.actionEdges).catch(() => [])) {
- await kv.delete(KV.actionEdges, e.id);
- }
- for (const r of await kv.list<Routine>(KV.routines).catch(() => [])) {
- await kv.delete(KV.routines, r.id);
- }
- for (const s of await kv.list<Signal>(KV.signals).catch(() => [])) {
- await kv.delete(KV.signals, s.id);
- }
- for (const c of await kv.list<Checkpoint>(KV.checkpoints).catch(() => [])) {
- await kv.delete(KV.checkpoints, c.id);
- }
- for (const s of await kv.list<Sentinel>(KV.sentinels).catch(() => [])) {
- await kv.delete(KV.sentinels, s.id);
- }
- for (const s of await kv.list<Sketch>(KV.sketches).catch(() => [])) {
- await kv.delete(KV.sketches, s.id);
- }
- for (const c of await kv.list<Crystal>(KV.crystals).catch(() => [])) {
- await kv.delete(KV.crystals, c.id);
- }
- for (const f of await kv.list<Facet>(KV.facets).catch(() => [])) {
- await kv.delete(KV.facets, f.id);
- }
- for (const l of await kv.list<Lesson>(KV.lessons).catch(() => [])) {
- await kv.delete(KV.lessons, l.id);
- }
- for (const i of await kv.list<Insight>(KV.insights).catch(() => [])) {
- await kv.delete(KV.insights, i.id);
- }
- for (const n of await kv.list<{ id: string }>(KV.graphNodes).catch(() => [])) {
- await kv.delete(KV.graphNodes, n.id);
- }
- for (const e of await kv.list<{ id: string }>(KV.graphEdges).catch(() => [])) {
- await kv.delete(KV.graphEdges, e.id);
- }
- for (const s of await kv.list<{ id: string }>(KV.semantic).catch(() => [])) {
- await kv.delete(KV.semantic, s.id);
- }
- for (const p of await kv.list<{ id: string }>(KV.procedural).catch(() => [])) {
- await kv.delete(KV.procedural, p.id);
- }
- for (const profile of await kv.list<ProjectProfile>(KV.profiles).catch(() => [])) {
- await kv.delete(KV.profiles, profile.project);
- }
- for (const a of await kv.list<AccessLogExport>(KV.accessLog).catch(() => [])) {
- await kv.delete(KV.accessLog, a.memoryId);
- }
+ });
+ await runChunked(obsDeletes, (d) =>
+ kv.delete(KV.observations(d.sessionId), d.obsId),
+ );
+ await runChunked(await kv.list<Memory>(KV.memories), (m) =>
+ kv.delete(KV.memories, m.id),
+ );
+ await runChunked(
+ await kv.list<SessionSummary>(KV.summaries),
+ (s) => kv.delete(KV.summaries, s.sessionId),
+ );
+ await runChunked(await kv.list<Action>(KV.actions).catch(() => []), (a) =>
+ kv.delete(KV.actions, a.id),
+ );
+ await runChunked(
+ await kv.list<ActionEdge>(KV.actionEdges).catch(() => []),
+ (e) => kv.delete(KV.actionEdges, e.id),
+ );
+ await runChunked(
+ await kv.list<Routine>(KV.routines).catch(() => []),
+ (r) => kv.delete(KV.routines, r.id),
+ );
+ await runChunked(
+ await kv.list<Signal>(KV.signals).catch(() => []),
+ (s) => kv.delete(KV.signals, s.id),
+ );
+ await runChunked(
+ await kv.list<Checkpoint>(KV.checkpoints).catch(() => []),
+ (c) => kv.delete(KV.checkpoints, c.id),
+ );
+ await runChunked(
+ await kv.list<Sentinel>(KV.sentinels).catch(() => []),
+ (s) => kv.delete(KV.sentinels, s.id),
+ );
+ await runChunked(
+ await kv.list<Sketch>(KV.sketches).catch(() => []),
+ (s) => kv.delete(KV.sketches, s.id),
+ );
+ await runChunked(
+ await kv.list<Crystal>(KV.crystals).catch(() => []),
+ (c) => kv.delete(KV.crystals, c.id),
+ );
+ await runChunked(
+ await kv.list<Facet>(KV.facets).catch(() => []),
+ (f) => kv.delete(KV.facets, f.id),
+ );
+ await runChunked(
+ await kv.list<Lesson>(KV.lessons).catch(() => []),
+ (l) => kv.delete(KV.lessons, l.id),
+ );
+ await runChunked(
+ await kv.list<Insight>(KV.insights).catch(() => []),
+ (i) => kv.delete(KV.insights, i.id),
+ );
+ await runChunked(
+ await kv.list<{ id: string }>(KV.graphNodes).catch(() => []),
+ (n) => kv.delete(KV.graphNodes, n.id),
+ );
+ await runChunked(
+ await kv.list<{ id: string }>(KV.graphEdges).catch(() => []),
+ (e) => kv.delete(KV.graphEdges, e.id),
+ );
+ await runChunked(
+ await kv.list<{ id: string }>(KV.semantic).catch(() => []),
+ (s) => kv.delete(KV.semantic, s.id),
+ );
+ await runChunked(
+ await kv.list<{ id: string }>(KV.procedural).catch(() => []),
+ (p) => kv.delete(KV.procedural, p.id),
+ );
+ await runChunked(
+ await kv.list<ProjectProfile>(KV.profiles).catch(() => []),
+ (profile) => kv.delete(KV.profiles, profile.project),
+ );
+ await runChunked(
+ await kv.list<AccessLogExport>(KV.accessLog).catch(() => []),
+ (a) => kv.delete(KV.accessLog, a.memoryId),
+ );
}
- for (const session of importData.sessions) {
+ // Records actually written this run, accumulated for search
+ // indexing after the KV writes settle. Skipped (already-present)
+ // and merge-overwritten rows are already in the index or will be
+ // re-added below, so re-indexing them is harmless; we only skip the
+ // ones the "skip" strategy declined to write.
+ const indexObs: CompressedObservation[] = [];
+ const indexMems: Memory[] = [];
+
+ await runChunked(importData.sessions, async (session) => {
if (strategy === "skip") {
const existing = await kv
.get<Session>(KV.sessions, session.id)
.catch(() => null);
if (existing) {
stats.skipped++;
- continue;
+ return;
}
}
await kv.set(KV.sessions, session.id, session);
stats.sessions++;
- }
+ });
for (const [sessionId, obs] of Object.entries(importData.observations)) {
- for (const o of obs) {
+ await runChunked(obs, async (o) => {
if (strategy === "skip") {
const existing = await kv
.get<CompressedObservation>(KV.observations(sessionId), o.id)
.catch(() => null);
if (existing) {
stats.skipped++;
- continue;
+ return;
}
}
await kv.set(KV.observations(sessionId), o.id, o);
stats.observations++;
- }
+ indexObs.push(o);
+ });
}
- for (const memory of importData.memories) {
+ await runChunked(importData.memories, async (memory) => {
if (strategy === "skip") {
const existing = await kv
.get<Memory>(KV.memories, memory.id)
.catch(() => null);
if (existing) {
stats.skipped++;
- continue;
+ return;
}
}
// Older exports + hand-edited dumps can omit this field.
@@ -381,171 +436,172 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
}
await kv.set(KV.memories, memory.id, memory);
stats.memories++;
- }
+ indexMems.push(memory);
+ });
- for (const summary of importData.summaries) {
+ await runChunked(importData.summaries, async (summary) => {
if (strategy === "skip") {
const existing = await kv
.get<SessionSummary>(KV.summaries, summary.sessionId)
.catch(() => null);
if (existing) {
stats.skipped++;
- continue;
+ return;
}
}
await kv.set(KV.summaries, summary.sessionId, summary);
stats.summaries++;
- }
+ });
if (importData.graphNodes) {
- for (const node of importData.graphNodes) {
+ await runChunked(importData.graphNodes, async (node) => {
if (strategy === "skip") {
const existing = await kv.get(KV.graphNodes, node.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.graphNodes, node.id, node);
- }
+ });
}
if (importData.graphEdges) {
- for (const edge of importData.graphEdges) {
+ await runChunked(importData.graphEdges, async (edge) => {
if (strategy === "skip") {
const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.graphEdges, edge.id, edge);
- }
+ });
}
if (importData.semanticMemories) {
- for (const sem of importData.semanticMemories) {
+ await runChunked(importData.semanticMemories, async (sem) => {
if (strategy === "skip") {
const existing = await kv.get(KV.semantic, sem.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.semantic, sem.id, sem);
- }
+ });
}
if (importData.proceduralMemories) {
- for (const proc of importData.proceduralMemories) {
+ await runChunked(importData.proceduralMemories, async (proc) => {
if (strategy === "skip") {
const existing = await kv.get(KV.procedural, proc.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.procedural, proc.id, proc);
- }
+ });
}
if (importData.profiles) {
- for (const profile of importData.profiles) {
+ await runChunked(importData.profiles, async (profile) => {
if (strategy === "skip") {
const existing = await kv
.get<ProjectProfile>(KV.profiles, profile.project)
.catch(() => null);
if (existing) {
stats.skipped++;
- continue;
+ return;
}
}
await kv.set(KV.profiles, profile.project, profile);
- }
+ });
}
if (importData.actions) {
- for (const action of importData.actions) {
+ await runChunked(importData.actions, async (action) => {
if (strategy === "skip") {
const existing = await kv.get(KV.actions, action.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.actions, action.id, action);
- }
+ });
}
if (importData.actionEdges) {
- for (const edge of importData.actionEdges) {
+ await runChunked(importData.actionEdges, async (edge) => {
if (strategy === "skip") {
const existing = await kv.get(KV.actionEdges, edge.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.actionEdges, edge.id, edge);
- }
+ });
}
if (importData.routines) {
- for (const routine of importData.routines) {
+ await runChunked(importData.routines, async (routine) => {
if (strategy === "skip") {
const existing = await kv.get(KV.routines, routine.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.routines, routine.id, routine);
- }
+ });
}
if (importData.signals) {
- for (const signal of importData.signals) {
+ await runChunked(importData.signals, async (signal) => {
if (strategy === "skip") {
const existing = await kv.get(KV.signals, signal.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.signals, signal.id, signal);
- }
+ });
}
if (importData.checkpoints) {
- for (const checkpoint of importData.checkpoints) {
+ await runChunked(importData.checkpoints, async (checkpoint) => {
if (strategy === "skip") {
const existing = await kv.get(KV.checkpoints, checkpoint.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.checkpoints, checkpoint.id, checkpoint);
- }
+ });
}
if (importData.sentinels) {
- for (const sentinel of importData.sentinels) {
+ await runChunked(importData.sentinels, async (sentinel) => {
if (strategy === "skip") {
const existing = await kv.get(KV.sentinels, sentinel.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.sentinels, sentinel.id, sentinel);
- }
+ });
}
if (importData.sketches) {
- for (const sketch of importData.sketches) {
+ await runChunked(importData.sketches, async (sketch) => {
if (strategy === "skip") {
const existing = await kv.get(KV.sketches, sketch.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.sketches, sketch.id, sketch);
- }
+ });
}
if (importData.crystals) {
- for (const crystal of importData.crystals) {
+ await runChunked(importData.crystals, async (crystal) => {
if (strategy === "skip") {
const existing = await kv.get(KV.crystals, crystal.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.crystals, crystal.id, crystal);
- }
+ });
}
if (importData.facets) {
- for (const facet of importData.facets) {
+ await runChunked(importData.facets, async (facet) => {
if (strategy === "skip") {
const existing = await kv.get(KV.facets, facet.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.facets, facet.id, facet);
- }
+ });
}
if (importData.lessons) {
- for (const lesson of importData.lessons) {
+ await runChunked(importData.lessons, async (lesson) => {
if (strategy === "skip") {
const existing = await kv.get(KV.lessons, lesson.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.lessons, lesson.id, lesson);
- }
+ });
}
if (importData.insights) {
- for (const insight of importData.insights) {
+ await runChunked(importData.insights, async (insight) => {
if (strategy === "skip") {
const existing = await kv.get(KV.insights, insight.id).catch(() => null);
- if (existing) { stats.skipped++; continue; }
+ if (existing) { stats.skipped++; return; }
}
await kv.set(KV.insights, insight.id, insight);
- }
+ });
}
if (importData.accessLogs) {
if (!Array.isArray(importData.accessLogs)) {
@@ -560,20 +616,37 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
const memoryIds = new Set<string>(
importData.memories.map((m) => m.id),
);
- for (const raw of importData.accessLogs) {
+ await runChunked(importData.accessLogs, async (raw) => {
const log = normalizeAccessLog(raw);
- if (!log.memoryId || !memoryIds.has(log.memoryId)) continue;
+ if (!log.memoryId || !memoryIds.has(log.memoryId)) return;
if (strategy === "skip") {
const existing = await kv
.get(KV.accessLog, log.memoryId)
.catch(() => null);
if (existing) {
stats.skipped++;
- continue;
+ return;
}
}
await kv.set(KV.accessLog, log.memoryId, log);
- }
+ });
+ }
+
+ // Imported rows are now in KV but invisible to search: the boot
+ // rebuild gate only fires when BM25 is empty, so on any existing
+ // install (non-empty persisted index) imported observations and
+ // memories never surface via mem::search / smart-search until a
+ // manual rebuild. Add them to BM25 (synchronous) and enqueue the
+ // vector embeddings in batches (one embedBatch call per chunk)
+ // rather than one giant Promise.all over 500k docs. Indexing
+ // failures are logged, not fatal — the KV writes already committed
+ // and the restart rebuild is the backstop.
+ try {
+ await indexRecords(indexObs, indexMems);
+ } catch (err) {
+ logger.warn("Import indexing failed; restart rebuild will recover", {
+ error: err instanceof Error ? err.message : String(err),
+ });
}
logger.info("Import complete", { strategy, ...stats });
diff --git a/src/functions/graph-import.ts b/src/functions/graph-import.ts
new file mode 100644
index 00000000..520decc6
--- /dev/null
+++ b/src/functions/graph-import.ts
@@ -0,0 +1,265 @@
+import { readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+import type { ISdk } from "iii-sdk";
+import type { GraphEdge, GraphEdgeType, GraphNode, GraphNodeType } from "../types.js";
+import { KV, generateId } from "../state/schema.js";
+import type { StateKV } from "../state/kv.js";
+import { persistGraphDelta } from "./graph.js";
+import { recordAudit } from "./audit.js";
+import { logger } from "../logger.js";
+
+// Import graphify's structural knowledge graph (graphify-out/graph.json) into
+// the memory graph, so context injection and graph retrieval see codebase
+// structure the developer never touched in a session. graphify's extraction is
+// deterministic AST analysis; ours is session-derived. The two meet here.
+//
+// Idempotency comes from persistGraphDelta's (type, name) name-index upsert:
+// re-importing after `graphify update .` merges into existing nodes instead of
+// duplicating. Provenance is kept on properties.source so imported structure
+// is distinguishable from session-derived entities.
+
+// Bounds protect the KV store and the iii invocation budget from very large
+// graphs (graphify caps graph.json at 512MiB; a 100k-node import would blow
+// the invocation window). Truncation is reported loudly in the result.
+const MAX_FILE_BYTES = 32 * 1024 * 1024;
+const MAX_NODES = 5000;
+const MAX_EDGES = 20000;
+
+// graphify file_type enum: code|document|paper|image|rationale|concept.
+// Mapped to the closest memory-graph node type; code symbols carrying a file
+// extension in the label are files, the rest are treated as functions.
+function mapNodeType(fileType: unknown, label: string): GraphNodeType {
+ switch (fileType) {
+ case "rationale":
+ return "decision";
+ case "document":
+ case "paper":
+ case "image":
+ case "concept":
+ return "concept";
+ case "code":
+ return /\.[a-z0-9]{1,10}$/i.test(label) ? "file" : "function";
+ default:
+ return /\.[a-z0-9]{1,10}$/i.test(label) ? "file" : "concept";
+ }
+}
+
+// graphify relations observed in its extractors: calls, imports, uses,
+// requires, inherits, references, source. Anything unrecognized stays a
+// generic related_to edge rather than being dropped.
+function mapEdgeType(relation: unknown): GraphEdgeType {
+ switch (relation) {
+ case "imports":
+ case "source":
+ return "imports";
+ case "calls":
+ case "uses":
+ return "uses";
+ case "requires":
+ case "inherits":
+ case "depends_on":
+ return "depends_on";
+ default:
+ return "related_to";
+ }
+}
+
+// graphify edge confidence tags: EXTRACTED (deterministic AST) beats
+// INFERRED (heuristic) beats AMBIGUOUS.
+function mapConfidence(confidence: unknown): number {
+ switch (confidence) {
+ case "EXTRACTED":
+ return 0.9;
+ case "INFERRED":
+ return 0.6;
+ case "AMBIGUOUS":
+ return 0.3;
+ default:
+ return 0.5;
+ }
+}
+
+type RawGraph = {
+ nodes?: unknown;
+ links?: unknown;
+ edges?: unknown;
+};
+
+export type GraphifyImportResult = {
+ success: boolean;
+ error?: string;
+ path?: string;
+ nodesRead?: number;
+ edgesRead?: number;
+ nodesImported?: number;
+ edgesImported?: number;
+ newNodes?: number;
+ newEdges?: number;
+ skippedEdges?: number;
+ truncated?: { nodes: number; edges: number } | null;
+};
+
+export function parseGraphifyGraph(raw: string): {
+ nodes: GraphNode[];
+ edges: GraphEdge[];
+ nodesRead: number;
+ edgesRead: number;
+ skippedEdges: number;
+ truncated: { nodes: number; edges: number } | null;
+} {
+ const parsed = JSON.parse(raw) as RawGraph;
+ const rawNodes = Array.isArray(parsed.nodes) ? parsed.nodes : [];
+ // graphify's clustered output stores edges under "links" (NetworkX
+ // node_link); --no-cluster graphs store them under "edges". Accept both.
+ const rawEdges = Array.isArray(parsed.links)
+ ? parsed.links
+ : Array.isArray(parsed.edges)
+ ? parsed.edges
+ : [];
+
+ const now = new Date().toISOString();
+ const truncatedNodes = Math.max(0, rawNodes.length - MAX_NODES);
+ const truncatedEdges = Math.max(0, rawEdges.length - MAX_EDGES);
+
+ const nodes: GraphNode[] = [];
+ const byGraphifyId = new Map<string, GraphNode>();
+ for (const entry of rawNodes.slice(0, MAX_NODES)) {
+ if (!entry || typeof entry !== "object") continue;
+ const n = entry as Record<string, unknown>;
+ const graphifyId = typeof n.id === "string" || typeof n.id === "number" ? String(n.id) : null;
+ const label =
+ typeof n.label === "string" && n.label.trim()
+ ? n.label.trim()
+ : graphifyId;
+ if (!graphifyId || !label) continue;
+
+ const properties: Record<string, unknown> = { source: "graphify" };
+ if (typeof n.source_file === "string") properties.sourceFile = n.source_file;
+ if (n.community !== undefined) properties.community = n.community;
+ if (typeof n.file_type === "string") properties.fileType = n.file_type;
+
+ const node: GraphNode = {
+ id: generateId("gn"),
+ type: mapNodeType(n.file_type, label),
+ name: label,
+ properties,
+ sourceObservationIds: [],
+ createdAt: now,
+ };
+ nodes.push(node);
+ byGraphifyId.set(graphifyId, node);
+ }
+
+ const edges: GraphEdge[] = [];
+ let skippedEdges = 0;
+ for (const entry of rawEdges.slice(0, MAX_EDGES)) {
+ if (!entry || typeof entry !== "object") {
+ skippedEdges++;
+ continue;
+ }
+ const e = entry as Record<string, unknown>;
+ const source = e.source !== undefined ? String(e.source) : null;
+ const target = e.target !== undefined ? String(e.target) : null;
+ const sourceNode = source ? byGraphifyId.get(source) : undefined;
+ const targetNode = target ? byGraphifyId.get(target) : undefined;
+ if (!sourceNode || !targetNode) {
+ // Endpoint outside the imported node set (dropped by the cap, or a
+ // dangling reference in the file). Skipped, counted, never guessed.
+ skippedEdges++;
+ continue;
+ }
+ edges.push({
+ id: generateId("ge"),
+ type: mapEdgeType(e.relation ?? e.type),
+ sourceNodeId: sourceNode.id,
+ targetNodeId: targetNode.id,
+ weight: mapConfidence(e.confidence),
+ sourceObservationIds: [],
+ createdAt: now,
+ });
+ }
+
+ return {
+ nodes,
+ edges,
+ nodesRead: rawNodes.length,
+ edgesRead: rawEdges.length,
+ skippedEdges,
+ truncated:
+ truncatedNodes > 0 || truncatedEdges > 0
+ ? { nodes: truncatedNodes, edges: truncatedEdges }
+ : null,
+ };
+}
+
+export function registerGraphImportFunction(sdk: ISdk, kv: StateKV): void {
+ sdk.registerFunction(
+ "mem::graph::import-graphify",
+ async (data?: { path?: string; cwd?: string }): Promise<GraphifyImportResult> => {
+ const explicitPath = typeof data?.path === "string" ? data.path : undefined;
+ const cwd = typeof data?.cwd === "string" ? data.cwd : process.cwd();
+ const path = explicitPath ?? join(cwd, "graphify-out", "graph.json");
+
+ try {
+ let size: number;
+ try {
+ size = (await stat(path)).size;
+ } catch {
+ return {
+ success: false,
+ error: `graph.json not found at ${path}. Run graphify first, or pass an explicit path.`,
+ path,
+ };
+ }
+ if (size > MAX_FILE_BYTES) {
+ return {
+ success: false,
+ error: `graph.json is ${Math.round(size / 1024 / 1024)}MB, over the ${MAX_FILE_BYTES / 1024 / 1024}MB import cap.`,
+ path,
+ };
+ }
+
+ const parsed = parseGraphifyGraph(await readFile(path, "utf-8"));
+ const { newNodeCount, newEdgeCount } = await persistGraphDelta(
+ kv,
+ parsed.nodes,
+ parsed.edges,
+ [],
+ );
+
+ await recordAudit(kv, "import", "mem::graph::import-graphify", [], {
+ path,
+ nodesImported: parsed.nodes.length,
+ edgesImported: parsed.edges.length,
+ newNodes: newNodeCount,
+ newEdges: newEdgeCount,
+ });
+
+ logger.info("graphify graph imported", {
+ path,
+ nodes: parsed.nodes.length,
+ edges: parsed.edges.length,
+ newNodes: newNodeCount,
+ newEdges: newEdgeCount,
+ });
+
+ return {
+ success: true,
+ path,
+ nodesRead: parsed.nodesRead,
+ edgesRead: parsed.edgesRead,
+ nodesImported: parsed.nodes.length,
+ edgesImported: parsed.edges.length,
+ newNodes: newNodeCount,
+ newEdges: newEdgeCount,
+ skippedEdges: parsed.skippedEdges,
+ truncated: parsed.truncated,
+ };
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ logger.error("graphify import failed", { path, error: msg });
+ return { success: false, error: msg, path };
+ }
+ },
+ );
+}
diff --git a/src/functions/graph.ts b/src/functions/graph.ts
index bb0d7272..86332a30 100644
--- a/src/functions/graph.ts
+++ b/src/functions/graph.ts
@@ -450,6 +450,148 @@ function parseGraphXml(
return { nodes, edges };
}
+// Shared persistence for a batch of extracted/imported nodes and edges.
+// Factored out of mem::graph-extract so structural importers (graphify)
+// reuse the exact same name-index upsert, degree bookkeeping, and snapshot
+// maintenance — which also makes re-imports idempotent: an existing
+// (type, name) resolves through the name index and merges instead of
+// duplicating.
+//
+// #814 v2: targeted name-index lookups replace the O(n) scan over
+// `kv.list<GraphNode>(KV.graphNodes)`. At 75K nodes the list payload
+// exceeds the iii heartbeat budget and the worker dies before merge can
+// complete. Each name-index entry is a single small kv.get/set pair.
+export async function persistGraphDelta(
+ kv: StateKV,
+ nodes: GraphNode[],
+ edges: GraphEdge[],
+ obsIds: string[],
+): Promise<{ newNodeCount: number; newEdgeCount: number }> {
+ const snap = (await readSnapshot(kv)) ?? emptySnapshot();
+ const capturedAt = new Date().toISOString();
+ let newNodeCount = 0;
+ let newEdgeCount = 0;
+ // Merge-only batches mutate cached topNodes/topEdges entries without
+ // changing the counts; track that separately so the snapshot still persists.
+ let snapMutated = false;
+ const newEdgesForTopCheck: GraphEdge[] = [];
+ // When a freshly-minted node merges into an existing row via the name
+ // index, edges in the same batch still reference the fresh id. Remap edge
+ // endpoints to the persisted ids so edges never dangle and re-runs hit the
+ // same edge-index key instead of duplicating.
+ const idRemap = new Map<string, string>();
+
+ for (const node of nodes) {
+ const indexKey = nameIndexKey(node.type, node.name);
+ const existingId = await kv.get<string>(KV.graphNameIndex, indexKey);
+
+ let existing: GraphNode | null = null;
+ if (existingId) {
+ existing = await kv.get<GraphNode>(KV.graphNodes, existingId);
+ // #825 follow-up: name-index lookups can resolve into
+ // pre-reset rows. Drop them so extract writes a fresh
+ // node + index entry instead of silently reconnecting
+ // to a legacy orphan (which would keep the snapshot at
+ // 0 forever after a reset).
+ if (
+ existing &&
+ snap.resetAt &&
+ typeof existing.createdAt === "string" &&
+ existing.createdAt < snap.resetAt
+ ) {
+ existing = null;
+ }
+ }
+
+ if (existing) {
+ idRemap.set(node.id, existing.id);
+ const merged = mergeNode(existing, node, obsIds, capturedAt);
+ await kv.set(KV.graphNodes, existing.id, merged);
+ // Update topNodes entry if present so a stale clone isn't
+ // returned from the snapshot fast path.
+ const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id);
+ if (topIdx !== -1) {
+ snap.topNodes[topIdx] = merged;
+ snapMutated = true;
+ }
+ } else {
+ await kv.set(KV.graphNodes, node.id, node);
+ await kv.set(KV.graphNameIndex, indexKey, node.id);
+ await kv.set(KV.graphNodeDegree, node.id, 0);
+ snap.stats.totalNodes += 1;
+ snap.stats.nodesByType[node.type] =
+ (snap.stats.nodesByType[node.type] ?? 0) + 1;
+ newNodeCount += 1;
+ if (snap.topNodes.length < SNAPSHOT_TOP_NODES) {
+ // Degree 0 still beats an empty slot — sit at the tail
+ // until edges arrive and promote.
+ snap.topNodes.push(node);
+ snap.topDegrees[node.id] = 0;
+ }
+ }
+ }
+
+ for (const rawEdge of edges) {
+ const edge: GraphEdge = {
+ ...rawEdge,
+ sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId,
+ targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId,
+ };
+ const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type);
+ const existingId = await kv.get<string>(KV.graphEdgeKey, eKey);
+
+ let existing: GraphEdge | null = null;
+ if (existingId) {
+ existing = await kv.get<GraphEdge>(KV.graphEdges, existingId);
+ // Same #825 orphan check as the node path above.
+ if (
+ existing &&
+ snap.resetAt &&
+ typeof existing.createdAt === "string" &&
+ existing.createdAt < snap.resetAt
+ ) {
+ existing = null;
+ }
+ }
+
+ if (existing) {
+ const merged = mergeEdge(existing, obsIds);
+ await kv.set(KV.graphEdges, existing.id, merged);
+ // Replace cached topEdges entry too if present.
+ const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id);
+ if (topIdx !== -1) {
+ snap.topEdges[topIdx] = merged;
+ snapMutated = true;
+ }
+ } else {
+ await kv.set(KV.graphEdges, edge.id, edge);
+ await kv.set(KV.graphEdgeKey, eKey, edge.id);
+ snap.stats.totalEdges += 1;
+ snap.stats.edgesByType[edge.type] =
+ (snap.stats.edgesByType[edge.type] ?? 0) + 1;
+ newEdgeCount += 1;
+ await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1);
+ await applyDegreeDelta(kv, snap, edge.targetNodeId, +1);
+ newEdgesForTopCheck.push(edge);
+ }
+ }
+
+ // Push newly-added edges into snapshot.topEdges if both
+ // endpoints are in the top-N (post-degree-delta). Done after
+ // all degree updates so the topIds set is stable.
+ for (const edge of newEdgesForTopCheck) {
+ snapshotPushEdgeIfBothInTop(snap, edge);
+ }
+
+ if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) {
+ snap.updatedAt = capturedAt;
+ snap.dirty = false;
+ await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);
+ }
+
+ return { newNodeCount, newEdgeCount };
+}
+
export function registerGraphFunction(
sdk: ISdk,
kv: StateKV,
@@ -480,123 +622,12 @@ export function registerGraphFunction(
const obsIds = data.observations.map((o) => o.id);
const { nodes, edges } = parseGraphXml(response, obsIds);
- // #814 v2: targeted name-index lookups replace the O(n) scan
- // over `kv.list<GraphNode>(KV.graphNodes)`. At 75K nodes the
- // list payload exceeds the iii heartbeat budget and the worker
- // dies before merge can complete. Each name-index entry is a
- // single small kv.get/set pair.
- const snap = (await readSnapshot(kv)) ?? emptySnapshot();
- const capturedAt = new Date().toISOString();
- let newNodeCount = 0;
- let newEdgeCount = 0;
- const newEdgesForTopCheck: GraphEdge[] = [];
-
- for (const node of nodes) {
- const indexKey = nameIndexKey(node.type, node.name);
- const existingId = await kv.get<string>(
- KV.graphNameIndex,
- indexKey,
- );
-
- let existing: GraphNode | null = null;
- if (existingId) {
- existing = await kv.get<GraphNode>(KV.graphNodes, existingId);
- // #825 follow-up: name-index lookups can resolve into
- // pre-reset rows. Drop them so extract writes a fresh
- // node + index entry instead of silently reconnecting
- // to a legacy orphan (which would keep the snapshot at
- // 0 forever after a reset).
- if (
- existing &&
- snap.resetAt &&
- typeof existing.createdAt === "string" &&
- existing.createdAt < snap.resetAt
- ) {
- existing = null;
- }
- }
-
- if (existing) {
- const merged = mergeNode(existing, node, obsIds, capturedAt);
- await kv.set(KV.graphNodes, existing.id, merged);
- // Update topNodes entry if present so a stale clone isn't
- // returned from the snapshot fast path.
- const topIdx = snap.topNodes.findIndex(
- (n) => n.id === existing!.id,
- );
- if (topIdx !== -1) snap.topNodes[topIdx] = merged;
- } else {
- await kv.set(KV.graphNodes, node.id, node);
- await kv.set(KV.graphNameIndex, indexKey, node.id);
- await kv.set(KV.graphNodeDegree, node.id, 0);
- snap.stats.totalNodes += 1;
- snap.stats.nodesByType[node.type] =
- (snap.stats.nodesByType[node.type] ?? 0) + 1;
- newNodeCount += 1;
- if (snap.topNodes.length < SNAPSHOT_TOP_NODES) {
- // Degree 0 still beats an empty slot — sit at the tail
- // until edges arrive and promote.
- snap.topNodes.push(node);
- snap.topDegrees[node.id] = 0;
- }
- }
- }
-
- for (const edge of edges) {
- const eKey = edgeIndexKey(
- edge.sourceNodeId,
- edge.targetNodeId,
- edge.type,
- );
- const existingId = await kv.get<string>(KV.graphEdgeKey, eKey);
-
- let existing: GraphEdge | null = null;
- if (existingId) {
- existing = await kv.get<GraphEdge>(KV.graphEdges, existingId);
- // Same #825 orphan check as the node path above.
- if (
- existing &&
- snap.resetAt &&
- typeof existing.createdAt === "string" &&
- existing.createdAt < snap.resetAt
- ) {
- existing = null;
- }
- }
-
- if (existing) {
- const merged = mergeEdge(existing, obsIds);
- await kv.set(KV.graphEdges, existing.id, merged);
- // Replace cached topEdges entry too if present.
- const topIdx = snap.topEdges.findIndex(
- (e) => e.id === existing!.id,
- );
- if (topIdx !== -1) snap.topEdges[topIdx] = merged;
- } else {
- await kv.set(KV.graphEdges, edge.id, edge);
- await kv.set(KV.graphEdgeKey, eKey, edge.id);
- snap.stats.totalEdges += 1;
- snap.stats.edgesByType[edge.type] =
- (snap.stats.edgesByType[edge.type] ?? 0) + 1;
- newEdgeCount += 1;
- await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1);
- await applyDegreeDelta(kv, snap, edge.targetNodeId, +1);
- newEdgesForTopCheck.push(edge);
- }
- }
-
- // Push newly-added edges into snapshot.topEdges if both
- // endpoints are in the top-N (post-degree-delta). Done after
- // all degree updates so the topIds set is stable.
- for (const edge of newEdgesForTopCheck) {
- snapshotPushEdgeIfBothInTop(snap, edge);
- }
-
- if (newNodeCount > 0 || newEdgeCount > 0) {
- snap.updatedAt = capturedAt;
- snap.dirty = false;
- await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);
- }
+ const { newNodeCount, newEdgeCount } = await persistGraphDelta(
+ kv,
+ nodes,
+ edges,
+ obsIds,
+ );
await recordAudit(kv, "observe", "mem::graph-extract", obsIds, {
nodesExtracted: nodes.length,
diff --git a/src/functions/patterns.ts b/src/functions/patterns.ts
index 8d1365fc..ef6821cb 100644
--- a/src/functions/patterns.ts
+++ b/src/functions/patterns.ts
@@ -29,37 +29,54 @@ export function registerPatternsFunction(sdk: ISdk, kv: StateKV): void {
{ count: number; sessions: Set<string> }
>();
- for (const session of filtered) {
- const observations = await kv.list<CompressedObservation>(
- KV.observations(session.id),
+ // Bounded fan-out: load observations for up to 10 sessions in
+ // parallel per batch (like consolidate), then fold each session's
+ // observations into the shared maps serially so the accumulation
+ // stays race-free. Parallelizing the kv.list I/O without exceeding
+ // the invocation pool cuts wall time versus the old serial loop.
+ for (let batch = 0; batch < filtered.length; batch += 10) {
+ const chunk = filtered.slice(batch, batch + 10);
+ const loaded = await Promise.all(
+ chunk.map(async (session) => ({
+ session,
+ observations: await kv.list<CompressedObservation>(
+ KV.observations(session.id),
+ ),
+ })),
);
- if (!observations.length) continue;
-
- const sessionFiles = new Set<string>();
- for (const obs of observations) {
- if (!obs.files) continue;
- for (const f of obs.files) {
- sessionFiles.add(f);
- if (!fileSessionMap.has(f)) fileSessionMap.set(f, new Set());
- fileSessionMap.get(f)!.add(session.id);
- }
- if (obs.type === "error" && obs.title) {
- const key = obs.title.toLowerCase();
- if (!errorPatterns.has(key)) {
- errorPatterns.set(key, { count: 0, sessions: new Set() });
+ for (const { session, observations } of loaded) {
+ if (!observations.length) continue;
+
+ const sessionFiles = new Set<string>();
+ for (const obs of observations) {
+ if (!obs.files) continue;
+ for (const f of obs.files) {
+ sessionFiles.add(f);
+ if (!fileSessionMap.has(f)) fileSessionMap.set(f, new Set());
+ fileSessionMap.get(f)!.add(session.id);
+ }
+
+ if (obs.type === "error" && obs.title) {
+ const key = obs.title.toLowerCase();
+ if (!errorPatterns.has(key)) {
+ errorPatterns.set(key, { count: 0, sessions: new Set() });
+ }
+ const ep = errorPatterns.get(key)!;
+ ep.count++;
+ ep.sessions.add(session.id);
}
- const ep = errorPatterns.get(key)!;
- ep.count++;
- ep.sessions.add(session.id);
}
- }
- const fileList = [...sessionFiles].sort();
- for (let i = 0; i < fileList.length; i++) {
- for (let j = i + 1; j < fileList.length; j++) {
- const pair = `${fileList[i]}::${fileList[j]}`;
- fileCoOccurrences.set(pair, (fileCoOccurrences.get(pair) || 0) + 1);
+ const fileList = [...sessionFiles].sort();
+ for (let i = 0; i < fileList.length; i++) {
+ for (let j = i + 1; j < fileList.length; j++) {
+ const pair = `${fileList[i]}::${fileList[j]}`;
+ fileCoOccurrences.set(
+ pair,
+ (fileCoOccurrences.get(pair) || 0) + 1,
+ );
+ }
}
}
}
diff --git a/src/functions/remember.ts b/src/functions/remember.ts
index 5735b4f2..5cbfb693 100644
--- a/src/functions/remember.ts
+++ b/src/functions/remember.ts
@@ -10,6 +10,15 @@ import { getSearchIndex, vectorIndexAddGuarded, vectorIndexRemove, flushIndexSav
import { getAgentId } from "../config.js";
import { logger } from "../logger.js";
+// Slicing by UTF-16 code unit can cut an astral character (emoji, some CJK
+// extensions) mid surrogate pair, leaving a lone high surrogate that renders
+// as a replacement glyph. Drop a dangling trailing high surrogate so the
+// title stays valid.
+function safeSlice(text: string, length: number): string {
+ const sliced = text.slice(0, length);
+ return /[\uD800-\uDBFF]$/.test(sliced) ? sliced.slice(0, -1) : sliced;
+}
+
export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
sdk.registerFunction("mem::remember",
async (data: {
@@ -100,7 +109,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
createdAt: now,
updatedAt: now,
type: memType,
- title: data.content.slice(0, 80),
+ title: safeSlice(data.content, 80),
content: data.content,
concepts: data.concepts || [],
files: data.files || [],
diff --git a/src/functions/replay.ts b/src/functions/replay.ts
index e44ca8cc..e9185050 100644
--- a/src/functions/replay.ts
+++ b/src/functions/replay.ts
@@ -15,7 +15,7 @@ import { parseJsonlText } from "../replay/jsonl-parser.js";
import { projectTimeline, type Timeline } from "../replay/timeline.js";
import { safeAudit } from "./audit.js";
import { buildSyntheticCompression } from "./compress-synthetic.js";
-import { getSearchIndex } from "./search.js";
+import { indexRecords } from "./search.js";
import { logger } from "../logger.js";
export const MAX_FILES_DEFAULT = 200;
@@ -432,16 +432,23 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
await kv.set(KV.sessions, session.id, session);
}
- const searchIndex = getSearchIndex();
const compressed: CompressedObservation[] = [];
await Promise.all(
parsed.observations.map(async (obs) => {
const synthetic = buildSyntheticCompression(obs);
compressed.push(synthetic);
await kv.set(KV.observations(parsed.sessionId), obs.id, synthetic);
- searchIndex.add(synthetic);
}),
);
+ // BM25 + vector in one path so jsonl-imported observations are
+ // reachable by semantic search, not just keyword.
+ try {
+ await indexRecords(compressed, []);
+ } catch (err) {
+ logger.warn("Import indexing failed; restart rebuild will recover", {
+ error: err instanceof Error ? err.message : String(err),
+ });
+ }
observationCount += parsed.observations.length;
sessionIds.push(parsed.sessionId);
diff --git a/src/functions/search.ts b/src/functions/search.ts
index df699a1a..9bcda6ae 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -14,6 +14,15 @@ let index: SearchIndex | null = null
let vectorIndex: VectorIndex | null = null
let currentEmbeddingProvider: EmbeddingProvider | null = null
+// Dedupes the lazy cold-start rebuild kicked off from the mem::search
+// request path. A full rebuildIndex walks every observation across every
+// session, so N concurrent queries against an empty index would each
+// launch their own rebuild and saturate the engine invocation pool. The
+// first query with an empty index starts one rebuild and shares its
+// promise; concurrent queries await the same rebuild instead of spawning
+// duplicates. The boot-time rebuild in index.ts is unaffected.
+let rebuildPromise: Promise<number> | null = null
+
export function getSearchIndex(): SearchIndex {
if (!index) index = new SearchIndex()
return index
@@ -218,19 +227,20 @@ function getRebuildEmbedBatchSize(): number {
return Number.isFinite(n) && n > 0 ? n : DEFAULT_REBUILD_EMBED_BATCH
}
-export async function rebuildIndex(kv: StateKV): Promise<number> {
+// Shared BM25 + batched-vector indexing for a set of records. The full
+// rebuild and every import path (export-import, jsonl replay) funnel
+// through this so they index identically and none can silently skip the
+// vector side. It does NOT clear the index — callers that rebuild clear
+// first; importers add. When no embedding provider is configured it skips
+// the vector enqueue entirely, so a keyless install never allocates embed
+// jobs it would immediately discard.
+export async function indexRecords(
+ observations: CompressedObservation[],
+ memories: Memory[],
+): Promise<number> {
const idx = getSearchIndex()
- idx.clear()
-
- // BM25 clear above wipes stale doc entries; the vector index has the
- // symmetric concern — memories/observations deleted between runs
- // would leave orphan embeddings here forever. Clear both before the
- // repopulation loops run, so BM25 and vector stay in sync.
- vectorIndex?.clear()
-
+ const vectorEnabled = Boolean(vectorIndex && currentEmbeddingProvider)
const batchSize = getRebuildEmbedBatchSize()
- // Accumulator for the batched embed flush. BM25 add is synchronous and
- // doesn't need batching — only the vector path benefits.
type EmbedJob = {
id: string
sessionId: string
@@ -238,36 +248,62 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {
context: { kind: "memory" | "observation" | "synthetic"; logId: string }
}
const pending: EmbedJob[] = []
- let count = 0
-
const flush = async (): Promise<void> => {
if (pending.length === 0) return
await vectorIndexAddBatchGuarded(pending)
pending.length = 0
}
const enqueue = async (job: EmbedJob): Promise<void> => {
+ if (!vectorEnabled) return
pending.push(job)
if (pending.length >= batchSize) await flush()
}
+ let count = 0
+ for (const memory of memories) {
+ if (memory.isLatest === false) continue
+ if (!memory.title || !memory.content) continue
+ idx.add(memoryToObservation(memory))
+ await enqueue({
+ id: memory.id,
+ sessionId: memory.sessionIds?.[0] ?? 'memory',
+ text: memory.title + ' ' + memory.content,
+ context: { kind: "memory", logId: memory.id },
+ })
+ count++
+ }
+ for (const obs of observations) {
+ if (!obs.title || !obs.narrative) continue
+ idx.add(obs)
+ await enqueue({
+ id: obs.id,
+ sessionId: obs.sessionId,
+ text: obs.title + ' ' + obs.narrative,
+ context: { kind: "observation", logId: obs.id },
+ })
+ count++
+ }
+ await flush()
+ return count
+}
+
+export async function rebuildIndex(kv: StateKV): Promise<number> {
+ const idx = getSearchIndex()
+ idx.clear()
+
+ // BM25 clear above wipes stale doc entries; the vector index has the
+ // symmetric concern — memories/observations deleted between runs
+ // would leave orphan embeddings here forever. Clear both before the
+ // repopulation loops run, so BM25 and vector stay in sync.
+ vectorIndex?.clear()
+
// Memories live in their own KV scope outside per-session observation
// scopes, so they need a separate walk. Without this, mem::remember
// entries vanish from BM25 on every restart even after the live-write
- // fix in remember.ts (#257).
+ // fix in remember.ts.
+ let memories: Memory[] = []
try {
- const memories = await kv.list<Memory>(KV.memories)
- for (const memory of memories) {
- if (memory.isLatest === false) continue
- if (!memory.title || !memory.content) continue
- idx.add(memoryToObservation(memory))
- await enqueue({
- id: memory.id,
- sessionId: memory.sessionIds?.[0] ?? 'memory',
- text: memory.title + ' ' + memory.content,
- context: { kind: "memory", logId: memory.id },
- })
- count++
- }
+ memories = await kv.list<Memory>(KV.memories)
} catch (err) {
logger.warn('rebuildIndex: failed to load memories', {
error: err instanceof Error ? err.message : String(err),
@@ -275,13 +311,10 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {
}
const sessions = await kv.list<Session>(KV.sessions)
- if (!sessions.length) {
- await flush()
- return count
- }
-
- const obsPerSession: CompressedObservation[][] = []
const failedSessions: string[] = []
+ // Index each session chunk as it loads instead of accumulating every
+ // observation first, so peak memory stays bounded to one chunk.
+ let indexed = 0
for (let batch = 0; batch < sessions.length; batch += 10) {
const chunk = sessions.slice(batch, batch + 10)
const results = await Promise.all(
@@ -294,29 +327,17 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {
}
})
)
- obsPerSession.push(...results)
+ const chunkObs = results.flat()
+ if (chunkObs.length > 0) {
+ indexed += await indexRecords(chunkObs, [])
+ }
}
if (failedSessions.length > 0) {
logger.warn('rebuildIndex: failed to load observations for sessions', { failedSessions })
}
- for (const observations of obsPerSession) {
- for (const obs of observations) {
- if (obs.title && obs.narrative) {
- idx.add(obs)
- await enqueue({
- id: obs.id,
- sessionId: obs.sessionId,
- text: obs.title + ' ' + obs.narrative,
- context: { kind: "observation", logId: obs.id },
- })
- count++
- }
- }
- }
- // Drain the last partial batch.
- await flush()
- return count
+ indexed += await indexRecords([], memories)
+ return indexed
}
export function registerSearchFunction(sdk: ISdk, kv: StateKV): void {
@@ -397,8 +418,25 @@ export function registerSearchFunction(sdk: ISdk, kv: StateKV): void {
}
if (idx.size === 0) {
- const count = await rebuildIndex(kv)
- logger.info('Search index rebuilt', { entries: count })
+ // Share one rebuild across concurrent cold-start queries so they
+ // don't each walk the whole corpus and saturate the pool.
+ if (!rebuildPromise) {
+ rebuildPromise = rebuildIndex(kv)
+ .then((count) => {
+ logger.info('Search index rebuilt', { entries: count })
+ return count
+ })
+ .catch((err) => {
+ logger.warn('Index rebuild failed', {
+ error: err instanceof Error ? err.message : String(err),
+ })
+ return 0
+ })
+ .finally(() => {
+ rebuildPromise = null
+ })
+ }
+ await rebuildPromise
}
// When filtering by project/cwd, over-fetch from the index so the
diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts
index 40430ce5..99c45c06 100644
--- a/src/functions/snapshot.ts
+++ b/src/functions/snapshot.ts
@@ -41,8 +41,18 @@ export function registerSnapshotFunction(
kv: StateKV,
snapshotDir: string,
): void {
- sdk.registerFunction("mem::snapshot-create",
+ // Serialize snapshots: the periodic timer, REST (api::snapshot-create), and
+ // MCP can all trigger this concurrently. Two runs writing state.json and
+ // committing in the same git repo at once race on the index lock. An
+ // overlapping call is a no-op success; the winner captures current state.
+ let snapshotInFlight = false;
+
+ sdk.registerFunction("mem::snapshot-create",
async (data?: { message?: string }) => {
+ if (snapshotInFlight) {
+ return { success: true, message: "Snapshot already in progress" };
+ }
+ snapshotInFlight = true;
try {
await ensureGitRepo(snapshotDir);
@@ -124,6 +134,8 @@ export function registerSnapshotFunction(
const msg = err instanceof Error ? err.message : String(err);
logger.error("Snapshot failed", { error: msg });
return { success: false, error: msg };
+ } finally {
+ snapshotInFlight = false;
}
},
);
diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts
index db149abd..c1f6cc98 100644
--- a/src/hooks/session-end.ts
+++ b/src/hooks/session-end.ts
@@ -40,22 +40,6 @@ async function main() {
signal: AbortSignal.timeout(30000),
}).catch(() => {});
- if (process.env["CONSOLIDATION_ENABLED"] === "true") {
- fetch(`${REST_URL}/agentmemory/crystals/auto`, {
- method: "POST",
- headers: authHeaders(),
- body: JSON.stringify({ olderThanDays: 0 }),
- signal: AbortSignal.timeout(60000),
- }).catch(() => {});
-
- fetch(`${REST_URL}/agentmemory/consolidate-pipeline`, {
- method: "POST",
- headers: authHeaders(),
- body: JSON.stringify({ tier: "all", force: true }),
- signal: AbortSignal.timeout(120000),
- }).catch(() => {});
- }
-
if (process.env["CLAUDE_MEMORY_BRIDGE"] === "true") {
fetch(`${REST_URL}/agentmemory/claude-bridge/sync`, {
method: "POST",
diff --git a/src/index.ts b/src/index.ts
index 1e623eae..5f66d76c 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,5 +1,6 @@
-import { registerWorker } from "iii-sdk";
+import { registerWorker, TriggerAction } from "iii-sdk";
import {
+ hydrateProcessEnvFromFile,
loadConfig,
getEnvVar,
loadEmbeddingConfig,
@@ -57,6 +58,7 @@ import { registerExportImportFunction } from "./functions/export-import.js";
import { registerEnrichFunction } from "./functions/enrich.js";
import { registerClaudeBridgeFunction } from "./functions/claude-bridge.js";
import { registerGraphFunction } from "./functions/graph.js";
+import { registerGraphImportFunction } from "./functions/graph-import.js";
import { registerConsolidationPipelineFunction } from "./functions/consolidation-pipeline.js";
import { registerTeamFunction } from "./functions/team.js";
import { registerGovernanceFunction } from "./functions/governance.js";
@@ -158,6 +160,10 @@ process.on("unhandledRejection", (reason) => {
});
async function main() {
+ // Fold ~/.agentmemory/.env into process.env before anything reads config
+ // or raw process.env. Only-if-unset, so real process.env still wins.
+ hydrateProcessEnvFromFile();
+
const config = loadConfig();
const embeddingConfig = loadEmbeddingConfig();
const fallbackConfig = loadFallbackConfig();
@@ -268,6 +274,7 @@ async function main() {
if (isGraphExtractionEnabled()) {
registerGraphFunction(sdk, kv, provider);
+ registerGraphImportFunction(sdk, kv);
bootLog(`Knowledge graph: extraction enabled`);
}
@@ -347,6 +354,21 @@ async function main() {
const snapshotConfig = loadSnapshotConfig();
if (snapshotConfig.enabled) {
registerSnapshotFunction(sdk, kv, snapshotConfig.dir);
+ // The boot line promised "every <interval>s" but nothing ever fired
+ // mem::snapshot-create. Drive it on a periodic timer (unref'd so it
+ // never keeps the process alive), mirroring the auto-forget timer.
+ // mem::snapshot-create serializes overlapping runs internally (git-lock
+ // safety), so the timer can stay a simple fire-and-forget tick.
+ const snapshotTimer = setInterval(() => {
+ sdk
+ .trigger({
+ function_id: "mem::snapshot-create",
+ payload: {},
+ action: TriggerAction.Void(),
+ })
+ .catch(() => {});
+ }, snapshotConfig.interval * 1000);
+ snapshotTimer.unref();
bootLog(
`Git snapshots: ${snapshotConfig.dir} (every ${snapshotConfig.interval}s)`,
);
@@ -518,7 +540,7 @@ async function main() {
`Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`,
);
bootLog(
- `REST API: 128 endpoints at http://localhost:${config.restPort}/agentmemory/*`,
+ `REST API: 129 endpoints at http://localhost:${config.restPort}/agentmemory/*`,
);
bootLog(
`MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`,
diff --git a/src/providers/_fetch.ts b/src/providers/_fetch.ts
index ed9b5e89..3b02d935 100644
--- a/src/providers/_fetch.ts
+++ b/src/providers/_fetch.ts
@@ -1,15 +1,52 @@
import { getEnvVar } from "../config.js";
-export function fetchWithTimeout(
+// Bounded retry for transient rate-limit / unavailable responses. Attempts is
+// total tries (initial + retries). Retries are bounded by a TOTAL elapsed
+// deadline — not per-attempt — so the worst case never blows past the caller's
+// timeout budget or the iii invocation timeout. A single retry delay is capped
+// low so a hostile Retry-After header can't dominate the budget.
+const MAX_ATTEMPTS = 3;
+const MAX_RETRY_DELAY_MS = 5000;
+// Absolute ceiling on the total budget, kept well under the iii 180s invocation
+// timeout so retries + sleeps + per-attempt timeouts can never overrun it.
+const HARD_BUDGET_CAP_MS = 170000;
+// A retry only makes sense if there's room for at least a token attempt after
+// the sleep; without this floor we'd sleep, fire, and get instantly cut off.
+const MIN_ATTEMPT_FLOOR_MS = 100;
+const RETRY_STATUS = new Set([429, 503]);
+
+const sleep = (ms: number): Promise<void> =>
+ new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Parse a Retry-After header into a delay in milliseconds. Supports both the
+ * integer-seconds form and the HTTP-date form. Returns undefined when absent
+ * or unparseable, so the caller falls back to exponential backoff. Negative
+ * or past values clamp to 0.
+ */
+function parseRetryAfter(header: string | null): number | undefined {
+ if (!header) return undefined;
+ const trimmed = header.trim();
+ if (trimmed === "") return undefined;
+
+ const seconds = Number(trimmed);
+ if (Number.isFinite(seconds)) {
+ return Math.max(0, seconds * 1000);
+ }
+
+ const date = Date.parse(trimmed);
+ if (Number.isFinite(date)) {
+ return Math.max(0, date - Date.now());
+ }
+
+ return undefined;
+}
+
+async function fetchOnce(
url: string,
init: RequestInit,
- timeoutMs?: number,
+ ms: number,
): Promise<Response> {
- const parsed =
- timeoutMs ??
- Number.parseInt(getEnvVar("AGENTMEMORY_LLM_TIMEOUT_MS") ?? "60000", 10);
- const ms = Number.isFinite(parsed) && parsed > 0 ? parsed : 60000;
-
const ctl = new AbortController();
const signal = init.signal
? AbortSignal.any([init.signal, ctl.signal])
@@ -17,3 +54,53 @@ export function fetchWithTimeout(
const t = setTimeout(() => ctl.abort(), ms);
return fetch(url, { ...init, signal }).finally(() => clearTimeout(t));
}
+
+export async function fetchWithTimeout(
+ url: string,
+ init: RequestInit,
+ timeoutMs?: number,
+): Promise<Response> {
+ const parsed =
+ timeoutMs ??
+ Number.parseInt(getEnvVar("AGENTMEMORY_LLM_TIMEOUT_MS") ?? "60000", 10);
+ const ms = Number.isFinite(parsed) && parsed > 0 ? parsed : 60000;
+
+ // The caller's timeout is the TOTAL budget for all attempts + sleeps, hard
+ // capped so we never approach the iii invocation timeout.
+ const budgetMs = Math.min(ms, HARD_BUDGET_CAP_MS);
+ const start = Date.now();
+
+ // The first attempt must honor the capped budget too — passing raw `ms`
+ // here would let a large caller timeout hang past HARD_BUDGET_CAP_MS (and
+ // the iii 180s invocation timeout) before any retry logic runs.
+ let response: Response = await fetchOnce(url, init, budgetMs);
+ for (let attempt = 1; attempt < MAX_ATTEMPTS; attempt++) {
+ if (!RETRY_STATUS.has(response.status)) return response;
+
+ const retryAfter = parseRetryAfter(response.headers.get("Retry-After"));
+ // Exponential backoff fallback when no Retry-After: 500ms, 1000ms, ...
+ const backoff = 500 * 2 ** (attempt - 1);
+ const delay = Math.min(retryAfter ?? backoff, MAX_RETRY_DELAY_MS);
+
+ // Stop retrying if the sleep plus a minimal attempt would overrun the total
+ // budget — a hostile Retry-After that alone exceeds the remaining budget
+ // returns the last response instead of stalling the caller.
+ const elapsed = Date.now() - start;
+ const remaining = budgetMs - elapsed;
+ if (delay + MIN_ATTEMPT_FLOOR_MS > remaining) return response;
+
+ // This response is being discarded for a retry; release its body so the
+ // underlying connection is returned to the pool instead of leaking.
+ await response.body?.cancel().catch(() => {});
+ await sleep(delay);
+
+ // Cap the per-attempt timeout to whatever budget is left so a late attempt
+ // can't push total elapsed past the deadline.
+ const attemptMs = Math.max(
+ MIN_ATTEMPT_FLOOR_MS,
+ Math.min(ms, budgetMs - (Date.now() - start)),
+ );
+ response = await fetchOnce(url, init, attemptMs);
+ }
+ return response;
+}
diff --git a/src/providers/embedding/_dimensions.ts b/src/providers/embedding/_dimensions.ts
new file mode 100644
index 00000000..f264e8a9
--- /dev/null
+++ b/src/providers/embedding/_dimensions.ts
@@ -0,0 +1,48 @@
+/**
+ * Shared embedding-dimension logic for OpenAI-compatible providers.
+ *
+ * OpenAI and OpenRouter expose the same underlying embedding models, so they
+ * share one dimension table and one resolver. OpenRouter namespaces model ids
+ * (e.g. "openai/text-embedding-3-small"); the lookup strips a leading
+ * "provider/" prefix so both bare and namespaced keys resolve to the same
+ * dimensions.
+ *
+ * The dimension guard (index.ts) throws on mismatch, so a wrong value here
+ * breaks every embed call — keep entries accurate. Callers pass the relevant
+ * env-var name (OPENAI_EMBEDDING_DIMENSIONS / OPENROUTER_EMBEDDING_DIMENSIONS)
+ * so error messages point at the knob the operator actually set.
+ */
+const MODEL_DIMENSIONS: Record<string, number> = {
+ "text-embedding-3-small": 1536,
+ "text-embedding-3-large": 3072,
+ "text-embedding-ada-002": 1536,
+};
+
+const DEFAULT_DIMENSIONS = 1536;
+
+function lookupModelDimensions(model: string): number | undefined {
+ if (model in MODEL_DIMENSIONS) return MODEL_DIMENSIONS[model];
+ const slash = model.indexOf("/");
+ if (slash === -1) return undefined;
+ const bare = model.slice(slash + 1);
+ return MODEL_DIMENSIONS[bare];
+}
+
+export function resolveDimensions(
+ model: string,
+ override: string | undefined,
+ envName: string,
+): number {
+ if (override !== undefined && override.trim().length > 0) {
+ const parsed = parseInt(override, 10);
+ if (!Number.isFinite(parsed) || parsed <= 0) {
+ throw new Error(
+ `${envName} must be a positive integer, got: ${override}`,
+ );
+ }
+ return parsed;
+ }
+ return lookupModelDimensions(model) ?? DEFAULT_DIMENSIONS;
+}
+
+export { MODEL_DIMENSIONS, DEFAULT_DIMENSIONS };
diff --git a/src/providers/embedding/openai.ts b/src/providers/embedding/openai.ts
index 7384d513..50344626 100644
--- a/src/providers/embedding/openai.ts
+++ b/src/providers/embedding/openai.ts
@@ -8,35 +8,10 @@ import {
detectAzure,
normalizeBaseUrl,
} from "../_openai-shared.js";
+import { resolveDimensions } from "./_dimensions.js";
const DEFAULT_MODEL = "text-embedding-3-small";
-/**
- * Known OpenAI embedding model dimensions. Extend as new models ship.
- * Override in any case via OPENAI_EMBEDDING_DIMENSIONS for custom or
- * self-hosted OpenAI-compatible endpoints returning non-standard sizes.
- */
-const MODEL_DIMENSIONS: Record<string, number> = {
- "text-embedding-3-small": 1536,
- "text-embedding-3-large": 3072,
- "text-embedding-ada-002": 1536,
-};
-
-const DEFAULT_DIMENSIONS = MODEL_DIMENSIONS[DEFAULT_MODEL] ?? 1536;
-
-function resolveDimensions(model: string, override: string | undefined): number {
- if (override !== undefined && override.trim().length > 0) {
- const parsed = parseInt(override, 10);
- if (!Number.isFinite(parsed) || parsed <= 0) {
- throw new Error(
- `OPENAI_EMBEDDING_DIMENSIONS must be a positive integer, got: ${override}`,
- );
- }
- return parsed;
- }
- return MODEL_DIMENSIONS[model] ?? DEFAULT_DIMENSIONS;
-}
-
/**
* OpenAI-compatible embedding provider.
*
@@ -71,7 +46,7 @@ function resolveDimensions(model: string, override: string | undefined): number
* OPENAI_EMBEDDING_MODEL — model name (default: text-embedding-3-small)
* OPENAI_EMBEDDING_DIMENSIONS — override reported dimensions (required for
* custom / self-hosted models not in the
- * MODEL_DIMENSIONS table above)
+ * shared MODEL_DIMENSIONS table)
*/
export class OpenAIEmbeddingProvider implements EmbeddingProvider {
readonly name = "openai";
@@ -107,6 +82,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
this.dimensions = resolveDimensions(
this.model,
getEnvVar("OPENAI_EMBEDDING_DIMENSIONS"),
+ "OPENAI_EMBEDDING_DIMENSIONS",
);
this.isAzure = detectAzure(this.baseUrl);
this.azureApiVersion =
diff --git a/src/providers/embedding/openrouter.ts b/src/providers/embedding/openrouter.ts
index 46999e55..c4aa0bcf 100644
--- a/src/providers/embedding/openrouter.ts
+++ b/src/providers/embedding/openrouter.ts
@@ -1,21 +1,27 @@
import type { EmbeddingProvider } from "../../types.js";
import { getEnvVar } from "../../config.js";
import { fetchWithTimeout } from "../_fetch.js";
+import { resolveDimensions } from "./_dimensions.js";
const API_URL = "https://openrouter.ai/api/v1/embeddings";
+const DEFAULT_MODEL = "openai/text-embedding-3-small";
+
export class OpenRouterEmbeddingProvider implements EmbeddingProvider {
readonly name = "openrouter";
- readonly dimensions = 1536;
+ readonly dimensions: number;
private apiKey: string;
private model: string;
constructor(apiKey?: string) {
this.apiKey = apiKey || getEnvVar("OPENROUTER_API_KEY") || "";
if (!this.apiKey) throw new Error("OPENROUTER_API_KEY is required");
- this.model =
- getEnvVar("OPENROUTER_EMBEDDING_MODEL") ||
- "openai/text-embedding-3-small";
+ this.model = getEnvVar("OPENROUTER_EMBEDDING_MODEL") || DEFAULT_MODEL;
+ this.dimensions = resolveDimensions(
+ this.model,
+ getEnvVar("OPENROUTER_EMBEDDING_DIMENSIONS"),
+ "OPENROUTER_EMBEDDING_DIMENSIONS",
+ );
}
async embed(text: string): Promise<Float32Array> {
diff --git a/src/state/schema.ts b/src/state/schema.ts
index cb29d41a..773f147c 100644
--- a/src/state/schema.ts
+++ b/src/state/schema.ts
@@ -1,4 +1,5 @@
import { createHash } from "node:crypto";
+import { hasCjk, segmentCjk } from "./cjk-segmenter.js";
export const KV = {
sessions: "mem:sessions",
@@ -91,11 +92,60 @@ export function fingerprintId(prefix: string, content: string): string {
return `${prefix}_${hash.slice(0, 16)}`;
}
+// CJK/Japanese/Thai text carries no inter-word whitespace, so a plain
+// split(/\s+/) collapses a whole sentence into one token and the two token
+// sets never overlap — dedup silently stops working. When either input
+// contains CJK we segment the CJK runs (word tokens via the shared
+// segmenter, plus character bigram shingles so near-identical strings still
+// overlap even when the optional segmenter deps fall back to whole-string).
+// ASCII text keeps the original word-level behavior.
+function jaccardTokens(text: string): Set<string> {
+ if (!hasCjk(text)) {
+ return new Set(text.split(/\s+/).filter((t) => t.length > 2));
+ }
+ const tokens = new Set<string>();
+ for (const raw of text.split(/\s+/)) {
+ if (!raw) continue;
+ if (hasCjk(raw)) {
+ for (const seg of segmentCjk(raw)) {
+ if (seg) tokens.add(seg);
+ }
+ // Character bigram shingles keep the CJK signal even when the
+ // segmenter returns the whole run: overlapping bigrams make
+ // near-identical strings match while unrelated ones ("北京" vs
+ // "上海") share none. Never drop short CJK tokens.
+ const chars = Array.from(raw);
+ if (chars.length === 1) {
+ tokens.add(chars[0]);
+ } else {
+ for (let i = 0; i < chars.length - 1; i++) {
+ tokens.add(chars[i] + chars[i + 1]);
+ }
+ }
+ } else if (raw.length > 2) {
+ tokens.add(raw);
+ }
+ }
+ return tokens;
+}
+
export function jaccardSimilarity(a: string, b: string): number {
- const setA = new Set(a.split(/\s+/).filter((t) => t.length > 2));
- const setB = new Set(b.split(/\s+/).filter((t) => t.length > 2));
- if (setA.size === 0 && setB.size === 0) return 1;
- if (setA.size === 0 || setB.size === 0) return 0;
+ const na = a.normalize("NFC");
+ const nb = b.normalize("NFC");
+ const setA = jaccardTokens(na);
+ const setB = jaccardTokens(nb);
+ // An empty token set carries no signal for the overlap metric. Very short
+ // ASCII text ("AI", "a b", "go") tokenizes to nothing because words <=2
+ // chars are dropped. Fall back to exact normalized equality so re-saving
+ // the identical short memory still supersedes, while unrelated short
+ // strings ("AI" vs "ML") correctly score 0. Returning 1 unconditionally
+ // here (as an "both empty" shortcut once did) would let any short memory
+ // falsely supersede another, silently marking a real memory not-latest.
+ if (setA.size === 0 || setB.size === 0) {
+ return na.trim().replace(/\s+/g, " ") === nb.trim().replace(/\s+/g, " ")
+ ? 1
+ : 0;
+ }
let intersection = 0;
for (const word of setA) {
if (setB.has(word)) intersection++;
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 99fb9908..ce339ae0 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -851,11 +851,21 @@ export function registerApiTriggers(
const filtered = filterAgentId
? sessions.filter((s) => s.agentId === filterAgentId)
: sessions;
- const summaries = await Promise.all(
- filtered.map((s) =>
- kv.get<SessionSummary>(KV.summaries, s.id).catch(() => null),
- ),
- );
+ // Bounded fan-out: each kv.get is a full engine invocation, so
+ // Promise.all over hundreds of sessions saturates the invocation
+ // pool. Batch in chunks of 10 (parallel within a chunk, sequential
+ // across chunks); the summaries array stays index-aligned with
+ // `filtered`.
+ const summaries: Array<SessionSummary | null> = [];
+ for (let batch = 0; batch < filtered.length; batch += 10) {
+ const chunk = filtered.slice(batch, batch + 10);
+ const results = await Promise.all(
+ chunk.map((s) =>
+ kv.get<SessionSummary>(KV.summaries, s.id).catch(() => null),
+ ),
+ );
+ summaries.push(...results);
+ }
const withSummary = filtered.map((s, i) =>
summaries[i] ? { ...s, summary: summaries[i] } : s,
);
@@ -1641,6 +1651,43 @@ export function registerApiTriggers(
config: { api_path: "/agentmemory/graph/build", http_method: "POST" },
});
+ // Import graphify's structural graph (graphify-out/graph.json) into the
+ // memory graph. Deterministic, no LLM call; idempotent via the graph
+ // name-index upsert.
+ sdk.registerFunction("api::graph-import-graphify",
+ async (req: ApiRequest<{ path?: string; cwd?: string }>): Promise<Response> => {
+ const authErr = checkAuth(req, secret);
+ if (authErr) return authErr;
+ const { path, cwd } = req.body ?? {};
+ if (
+ (path !== undefined && typeof path !== "string") ||
+ (cwd !== undefined && typeof cwd !== "string")
+ ) {
+ return {
+ status_code: 400,
+ body: { error: "path and cwd must be strings when provided" },
+ };
+ }
+ try {
+ const result = await sdk.trigger({
+ function_id: "mem::graph::import-graphify",
+ payload: {
+ ...(path !== undefined ? { path } : {}),
+ ...(cwd !== undefined ? { cwd } : {}),
+ },
+ });
+ return { status_code: 200, body: result };
+ } catch {
+ return graphDisabledResponse();
+ }
+ },
+ );
+ sdk.registerTrigger({
+ type: "http",
+ function_id: "api::graph-import-graphify",
+ config: { api_path: "/agentmemory/graph/import-graphify", http_method: "POST" },
+ });
+
sdk.registerFunction("api::consolidate-pipeline",
async (req: ApiRequest<{ tier?: string }>): Promise<Response> => {
const authErr = checkAuth(req, secret);
diff --git a/src/triggers/events.ts b/src/triggers/events.ts
index e75d615b..65db7035 100644
--- a/src/triggers/events.ts
+++ b/src/triggers/events.ts
@@ -3,9 +3,44 @@ import type { CompressedObservation, HookPayload, Session } from "../types.js";
import { KV, STREAM } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { isReflectEnabled } from "../functions/slots.js";
-import { getAgentId, isGraphExtractionEnabled } from "../config.js";
+import {
+ getAgentId,
+ getConsolidationCooldownMs,
+ isConsolidationEnabled,
+ isGraphExtractionEnabled,
+} from "../config.js";
import { logger } from "../logger.js";
+// Global marker recording when corpus consolidation last ran, used to debounce
+// the per-turn session-stop fan-out.
+const CONSOLIDATION_MARKER_KEY = "consolidation:lastRun";
+
+async function consolidationDueUnserialized(kv: StateKV): Promise<boolean> {
+ const cooldownMs = getConsolidationCooldownMs();
+ if (cooldownMs <= 0) return true; // debounce disabled
+ const now = Date.now();
+ const marker = await kv
+ .get<{ at?: number }>(KV.config, CONSOLIDATION_MARKER_KEY)
+ .catch(() => null);
+ const lastAt = typeof marker?.at === "number" ? marker.at : 0;
+ if (now - lastAt < cooldownMs) return false;
+ await kv.set(KV.config, CONSOLIDATION_MARKER_KEY, { at: now }).catch(() => {});
+ return true;
+}
+
+// Concurrent session-stop events would otherwise interleave the marker
+// read-check-write above and both pass the cooldown. Serialize the whole
+// check through an in-process chain so exactly one concurrent caller wins.
+let consolidationCheckChain: Promise<unknown> = Promise.resolve();
+
+function consolidationDue(kv: StateKV): Promise<boolean> {
+ const result = consolidationCheckChain.then(() =>
+ consolidationDueUnserialized(kv),
+ );
+ consolidationCheckChain = result.catch(() => false);
+ return result;
+}
+
export function registerEventTriggers(sdk: ISdk, kv: StateKV): void {
sdk.registerFunction(
"event::session::started",
@@ -59,21 +94,19 @@ export function registerEventTriggers(sdk: ISdk, kv: StateKV): void {
config: { topic: "agentmemory.observation" },
});
- sdk.registerFunction("event::session::stopped", async (data: { sessionId: string }) => {
+ sdk.registerFunction("event::session::stopped", async (data: { sessionId: string; skipConsolidation?: boolean }) => {
const summary = await sdk.trigger({ function_id: "mem::summarize", payload: data });
+ const fireVoid = (function_id: string, payload: unknown) =>
+ sdk
+ .trigger({ function_id, payload, action: TriggerAction.Void() })
+ .catch((err) =>
+ logger.warn(function_id + " trigger failed", {
+ sessionId: data.sessionId,
+ error: err instanceof Error ? err.message : String(err),
+ }),
+ );
if (isReflectEnabled()) {
- try {
- sdk.trigger({
- function_id: "mem::slot-reflect",
- payload: { sessionId: data.sessionId },
- action: TriggerAction.Void(),
- });
- } catch (err) {
- logger.warn("slot-reflect trigger failed", {
- sessionId: data.sessionId,
- error: err instanceof Error ? err.message : String(err),
- });
- }
+ fireVoid("mem::slot-reflect", { sessionId: data.sessionId });
}
if (isGraphExtractionEnabled()) {
try {
@@ -95,6 +128,28 @@ export function registerEventTriggers(sdk: ISdk, kv: StateKV): void {
});
}
}
+ // Crystals + lessons consolidation. The stop lifecycle is the single
+ // source of truth: event::session::stopped fires for ALL agents (the
+ // client-side session-end hook no longer drives consolidation directly).
+ // Gated so keyless/zero-LLM users don't fire no-op LLM calls.
+ //
+ // skipConsolidation suppresses the fan-out when this handler is driven
+ // by eviction's stale-session recovery: evict calls session::stopped
+ // once per recovered session, then runs ONE final consolidation pass.
+ // Without this guard, N recovered sessions launch N concurrent forced
+ // full-corpus consolidations plus N crystallizations.
+ //
+ // Debounce: /session/end is posted by the per-turn Stop hook, so this
+ // handler fires on every agent turn. consolidate-pipeline + auto-crystallize
+ // are full-corpus LLM work with no internal "nothing changed" guard, so
+ // firing them every turn is a cost/latency storm for connected agents.
+ // Bound the global corpus consolidation to once per cooldown window.
+ if (isConsolidationEnabled() && !data.skipConsolidation) {
+ if (await consolidationDue(kv)) {
+ fireVoid("mem::consolidate-pipeline", { tier: "all", force: true });
+ fireVoid("mem::auto-crystallize", { olderThanDays: 0 });
+ }
+ }
return summary;
});
sdk.registerTrigger({
diff --git a/src/viewer/server.ts b/src/viewer/server.ts
index c80e67f0..3f983ba2 100644
--- a/src/viewer/server.ts
+++ b/src/viewer/server.ts
@@ -158,7 +158,7 @@ function json(
function readBody(req: IncomingMessage): Promise<string> {
return new Promise((resolve, reject) => {
- let data = "";
+ const chunks: Buffer[] = [];
let size = 0;
req.on("data", (chunk: Buffer) => {
size += chunk.length;
@@ -167,9 +167,9 @@ function readBody(req: IncomingMessage): Promise<string> {
reject(new Error("too large"));
return;
}
- data += chunk.toString();
+ chunks.push(chunk);
});
- req.on("end", () => resolve(data));
+ req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
req.on("error", reject);
});
}
diff --git a/test/connect-guidelines.test.ts b/test/connect-guidelines.test.ts
new file mode 100644
index 00000000..91727236
--- /dev/null
+++ b/test/connect-guidelines.test.ts
@@ -0,0 +1,156 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { writeGuideline, guidelineTargets } from "../src/cli/connect/guidelines.js";
+
+let home: string;
+let cwd: string;
+
+beforeEach(() => {
+ home = mkdtempSync(join(tmpdir(), "am-guide-home-"));
+ cwd = mkdtempSync(join(tmpdir(), "am-guide-cwd-"));
+});
+
+afterEach(() => {
+ rmSync(home, { recursive: true, force: true });
+ rmSync(cwd, { recursive: true, force: true });
+});
+
+describe("writeGuideline", () => {
+ it("writes a Cursor .mdc project rule with alwaysApply frontmatter", () => {
+ const r = writeGuideline("cursor", { cwd, home });
+ expect(r.kind).toBe("written");
+ const path = join(cwd, ".cursor", "rules", "agentmemory.mdc");
+ expect(existsSync(path)).toBe(true);
+ const body = readFileSync(path, "utf8");
+ expect(body).toContain("alwaysApply: true");
+ expect(body).toContain("memory_recall");
+ expect(body).toContain("memory_save");
+ });
+
+ it("writes a Kiro steering file with inclusion: always (global)", () => {
+ const r = writeGuideline("kiro", { cwd, home });
+ expect(r.kind).toBe("written");
+ if (r.kind === "written") expect(r.scope).toBe("global");
+ const path = join(home, ".kiro", "steering", "agentmemory.md");
+ expect(existsSync(path)).toBe(true);
+ expect(readFileSync(path, "utf8")).toContain("inclusion: always");
+ });
+
+ it("writes a marked block into the agent's global AGENTS.md (Zed)", () => {
+ const r = writeGuideline("zed", { cwd, home });
+ expect(r.kind).toBe("written");
+ const path = join(home, ".config", "zed", "AGENTS.md");
+ const body = readFileSync(path, "utf8");
+ expect(body).toContain("<!-- agentmemory:start -->");
+ expect(body).toContain("<!-- agentmemory:end -->");
+ });
+
+ it("prefers the global path when the target defines one (Droid)", () => {
+ const r = writeGuideline("droid", { cwd, home });
+ expect(r.kind).toBe("written");
+ expect(existsSync(join(home, ".factory", "AGENTS.md"))).toBe(true);
+ // must NOT have written into the project cwd
+ expect(existsSync(join(cwd, "AGENTS.md"))).toBe(false);
+ });
+
+ it("falls back to a project path when the agent has no global rules file (Warp)", () => {
+ const r = writeGuideline("warp", { cwd, home });
+ expect(r.kind).toBe("written");
+ if (r.kind === "written") expect(r.scope).toBe("project");
+ expect(existsSync(join(cwd, "AGENTS.md"))).toBe(true);
+ });
+
+ it("preserves surrounding user content and updates only its own block", () => {
+ const path = join(home, ".factory", "AGENTS.md");
+ mkdirSync(join(home, ".factory"), { recursive: true });
+ writeFileSync(path, "# My rules\n\nUse tabs, not spaces.\n", "utf8");
+
+ writeGuideline("droid", { cwd, home });
+ const first = readFileSync(path, "utf8");
+ expect(first).toContain("# My rules");
+ expect(first).toContain("Use tabs, not spaces.");
+ expect(first).toContain("<!-- agentmemory:start -->");
+
+ // Re-running is idempotent (no duplicate block, reports unchanged).
+ const again = writeGuideline("droid", { cwd, home });
+ expect(again.kind).toBe("unchanged");
+ const second = readFileSync(path, "utf8");
+ expect(second).toBe(first);
+ expect(second.match(/agentmemory:start/g)?.length).toBe(1);
+ });
+
+ it("refuses to touch a file with a lone or reversed marker", () => {
+ const path = join(home, ".factory", "AGENTS.md");
+ mkdirSync(join(home, ".factory"), { recursive: true });
+ // Lone START marker: appending would let a later run pair this orphan
+ // with the appended block's END and cut the user's content in between.
+ const lone = "# Rules\n<!-- agentmemory:start -->\nuser notes here\n";
+ writeFileSync(path, lone, "utf8");
+ const r = writeGuideline("droid", { cwd, home });
+ expect(r.kind).toBe("unchanged");
+ expect(readFileSync(path, "utf8")).toBe(lone);
+
+ // Reversed pair: same refusal.
+ const reversed =
+ "<!-- agentmemory:end -->\nmiddle\n<!-- agentmemory:start -->\n";
+ writeFileSync(path, reversed, "utf8");
+ const r2 = writeGuideline("droid", { cwd, home });
+ expect(r2.kind).toBe("unchanged");
+ expect(readFileSync(path, "utf8")).toBe(reversed);
+ });
+
+ it("is idempotent for dedicated files (second run unchanged)", () => {
+ expect(writeGuideline("cursor", { cwd, home }).kind).toBe("written");
+ expect(writeGuideline("cursor", { cwd, home }).kind).toBe("unchanged");
+ });
+
+ it("dry-run reports would-write without creating the file", () => {
+ const r = writeGuideline("kiro", { cwd, home, dryRun: true });
+ expect(r.kind).toBe("would-write");
+ expect(existsSync(join(home, ".kiro", "steering", "agentmemory.md"))).toBe(false);
+ });
+
+ it("returns no-target for agents that already auto-capture (claude-code)", () => {
+ expect(writeGuideline("claude-code", { cwd, home }).kind).toBe("no-target");
+ expect(writeGuideline("codex", { cwd, home }).kind).toBe("no-target");
+ });
+
+ it("Gemini and Antigravity share ~/.gemini/GEMINI.md idempotently", () => {
+ writeGuideline("gemini-cli", { cwd, home });
+ const r2 = writeGuideline("antigravity", { cwd, home });
+ // Antigravity targets the same GEMINI.md; the block already exists.
+ expect(r2.kind).toBe("unchanged");
+ const body = readFileSync(join(home, ".gemini", "GEMINI.md"), "utf8");
+ expect(body.match(/agentmemory:start/g)?.length).toBe(1);
+ });
+});
+
+describe("guidelineTargets coverage", () => {
+ it("covers every MCP-only / partial agent and no others", () => {
+ const names = Object.keys(guidelineTargets("/home/x")).sort();
+ expect(names).toEqual(
+ [
+ "antigravity",
+ "cline",
+ "continue",
+ "copilot-cli",
+ "cursor",
+ "droid",
+ "gemini-cli",
+ "kiro",
+ "opencode",
+ "qwen",
+ "warp",
+ "zed",
+ ].sort(),
+ );
+ });
+
+ it("every target cites an official source URL", () => {
+ for (const t of Object.values(guidelineTargets("/home/x"))) {
+ expect(t.source).toMatch(/^https:\/\//);
+ }
+ });
+});
diff --git a/test/embedding-dimensions.test.ts b/test/embedding-dimensions.test.ts
new file mode 100644
index 00000000..51b3b084
--- /dev/null
+++ b/test/embedding-dimensions.test.ts
@@ -0,0 +1,101 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { resolveDimensions } from "../src/providers/embedding/_dimensions.js";
+import { OpenRouterEmbeddingProvider } from "../src/providers/embedding/openrouter.js";
+import { OpenAIEmbeddingProvider } from "../src/providers/embedding/openai.js";
+
+describe("resolveDimensions", () => {
+ const ENV = "OPENROUTER_EMBEDDING_DIMENSIONS";
+
+ it("resolves namespaced OpenRouter model ids to their real dimensions", () => {
+ expect(resolveDimensions("openai/text-embedding-3-large", undefined, ENV)).toBe(3072);
+ expect(resolveDimensions("openai/text-embedding-3-small", undefined, ENV)).toBe(1536);
+ expect(resolveDimensions("openai/text-embedding-ada-002", undefined, ENV)).toBe(1536);
+ });
+
+ it("resolves bare model ids to their real dimensions", () => {
+ expect(resolveDimensions("text-embedding-3-large", undefined, ENV)).toBe(3072);
+ expect(resolveDimensions("text-embedding-3-small", undefined, ENV)).toBe(1536);
+ expect(resolveDimensions("text-embedding-ada-002", undefined, ENV)).toBe(1536);
+ });
+
+ it("lets a valid override win over the model-derived dimensions", () => {
+ expect(resolveDimensions("openai/text-embedding-3-large", "1024", ENV)).toBe(1024);
+ expect(resolveDimensions("text-embedding-3-small", "768", ENV)).toBe(768);
+ });
+
+ it("throws with the given env name on invalid override values", () => {
+ for (const bad of ["abc", "0", "-5"]) {
+ expect(() => resolveDimensions("text-embedding-3-large", bad, ENV)).toThrow(
+ new RegExp(`${ENV} must be a positive integer, got: ${bad}`),
+ );
+ }
+ });
+
+ it("uses the supplied env name in the error message", () => {
+ expect(() => resolveDimensions("text-embedding-3-large", "abc", "OPENAI_EMBEDDING_DIMENSIONS")).toThrow(
+ /OPENAI_EMBEDDING_DIMENSIONS must be a positive integer, got: abc/,
+ );
+ });
+
+ it("falls back to the default (1536) for unknown models", () => {
+ expect(resolveDimensions("mystery-self-hosted-model", undefined, ENV)).toBe(1536);
+ expect(resolveDimensions("someprovider/unknown-model", undefined, ENV)).toBe(1536);
+ });
+});
+
+describe("OpenRouterEmbeddingProvider dimension regression", () => {
+ const originalEnv = { ...process.env };
+
+ beforeEach(() => {
+ process.env = { ...originalEnv };
+ delete process.env["OPENROUTER_EMBEDDING_MODEL"];
+ delete process.env["OPENROUTER_EMBEDDING_DIMENSIONS"];
+ });
+
+ afterEach(() => {
+ process.env = originalEnv;
+ });
+
+ it("reports 3072 for openai/text-embedding-3-large with no override (guard would throw on the old hardcoded 1536)", () => {
+ process.env["OPENROUTER_EMBEDDING_MODEL"] = "openai/text-embedding-3-large";
+ const provider = new OpenRouterEmbeddingProvider("test-key");
+ expect(provider.dimensions).toBe(3072);
+ });
+
+ it("defaults to 1536 for openai/text-embedding-3-small", () => {
+ const provider = new OpenRouterEmbeddingProvider("test-key");
+ expect(provider.dimensions).toBe(1536);
+ });
+
+ it("lets OPENROUTER_EMBEDDING_DIMENSIONS override the model-derived dimensions", () => {
+ process.env["OPENROUTER_EMBEDDING_MODEL"] = "openai/text-embedding-3-large";
+ process.env["OPENROUTER_EMBEDDING_DIMENSIONS"] = "1024";
+ const provider = new OpenRouterEmbeddingProvider("test-key");
+ expect(provider.dimensions).toBe(1024);
+ });
+});
+
+describe("OpenAIEmbeddingProvider defaults unchanged", () => {
+ const originalEnv = { ...process.env };
+
+ beforeEach(() => {
+ process.env = { ...originalEnv };
+ delete process.env["OPENAI_EMBEDDING_MODEL"];
+ delete process.env["OPENAI_EMBEDDING_DIMENSIONS"];
+ });
+
+ afterEach(() => {
+ process.env = originalEnv;
+ });
+
+ it("defaults to 1536 for text-embedding-3-small", () => {
+ const provider = new OpenAIEmbeddingProvider("test-key");
+ expect(provider.dimensions).toBe(1536);
+ });
+
+ it("reports 3072 for text-embedding-3-large", () => {
+ process.env["OPENAI_EMBEDDING_MODEL"] = "text-embedding-3-large";
+ const provider = new OpenAIEmbeddingProvider("test-key");
+ expect(provider.dimensions).toBe(3072);
+ });
+});
diff --git a/test/env-loader.test.ts b/test/env-loader.test.ts
index 17ff6a8e..4c31beba 100644
--- a/test/env-loader.test.ts
+++ b/test/env-loader.test.ts
@@ -3,6 +3,19 @@ import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
+const readFileSyncCalls: unknown[][] = [];
+
+vi.mock("node:fs", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("node:fs")>();
+ return {
+ ...actual,
+ readFileSync: (...args: unknown[]) => {
+ readFileSyncCalls.push(args);
+ return (actual.readFileSync as (...a: unknown[]) => unknown)(...args);
+ },
+ };
+});
+
const ORIGINAL_HOME = process.env["HOME"];
const ORIGINAL_USERPROFILE = process.env["USERPROFILE"];
@@ -90,3 +103,106 @@ describe("loadEnvFile", () => {
expect(cfg.isDropStaleIndexEnabled()).toBe(true);
});
});
+
+describe("hydrateProcessEnvFromFile", () => {
+ const TOUCHED = ["HYDRATE_ONLY", "HYDRATE_WINS"];
+
+ beforeEach(() => {
+ sandboxHome = mkdtempSync(join(tmpdir(), "agentmemory-hydrate-"));
+ process.env["HOME"] = sandboxHome;
+ process.env["USERPROFILE"] = sandboxHome;
+ for (const k of TOUCHED) delete process.env[k];
+ });
+
+ afterEach(() => {
+ if (ORIGINAL_HOME === undefined) delete process.env["HOME"];
+ else process.env["HOME"] = ORIGINAL_HOME;
+ if (ORIGINAL_USERPROFILE === undefined) delete process.env["USERPROFILE"];
+ else process.env["USERPROFILE"] = ORIGINAL_USERPROFILE;
+ for (const k of TOUCHED) delete process.env[k];
+ rmSync(sandboxHome, { recursive: true, force: true });
+ });
+
+ it("copies .env-only keys into process.env", async () => {
+ writeEnv("HYDRATE_ONLY=from-file");
+ const cfg = await freshConfig();
+ expect(process.env["HYDRATE_ONLY"]).toBeUndefined();
+ cfg.hydrateProcessEnvFromFile();
+ expect(process.env["HYDRATE_ONLY"]).toBe("from-file");
+ });
+
+ it("does not overwrite a value already set in process.env", async () => {
+ writeEnv("HYDRATE_WINS=from-file");
+ process.env["HYDRATE_WINS"] = "from-process";
+ const cfg = await freshConfig();
+ cfg.hydrateProcessEnvFromFile();
+ expect(process.env["HYDRATE_WINS"]).toBe("from-process");
+ });
+
+ it("exposes a .env-only key via getEnvVar and, after hydrate, via raw process.env", async () => {
+ writeEnv("HYDRATE_ONLY=from-file");
+ const cfg = await freshConfig();
+ expect(cfg.getEnvVar("HYDRATE_ONLY")).toBe("from-file");
+ expect(process.env["HYDRATE_ONLY"]).toBeUndefined();
+ cfg.hydrateProcessEnvFromFile();
+ expect(process.env["HYDRATE_ONLY"]).toBe("from-file");
+ });
+});
+
+describe("loadEnvFile cache", () => {
+ beforeEach(() => {
+ sandboxHome = mkdtempSync(join(tmpdir(), "agentmemory-cache-"));
+ process.env["HOME"] = sandboxHome;
+ process.env["USERPROFILE"] = sandboxHome;
+ delete process.env["CACHED_VAR"];
+ readFileSyncCalls.length = 0;
+ });
+
+ afterEach(() => {
+ if (ORIGINAL_HOME === undefined) delete process.env["HOME"];
+ else process.env["HOME"] = ORIGINAL_HOME;
+ if (ORIGINAL_USERPROFILE === undefined) delete process.env["USERPROFILE"];
+ else process.env["USERPROFILE"] = ORIGINAL_USERPROFILE;
+ delete process.env["CACHED_VAR"];
+ rmSync(sandboxHome, { recursive: true, force: true });
+ });
+
+ it("reads the .env file from disk only once across many getMergedEnv/getEnvVar calls", async () => {
+ writeEnv("CACHED_VAR=cached");
+ const cfg = await freshConfig();
+ const envPath = join(sandboxHome, ".agentmemory", ".env");
+ readFileSyncCalls.length = 0;
+
+ for (let i = 0; i < 25; i++) {
+ cfg.getEnvVar("CACHED_VAR");
+ cfg.loadEmbeddingConfig();
+ cfg.isConsolidationEnabled();
+ }
+
+ const envReads = readFileSyncCalls.filter(([p]) => p === envPath);
+ expect(envReads).toHaveLength(1);
+ expect(cfg.getEnvVar("CACHED_VAR")).toBe("cached");
+ });
+
+ it("sees updated .env content after vi.resetModules reloads the module", async () => {
+ writeEnv("CACHED_VAR=first");
+ const first = await freshConfig();
+ expect(first.getEnvVar("CACHED_VAR")).toBe("first");
+
+ writeEnv("CACHED_VAR=second");
+ const second = await freshConfig();
+ expect(second.getEnvVar("CACHED_VAR")).toBe("second");
+ });
+
+ it("re-reads disk after __resetEnvFileCache within the same module instance", async () => {
+ writeEnv("CACHED_VAR=first");
+ const cfg = await freshConfig();
+ expect(cfg.getEnvVar("CACHED_VAR")).toBe("first");
+
+ writeEnv("CACHED_VAR=second");
+ expect(cfg.getEnvVar("CACHED_VAR")).toBe("first");
+
+ cfg.__resetEnvFileCache();
+ expect(cfg.getEnvVar("CACHED_VAR")).toBe("second");
+ });
+});
diff --git a/test/events-consolidation.test.ts b/test/events-consolidation.test.ts
new file mode 100644
index 00000000..038566a8
--- /dev/null
+++ b/test/events-consolidation.test.ts
@@ -0,0 +1,324 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { readFileSync } from "node:fs";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+vi.mock("../src/config.js", () => ({
+ getAgentId: vi.fn(() => undefined),
+ isConsolidationEnabled: vi.fn(() => true),
+ isGraphExtractionEnabled: vi.fn(() => false),
+ getConsolidationCooldownMs: vi.fn(() => 300000),
+}));
+
+vi.mock("../src/functions/slots.js", () => ({
+ isReflectEnabled: vi.fn(() => false),
+}));
+
+import { registerEventTriggers } from "../src/triggers/events.js";
+import {
+ isConsolidationEnabled,
+ isGraphExtractionEnabled,
+ getConsolidationCooldownMs,
+} from "../src/config.js";
+import { isReflectEnabled } from "../src/functions/slots.js";
+import { logger } from "../src/logger.js";
+
+// event::session::stopped is the single source of truth for consolidation.
+// It fans out mem::summarize (awaited) plus fire-and-forget void triggers for
+// slot-reflect / consolidate-pipeline / auto-crystallize, each gated by config.
+// The client session-end hook no longer POSTs crystals/auto or
+// consolidate-pipeline, so these no longer double-fire for Claude Code.
+
+function mockKV() {
+ return {
+ get: vi.fn(async () => null),
+ set: vi.fn(async (_scope: string, _key: string, data: unknown) => data),
+ delete: vi.fn(async () => {}),
+ update: vi.fn(async () => {}),
+ list: vi.fn(async () => []),
+ };
+}
+
+type StoppedHandler = (data: {
+ sessionId: string;
+ skipConsolidation?: boolean;
+}) => Promise<unknown>;
+
+// Builds a spy-backed sdk. `trigger` resolves for mem::summarize with a fake
+// summary; void triggers resolve unless `rejectFor` matches the function_id,
+// in which case they reject (to exercise fireVoid's .catch()).
+function mockSdk(opts?: { rejectFor?: string }) {
+ const handlers = new Map<string, StoppedHandler>();
+ const trigger = vi.fn(
+ async (input: { function_id: string; payload?: unknown; action?: unknown }) => {
+ if (opts?.rejectFor && input.function_id === opts.rejectFor) {
+ throw new Error(`boom: ${input.function_id}`);
+ }
+ if (input.function_id === "mem::summarize") {
+ return { summary: "session summary", sessionId: "ses_1" };
+ }
+ return { ok: true };
+ },
+ );
+ return {
+ sdk: {
+ registerFunction: (id: string, handler: StoppedHandler) => handlers.set(id, handler),
+ registerTrigger: () => {},
+ trigger,
+ },
+ handlers,
+ trigger,
+ };
+}
+
+function functionIds(trigger: ReturnType<typeof vi.fn>): string[] {
+ return trigger.mock.calls.map((c) => (c[0] as { function_id: string }).function_id);
+}
+
+describe("event::session::stopped consolidation fan-out", () => {
+ beforeEach(() => {
+ vi.mocked(isConsolidationEnabled).mockReturnValue(true);
+ vi.mocked(isGraphExtractionEnabled).mockReturnValue(false);
+ vi.mocked(isReflectEnabled).mockReturnValue(false);
+ vi.mocked(logger.warn).mockClear();
+ });
+
+ it("fires consolidate-pipeline and auto-crystallize when consolidation enabled", async () => {
+ vi.mocked(isConsolidationEnabled).mockReturnValue(true);
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, mockKV() as never);
+
+ const stopped = handlers.get("event::session::stopped")!;
+ await stopped({ sessionId: "ses_1" });
+
+ const ids = functionIds(trigger);
+ expect(ids).toContain("mem::summarize");
+ expect(ids).toContain("mem::consolidate-pipeline");
+ expect(ids).toContain("mem::auto-crystallize");
+
+ const consolidateCall = trigger.mock.calls.find(
+ (c) => (c[0] as { function_id: string }).function_id === "mem::consolidate-pipeline",
+ );
+ expect((consolidateCall![0] as { payload: unknown }).payload).toEqual({
+ tier: "all",
+ force: true,
+ });
+ const crystallizeCall = trigger.mock.calls.find(
+ (c) => (c[0] as { function_id: string }).function_id === "mem::auto-crystallize",
+ );
+ expect((crystallizeCall![0] as { payload: unknown }).payload).toEqual({
+ olderThanDays: 0,
+ });
+ });
+
+ it("skips consolidate-pipeline and auto-crystallize when consolidation disabled but still summarizes", async () => {
+ vi.mocked(isConsolidationEnabled).mockReturnValue(false);
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, mockKV() as never);
+
+ const stopped = handlers.get("event::session::stopped")!;
+ await stopped({ sessionId: "ses_1" });
+
+ const ids = functionIds(trigger);
+ expect(ids).toContain("mem::summarize");
+ expect(ids).not.toContain("mem::consolidate-pipeline");
+ expect(ids).not.toContain("mem::auto-crystallize");
+ });
+
+ it("suppresses the consolidation fan-out when skipConsolidation is set (eviction recovery path)", async () => {
+ // Regression: mem::evict calls event::session::stopped once per recovered
+ // stale session, then runs ONE final consolidation pass. Without the
+ // skipConsolidation guard, N recovered sessions would launch N concurrent
+ // forced full-corpus consolidations + N crystallizations.
+ vi.mocked(isConsolidationEnabled).mockReturnValue(true);
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, mockKV() as never);
+
+ const stopped = handlers.get("event::session::stopped")!;
+ await stopped({ sessionId: "ses_1", skipConsolidation: true });
+
+ const ids = functionIds(trigger);
+ // Per-session work still happens...
+ expect(ids).toContain("mem::summarize");
+ // ...but the corpus-wide fan-out is deferred to evict's single pass.
+ expect(ids).not.toContain("mem::consolidate-pipeline");
+ expect(ids).not.toContain("mem::auto-crystallize");
+ });
+
+ it("still fans out when skipConsolidation is explicitly false (normal stop)", async () => {
+ vi.mocked(isConsolidationEnabled).mockReturnValue(true);
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, mockKV() as never);
+
+ await handlers.get("event::session::stopped")!({
+ sessionId: "ses_1",
+ skipConsolidation: false,
+ });
+
+ const ids = functionIds(trigger);
+ expect(ids).toContain("mem::consolidate-pipeline");
+ expect(ids).toContain("mem::auto-crystallize");
+ });
+
+ it("respects the cooldown across repeated stops but this suite's non-persistent KV fires each single call", async () => {
+ // Sanity: with the default cooldown and a fresh (marker-less) KV, a single
+ // stop still consolidates. The real debounce is exercised below with a
+ // persistent KV.
+ vi.mocked(getConsolidationCooldownMs).mockReturnValue(300000);
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, mockKV() as never);
+ await handlers.get("event::session::stopped")!({ sessionId: "ses_1" });
+ expect(functionIds(trigger)).toContain("mem::consolidate-pipeline");
+ });
+
+ it("respects isReflectEnabled gating for slot-reflect", async () => {
+ vi.mocked(isReflectEnabled).mockReturnValue(false);
+ const off = mockSdk();
+ registerEventTriggers(off.sdk as never, mockKV() as never);
+ await off.handlers.get("event::session::stopped")!({ sessionId: "ses_1" });
+ expect(functionIds(off.trigger)).not.toContain("mem::slot-reflect");
+
+ vi.mocked(isReflectEnabled).mockReturnValue(true);
+ const on = mockSdk();
+ registerEventTriggers(on.sdk as never, mockKV() as never);
+ await on.handlers.get("event::session::stopped")!({ sessionId: "ses_1" });
+ expect(functionIds(on.trigger)).toContain("mem::slot-reflect");
+ });
+
+ it("does not throw and still returns the summary when consolidate-pipeline trigger rejects", async () => {
+ vi.mocked(isConsolidationEnabled).mockReturnValue(true);
+ const { sdk, handlers } = mockSdk({ rejectFor: "mem::consolidate-pipeline" });
+ registerEventTriggers(sdk as never, mockKV() as never);
+
+ const stopped = handlers.get("event::session::stopped")!;
+ const summary = await stopped({ sessionId: "ses_1" });
+
+ expect(summary).toEqual({ summary: "session summary", sessionId: "ses_1" });
+ await Promise.resolve();
+ await Promise.resolve();
+ expect(logger.warn).toHaveBeenCalledWith(
+ "mem::consolidate-pipeline trigger failed",
+ expect.objectContaining({ sessionId: "ses_1" }),
+ );
+ });
+});
+
+// The client session-end hook is bundled into a standalone binary that reads
+// stdin and POSTs to REST, so it is exercised at the source level: after the
+// double-fire fix it must no longer POST the two direct consolidation
+// endpoints, leaving event::session::stopped as the only consolidation path.
+describe("session-end hook no longer double-fires consolidation", () => {
+ const src = readFileSync("src/hooks/session-end.ts", "utf-8");
+
+ it("does not POST /agentmemory/crystals/auto", () => {
+ expect(src).not.toContain("/agentmemory/crystals/auto");
+ });
+
+ it("does not POST /agentmemory/consolidate-pipeline", () => {
+ expect(src).not.toContain("/agentmemory/consolidate-pipeline");
+ });
+
+ it("no longer references the CONSOLIDATION_ENABLED gate", () => {
+ expect(src).not.toContain("CONSOLIDATION_ENABLED");
+ });
+
+ it("still POSTs /agentmemory/session/end (the single source of truth path)", () => {
+ expect(src).toContain("/agentmemory/session/end");
+ });
+
+ it("keeps the claude-bridge/sync block", () => {
+ expect(src).toContain("/agentmemory/claude-bridge/sync");
+ });
+
+ it("keeps the null-guard and main().catch() patterns", () => {
+ expect(src).toContain('if (!data || typeof data !== "object") return;');
+ expect(src).toContain("main().catch(() => process.exit(0));");
+ });
+});
+
+// A KV that actually persists writes, so the debounce marker survives between
+// simulated per-turn stops.
+function persistentKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: vi.fn(async (scope: string, key: string) => store.get(scope)?.get(key) ?? null),
+ set: vi.fn(async (scope: string, key: string, data: unknown) => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ }),
+ delete: vi.fn(async () => {}),
+ update: vi.fn(async () => {}),
+ list: vi.fn(async () => []),
+ };
+}
+
+// Regression: the Stop hook posts /session/end on every agent turn, which fires
+// event::session::stopped. consolidate-pipeline + auto-crystallize are full
+// corpus LLM work with no internal "nothing changed" guard, so firing them per
+// turn is a cost/latency storm for connected agents (Claude/Codex/Copilot/
+// Hermes). The debounce bounds corpus consolidation to once per cooldown.
+describe("session-stop consolidation debounce", () => {
+ beforeEach(() => {
+ vi.mocked(isConsolidationEnabled).mockReturnValue(true);
+ vi.mocked(isGraphExtractionEnabled).mockReturnValue(false);
+ vi.mocked(isReflectEnabled).mockReturnValue(false);
+ vi.mocked(getConsolidationCooldownMs).mockReturnValue(300000);
+ });
+
+ it("consolidates at most once across many per-turn stops within the cooldown", async () => {
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, persistentKV() as never);
+ const stopped = handlers.get("event::session::stopped")!;
+
+ // 5 per-turn Stop hooks in quick succession (all within the cooldown).
+ for (let i = 0; i < 5; i++) await stopped({ sessionId: "ses_1" });
+
+ const count = (id: string) =>
+ trigger.mock.calls.filter(
+ (c) => (c[0] as { function_id: string }).function_id === id,
+ ).length;
+
+ // Corpus consolidation runs ONCE, not five times.
+ expect(count("mem::consolidate-pipeline")).toBe(1);
+ expect(count("mem::auto-crystallize")).toBe(1);
+ // Per-turn summary capture still runs every turn (the cheap path).
+ expect(count("mem::summarize")).toBe(5);
+ });
+
+ it("consolidates once when stops arrive concurrently (serialized cooldown check)", async () => {
+ // Regression: without serialization, two stops racing through the marker
+ // read-check-write both observe the stale marker and both fire.
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, persistentKV() as never);
+ const stopped = handlers.get("event::session::stopped")!;
+
+ await Promise.all([
+ stopped({ sessionId: "ses_1" }),
+ stopped({ sessionId: "ses_2" }),
+ stopped({ sessionId: "ses_3" }),
+ ]);
+
+ const consolidateCount = trigger.mock.calls.filter(
+ (c) => (c[0] as { function_id: string }).function_id === "mem::consolidate-pipeline",
+ ).length;
+ expect(consolidateCount).toBe(1);
+ });
+
+ it("consolidates on every stop when the cooldown is disabled (0)", async () => {
+ vi.mocked(getConsolidationCooldownMs).mockReturnValue(0);
+ const { sdk, handlers, trigger } = mockSdk();
+ registerEventTriggers(sdk as never, persistentKV() as never);
+ const stopped = handlers.get("event::session::stopped")!;
+
+ await stopped({ sessionId: "ses_1" });
+ await stopped({ sessionId: "ses_1" });
+
+ const consolidateCount = trigger.mock.calls.filter(
+ (c) => (c[0] as { function_id: string }).function_id === "mem::consolidate-pipeline",
+ ).length;
+ expect(consolidateCount).toBe(2);
+ });
+});
diff --git a/test/evict.test.ts b/test/evict.test.ts
index d1aedac2..690e8a7c 100644
--- a/test/evict.test.ts
+++ b/test/evict.test.ts
@@ -11,6 +11,13 @@ vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
+// The recovered-session consolidation pass is gated on isConsolidationEnabled
+// (keyless installs skip it); force it on so these tests exercise the pass.
+vi.mock("../src/config.js", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../src/config.js")>()),
+ isConsolidationEnabled: () => true,
+}));
+
type Store = Map<string, Map<string, unknown>>;
type Handler = (payload: unknown) => unknown | Promise<unknown>;
@@ -126,7 +133,9 @@ describe("mem::evict stale sessions", () => {
registerEvictFunction(sdk as never, kv as never);
sdk.registerFunction("event::session::stopped", async (payload) => {
- expect(payload).toEqual({ sessionId });
+ // Recovery must pass skipConsolidation so the per-session fan-out is
+ // suppressed (evict runs a single corpus-wide pass afterwards).
+ expect(payload).toEqual({ sessionId, skipConsolidation: true });
expect(await kv.get(KV.sessions, sessionId)).toMatchObject({
id: sessionId,
});
@@ -135,6 +144,7 @@ describe("mem::evict stale sessions", () => {
sdk.registerFunction("mem::consolidate-pipeline", () => ({
success: true,
}));
+ sdk.registerFunction("mem::auto-crystallize", () => ({ success: true }));
const result = (await sdk.trigger({
function_id: "mem::evict",
@@ -157,6 +167,53 @@ describe("mem::evict stale sessions", () => {
);
});
+ it("bounds consolidation to one pass regardless of how many stale sessions are recovered", async () => {
+ // Regression (P1): before the skipConsolidation guard, N recovered
+ // sessions each triggered a forced full-corpus consolidate + crystallize
+ // via the session::stopped fan-out, on top of evict's final pass — an
+ // N+1 amplification of an expensive LLM path. Recovery must stay O(1).
+ const ids = ["ses_a", "ses_b", "ses_c"];
+ const store: Store = new Map([
+ [
+ KV.sessions,
+ new Map(ids.map((id) => [id, makeSession(id)])),
+ ],
+ [KV.summaries, new Map()],
+ [KV.config, new Map()],
+ [KV.audit, new Map()],
+ ]);
+ for (const id of ids) {
+ store.set(
+ KV.observations(id),
+ new Map([["obs_1", makeObservation(id)]]),
+ );
+ }
+ const kv = mockKV(store);
+ const { sdk, calls } = mockSdk();
+
+ registerEvictFunction(sdk as never, kv as never);
+ const stoppedPayloads: unknown[] = [];
+ sdk.registerFunction("event::session::stopped", (payload) => {
+ stoppedPayloads.push(payload);
+ return { success: true };
+ });
+ sdk.registerFunction("mem::consolidate-pipeline", () => ({ success: true }));
+ sdk.registerFunction("mem::auto-crystallize", () => ({ success: true }));
+
+ await sdk.trigger({ function_id: "mem::evict", payload: {} });
+
+ // session::stopped fires once per recovered session, each suppressing its
+ // own fan-out...
+ expect(stoppedPayloads).toHaveLength(3);
+ for (const p of stoppedPayloads) {
+ expect(p).toMatchObject({ skipConsolidation: true });
+ }
+ // ...and the corpus-wide consolidation + crystallization run exactly once.
+ const fnIds = calls.map((c) => c.function_id);
+ expect(fnIds.filter((f) => f === "mem::consolidate-pipeline")).toHaveLength(1);
+ expect(fnIds.filter((f) => f === "mem::auto-crystallize")).toHaveLength(1);
+ });
+
it("keeps a stale observed session when recovery fails", async () => {
const sessionId = "ses_unrecovered";
const store = storeForObservedSession(sessionId);
diff --git a/test/export-import.test.ts b/test/export-import.test.ts
index 39c17cab..a345ca3d 100644
--- a/test/export-import.test.ts
+++ b/test/export-import.test.ts
@@ -5,6 +5,7 @@ vi.mock("../src/logger.js", () => ({
}));
import { registerExportImportFunction } from "../src/functions/export-import.js";
+import { getSearchIndex } from "../src/functions/search.js";
import type {
Session,
CompressedObservation,
@@ -108,6 +109,10 @@ describe("Export/Import Functions", () => {
beforeEach(async () => {
sdk = mockSdk();
kv = mockKV();
+ // getSearchIndex() returns a module-level singleton shared across
+ // tests. Clear it so index assertions here don't see rows added by
+ // a prior test's import.
+ getSearchIndex().clear();
registerExportImportFunction(sdk as never, kv as never);
await kv.set("mem:sessions", "ses_1", testSession);
@@ -151,6 +156,59 @@ describe("Export/Import Functions", () => {
expect(allSessions.length).toBe(2);
});
+ it("import adds imported records to the search index", async () => {
+ // Regression: mem::import wrote rows to KV but never indexed them.
+ // On an existing install the boot rebuild gate (bm25.size === 0) is
+ // false, so imported data stayed invisible to mem::search forever.
+ const importedObs: CompressedObservation = {
+ id: "obs_imported",
+ sessionId: "ses_imported",
+ timestamp: "2026-03-01T10:00:00Z",
+ type: "file_edit",
+ title: "Kubernetes deployment rollout",
+ facts: ["Scaled replicas"],
+ narrative: "Adjusted the kubernetes deployment rollout strategy",
+ concepts: ["k8s"],
+ files: ["deploy.yaml"],
+ importance: 6,
+ };
+ const importedMem: Memory = {
+ ...testMemory,
+ id: "mem_imported",
+ title: "Postgres connection pooling",
+ content: "Use pgbouncer for postgres connection pooling",
+ };
+ const exportData: ExportData = {
+ version: "0.9.28",
+ exportedAt: new Date().toISOString(),
+ sessions: [
+ { ...testSession, id: "ses_imported", observationCount: 1 },
+ ],
+ observations: { ses_imported: [importedObs] },
+ memories: [importedMem],
+ summaries: [],
+ };
+
+ const result = (await sdk.trigger("mem::import", {
+ exportData,
+ strategy: "merge",
+ })) as { success: boolean; observations: number; memories: number };
+
+ expect(result.success).toBe(true);
+ expect(result.observations).toBe(1);
+ expect(result.memories).toBe(1);
+
+ const idx = getSearchIndex();
+ expect(idx.has("obs_imported")).toBe(true);
+ expect(idx.has("mem_imported")).toBe(true);
+
+ const obsHit = idx.search("kubernetes rollout");
+ expect(obsHit.some((r) => r.obsId === "obs_imported")).toBe(true);
+
+ const memHit = idx.search("postgres pooling");
+ expect(memHit.some((r) => r.obsId === "mem_imported")).toBe(true);
+ });
+
it("import with skip strategy does not overwrite existing", async () => {
const exportData: ExportData = {
version: "0.3.0",
diff --git a/test/fetch-timeout.test.ts b/test/fetch-timeout.test.ts
index 5b2cd7c9..03bf9580 100644
--- a/test/fetch-timeout.test.ts
+++ b/test/fetch-timeout.test.ts
@@ -74,6 +74,241 @@ describe("fetchWithTimeout", () => {
});
});
+// ─────────────────────────────────────────────────────────────
+// Bounded-retry total-deadline tests
+//
+// The retry wrapper must bound TOTAL elapsed time across every
+// attempt + sleep — not per-attempt — so worst case never blows
+// past the caller's budget or the iii 180s invocation timeout.
+// ─────────────────────────────────────────────────────────────
+describe("fetchWithTimeout bounded retry (total deadline)", () => {
+ // Builds a fetch mock that replays a queue of {status, headers} in order,
+ // repeating the last entry once the queue is exhausted. Records how many
+ // times it was invoked so we can assert attempt counts.
+ function queuedFetch(
+ responses: Array<{ status: number; headers?: Record<string, string> }>,
+ ): { fetch: typeof fetch; calls: () => number } {
+ let i = 0;
+ const impl = (async () => {
+ const spec = responses[Math.min(i, responses.length - 1)];
+ i++;
+ return new Response(null, {
+ status: spec.status,
+ headers: spec.headers,
+ });
+ }) as typeof fetch;
+ return { fetch: impl, calls: () => i };
+ }
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ delete process.env["AGENTMEMORY_LLM_TIMEOUT_MS"];
+ });
+
+ // 0. The FIRST attempt must honor the hard budget cap, not the raw caller
+ // timeout. Regression: fetchOnce was called with `ms` instead of the
+ // capped `budgetMs`, so a large caller timeout (e.g. 300s) could hang the
+ // initial request past HARD_BUDGET_CAP_MS (170s) and the iii 180s
+ // invocation timeout before any retry logic ran.
+ it("caps the first attempt at the hard budget, not the raw caller timeout", async () => {
+ const signals: AbortSignal[] = [];
+ const capturing = ((_url: string, init?: RequestInit) => {
+ const signal = init!.signal as AbortSignal;
+ signals.push(signal);
+ return new Promise<Response>((_resolve, reject) => {
+ signal.addEventListener("abort", () =>
+ reject(new DOMException("AbortError", "AbortError")),
+ );
+ });
+ }) as typeof fetch;
+ vi.spyOn(globalThis, "fetch").mockImplementation(capturing);
+ vi.useFakeTimers();
+
+ // 300s caller timeout — far above the 170s HARD_BUDGET_CAP_MS.
+ const p = fetchWithTimeout("https://example.com", {}, 300000);
+ p.catch(() => {}); // observe rejection so it isn't flagged unhandled
+
+ expect(signals).toHaveLength(1);
+ // Just before the cap the first attempt is still alive.
+ await vi.advanceTimersByTimeAsync(169999);
+ expect(signals[0].aborted).toBe(false);
+ // At the cap it must abort. Pre-fix (raw 300s) it would still be alive.
+ await vi.advanceTimersByTimeAsync(2);
+ expect(signals[0].aborted).toBe(true);
+
+ await expect(p).rejects.toThrow();
+ });
+
+ // 1. 429 then 200 → retried exactly once, resolves 200.
+ it("retries once on 429 then resolves the follow-up 200", async () => {
+ const q = queuedFetch([{ status: 429 }, { status: 200 }]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+
+ const p = fetchWithTimeout("https://example.com", {}, 60000);
+ // Drain the backoff sleep (500ms default) so the retry fires.
+ await vi.advanceTimersByTimeAsync(500);
+ const res = await p;
+
+ expect(res.status).toBe(200);
+ expect(q.calls()).toBe(2);
+ });
+
+ // 2. 429 with a hostile Retry-After (100000s) must NOT wait that long. The
+ // delay collapses to the low per-delay cap (5000ms) so the retry fires
+ // quickly and total elapsed stays nowhere near 100000s.
+ it("does not honour a hostile Retry-After literally — caps it and stays within budget", async () => {
+ const q = queuedFetch([
+ { status: 429, headers: { "Retry-After": "100000" } }, // 100000s
+ { status: 200 },
+ ]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+ const start = Date.now();
+
+ const p = fetchWithTimeout("https://example.com", {}, 60000);
+ // runAllTimers drains every scheduled sleep; if the code honored 100000s
+ // literally this would advance 100_000_000ms of simulated time.
+ await vi.runAllTimersAsync();
+ const res = await p;
+ const elapsed = Date.now() - start;
+
+ expect(res.status).toBe(200);
+ expect(q.calls()).toBe(2);
+ // The only sleep was the capped 5000ms delay — total elapsed is bounded to
+ // the cap, nowhere near the 100_000_000ms the header requested.
+ expect(elapsed).toBeLessThanOrEqual(5000);
+ expect(elapsed).toBeLessThan(60000);
+ });
+
+ // 2c. When the capped delay still cannot fit inside a small budget, we do NOT
+ // retry and return the last 429 within bound.
+ it("returns the last 429 without retrying when even the capped delay overruns a small budget", async () => {
+ const q = queuedFetch([
+ { status: 429, headers: { "Retry-After": "100000" } },
+ { status: 200 },
+ ]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+ const start = Date.now();
+
+ // Budget 1000ms: capped delay 5000ms + floor 100ms > remaining, no retry.
+ const p = fetchWithTimeout("https://example.com", {}, 1000);
+ await vi.runAllTimersAsync();
+ const res = await p;
+ const elapsed = Date.now() - start;
+
+ expect(res.status).toBe(429);
+ expect(q.calls()).toBe(1);
+ expect(elapsed).toBeLessThan(1000);
+ });
+
+ // 2b. Retry-After present but LARGER than the low per-delay cap collapses to
+ // MAX_RETRY_DELAY_MS (5000), still bounded, still retries when budget
+ // allows.
+ it("caps an oversized Retry-After to the max delay and still retries within budget", async () => {
+ const q = queuedFetch([
+ { status: 503, headers: { "Retry-After": "60" } }, // 60s requested
+ { status: 200 },
+ ]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+
+ const p = fetchWithTimeout("https://example.com", {}, 60000);
+ // Requested 60s, but the honored delay is capped at 5000ms.
+ await vi.advanceTimersByTimeAsync(5000);
+ const res = await p;
+
+ expect(res.status).toBe(200);
+ expect(q.calls()).toBe(2);
+ });
+
+ // 3. Persistent 503 → stops after the bounded attempts AND within the total
+ // deadline. Attempt count is bounded at MAX_ATTEMPTS (3) and total
+ // simulated elapsed stays under the budget.
+ it("stops persistent 503 at the attempt cap and within the deadline", async () => {
+ const q = queuedFetch([{ status: 503 }]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+ const start = Date.now();
+
+ const p = fetchWithTimeout("https://example.com", {}, 60000);
+ // Two retries with 500ms + 1000ms exponential backoff.
+ await vi.advanceTimersByTimeAsync(500);
+ await vi.advanceTimersByTimeAsync(1000);
+ const res = await p;
+ const elapsed = Date.now() - start;
+
+ expect(res.status).toBe(503);
+ // Initial attempt + 2 retries = MAX_ATTEMPTS.
+ expect(q.calls()).toBe(3);
+ expect(elapsed).toBeLessThan(60000);
+ // Total simulated sleep was only the two backoffs.
+ expect(elapsed).toBeLessThanOrEqual(1500);
+ });
+
+ // 3b. A tiny budget forces us to bail before even the first retry sleep —
+ // the sleep + minimal attempt floor already overruns the remaining
+ // budget, so we return the first 503 without retrying.
+ it("does not retry when the budget is too small to fit a retry", async () => {
+ const q = queuedFetch([{ status: 503 }, { status: 200 }]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+
+ // Budget of 200ms: backoff 500ms + floor 100ms > remaining, so no retry.
+ const p = fetchWithTimeout("https://example.com", {}, 200);
+ await vi.runAllTimersAsync();
+ const res = await p;
+
+ expect(res.status).toBe(503);
+ expect(q.calls()).toBe(1);
+ });
+
+ // 5. Retry-After as an HTTP-date is parsed, capped to the max delay, and the
+ // retry fires within budget.
+ it("parses an HTTP-date Retry-After and caps the honored delay", async () => {
+ // 2s in the future — under the max delay cap, so honored as-is.
+ const future = new Date(Date.now() + 2000).toUTCString();
+ const q = queuedFetch([
+ { status: 429, headers: { "Retry-After": future } },
+ { status: 200 },
+ ]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+
+ const p = fetchWithTimeout("https://example.com", {}, 60000);
+ await vi.advanceTimersByTimeAsync(2000);
+ const res = await p;
+
+ expect(res.status).toBe(200);
+ expect(q.calls()).toBe(2);
+ });
+
+ // 5b. A far-future HTTP-date collapses to the capped delay (5000ms), which
+ // still exceeds a small budget → no retry, last 503 returned in bound.
+ it("does not retry on a far-future HTTP-date Retry-After that overruns a small budget", async () => {
+ const farFuture = new Date(Date.now() + 3600_000).toUTCString(); // 1h out
+ const q = queuedFetch([
+ { status: 503, headers: { "Retry-After": farFuture } },
+ { status: 200 },
+ ]);
+ vi.spyOn(globalThis, "fetch").mockImplementation(q.fetch);
+ vi.useFakeTimers();
+ const start = Date.now();
+
+ // Budget 1000ms: capped delay 5000ms + floor > remaining, so no retry.
+ const p = fetchWithTimeout("https://example.com", {}, 1000);
+ await vi.runAllTimersAsync();
+ const res = await p;
+ const elapsed = Date.now() - start;
+
+ expect(res.status).toBe(503);
+ expect(q.calls()).toBe(1);
+ expect(elapsed).toBeLessThan(1000);
+ });
+});
+
// ─────────────────────────────────────────────────────────────
// Provider hang regression tests
// Each provider must call fetchWithTimeout, which honours the
diff --git a/test/fs-watcher.test.ts b/test/fs-watcher.test.ts
index 48c1b094..8375c686 100644
--- a/test/fs-watcher.test.ts
+++ b/test/fs-watcher.test.ts
@@ -98,6 +98,10 @@ describe("FilesystemWatcher", { retry: 2 }, () => {
});
it("throws if no watched roots could be attached", () => {
+ // Regression: on Linux with Node 24+, fs.watch on a nonexistent path no
+ // longer throws synchronously, so the watcher must stat roots itself or a
+ // missing root silently counts as attached (caught by the Node 24/26 CI
+ // matrix rows).
const w = new FilesystemWatcher({
roots: ["/definitely/does/not/exist/xyz123"],
baseUrl: "http://localhost:3111",
@@ -106,6 +110,17 @@ describe("FilesystemWatcher", { retry: 2 }, () => {
expect(() => w.start()).toThrow(/could not watch any of the configured roots/);
});
+ it("rejects a root that is a file, not a directory", () => {
+ const filePath = join(root, "not-a-dir.txt");
+ writeFileSync(filePath, "plain file");
+ const w = new FilesystemWatcher({
+ roots: [filePath],
+ baseUrl: "http://localhost:3111",
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+ });
+ expect(() => w.start()).toThrow(/could not watch any of the configured roots/);
+ });
+
it("ignores paths that match the default ignore set", async () => {
mkdirSync(join(root, "node_modules"), { recursive: true });
const w = new FilesystemWatcher({
diff --git a/test/graph-import.test.ts b/test/graph-import.test.ts
new file mode 100644
index 00000000..e342cc8a
--- /dev/null
+++ b/test/graph-import.test.ts
@@ -0,0 +1,218 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+ bootLog: vi.fn(),
+}));
+
+import { parseGraphifyGraph, registerGraphImportFunction } from "../src/functions/graph-import.js";
+import { KV } from "../src/state/schema.js";
+import type { GraphNode, GraphEdge } from "../src/types.js";
+
+// graphify's clustered graph.json is NetworkX node_link: nodes carry
+// label/source_file/community/file_type, links carry
+// source/target/relation/confidence. --no-cluster output stores the edge
+// array under "edges" instead of "links".
+const FIXTURE = {
+ nodes: [
+ { id: "n1", label: "extract", source_file: "extract.py", community: 0, file_type: "code" },
+ { id: "n2", label: "cluster.py", source_file: "cluster.py", community: 0, file_type: "code" },
+ { id: "n3", label: "retry rationale", community: 1, file_type: "rationale" },
+ { id: "n4", label: "architecture overview", file_type: "document" },
+ ],
+ links: [
+ { source: "n1", target: "n2", relation: "imports", confidence: "EXTRACTED" },
+ { source: "n1", target: "n3", relation: "references", confidence: "INFERRED" },
+ { source: "n2", target: "zz-missing", relation: "calls", confidence: "EXTRACTED" },
+ ],
+};
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> =>
+ (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> =>
+ Array.from(store.get(scope)?.values() ?? []) as T[],
+ _store: store,
+ };
+}
+
+function mockSdk() {
+ const fns = new Map<string, Function>();
+ return {
+ registerFunction: (id: string, handler: Function) => fns.set(id, handler),
+ registerTrigger: () => {},
+ trigger: async (input: { function_id: string; payload?: unknown }) => {
+ const fn = fns.get(input.function_id);
+ if (!fn) throw new Error(`missing handler: ${input.function_id}`);
+ return fn(input.payload);
+ },
+ } as never;
+}
+
+describe("parseGraphifyGraph", () => {
+ it("maps nodes with file_type-aware types and provenance", () => {
+ const parsed = parseGraphifyGraph(JSON.stringify(FIXTURE));
+ expect(parsed.nodesRead).toBe(4);
+ const byName = new Map(parsed.nodes.map((n) => [n.name, n]));
+ // code symbol without extension -> function; with extension -> file
+ expect(byName.get("extract")!.type).toBe("function");
+ expect(byName.get("cluster.py")!.type).toBe("file");
+ // rationale -> decision, document -> concept
+ expect(byName.get("retry rationale")!.type).toBe("decision");
+ expect(byName.get("architecture overview")!.type).toBe("concept");
+ // provenance kept on every imported node
+ for (const n of parsed.nodes) {
+ expect(n.properties.source).toBe("graphify");
+ }
+ expect(byName.get("extract")!.properties.sourceFile).toBe("extract.py");
+ });
+
+ it("maps relations to memory edge types and confidence to weight", () => {
+ const parsed = parseGraphifyGraph(JSON.stringify(FIXTURE));
+ // the edge to a missing endpoint is skipped and counted, never guessed
+ expect(parsed.edges).toHaveLength(2);
+ expect(parsed.skippedEdges).toBe(1);
+ const types = parsed.edges.map((e) => e.type).sort();
+ expect(types).toEqual(["imports", "related_to"]);
+ const weights = parsed.edges.map((e) => e.weight).sort((a, b) => a - b);
+ expect(weights).toEqual([0.6, 0.9]);
+ });
+
+ it("maps AMBIGUOUS confidence and unknown file_type defaults", () => {
+ const fixture = {
+ nodes: [
+ { id: "a", label: "mystery" },
+ { id: "b", label: "helper.rs", file_type: "wat" },
+ ],
+ links: [{ source: "a", target: "b", relation: "calls", confidence: "AMBIGUOUS" }],
+ };
+ const parsed = parseGraphifyGraph(JSON.stringify(fixture));
+ const byName = new Map(parsed.nodes.map((n) => [n.name, n]));
+ // no/unknown file_type: extension-looking labels are files, rest concepts
+ expect(byName.get("mystery")!.type).toBe("concept");
+ expect(byName.get("helper.rs")!.type).toBe("file");
+ expect(parsed.edges[0].weight).toBe(0.3);
+ });
+
+ it("accepts the --no-cluster shape where edges live under `edges`", () => {
+ const noCluster = { nodes: FIXTURE.nodes, edges: FIXTURE.links };
+ const parsed = parseGraphifyGraph(JSON.stringify(noCluster));
+ expect(parsed.edges).toHaveLength(2);
+ expect(parsed.edgesRead).toBe(3);
+ });
+
+ it("reports truncation loudly instead of silently capping", () => {
+ const big = {
+ nodes: Array.from({ length: 5010 }, (_, i) => ({ id: `n${i}`, label: `sym${i}` })),
+ links: [],
+ };
+ const parsed = parseGraphifyGraph(JSON.stringify(big));
+ expect(parsed.nodes).toHaveLength(5000);
+ expect(parsed.truncated).toEqual({ nodes: 10, edges: 0 });
+ });
+});
+
+describe("mem::graph::import-graphify", () => {
+ let tmp: string;
+ let kv: ReturnType<typeof mockKV>;
+ let sdk: ReturnType<typeof mockSdk>;
+
+ beforeEach(() => {
+ tmp = mkdtempSync(join(tmpdir(), "am-graphify-"));
+ mkdirSync(join(tmp, "graphify-out"), { recursive: true });
+ writeFileSync(join(tmp, "graphify-out", "graph.json"), JSON.stringify(FIXTURE));
+ kv = mockKV();
+ sdk = mockSdk();
+ registerGraphImportFunction(sdk, kv as never);
+ });
+
+ afterEach(() => {
+ rmSync(tmp, { recursive: true, force: true });
+ });
+
+ it("imports nodes and edges into the memory graph", async () => {
+ const result = (await (sdk as any).trigger({
+ function_id: "mem::graph::import-graphify",
+ payload: { cwd: tmp },
+ })) as { success: boolean; newNodes: number; newEdges: number; skippedEdges: number };
+
+ expect(result.success).toBe(true);
+ expect(result.newNodes).toBe(4);
+ expect(result.newEdges).toBe(2);
+ expect(result.skippedEdges).toBe(1);
+
+ const nodes = await kv.list<GraphNode>(KV.graphNodes);
+ const edges = await kv.list<GraphEdge>(KV.graphEdges);
+ expect(nodes).toHaveLength(4);
+ expect(edges).toHaveLength(2);
+ });
+
+ it("re-import is idempotent: second run merges instead of duplicating", async () => {
+ await (sdk as any).trigger({ function_id: "mem::graph::import-graphify", payload: { cwd: tmp } });
+ const second = (await (sdk as any).trigger({
+ function_id: "mem::graph::import-graphify",
+ payload: { cwd: tmp },
+ })) as { success: boolean; newNodes: number; newEdges: number };
+
+ expect(second.success).toBe(true);
+ // Everything resolves through the (type, name) index and merges.
+ expect(second.newNodes).toBe(0);
+ expect(second.newEdges).toBe(0);
+ expect(await kv.list(KV.graphNodes)).toHaveLength(4);
+ expect(await kv.list(KV.graphEdges)).toHaveLength(2);
+
+ // A merge-only run mutates cached snapshot entries even with zero new
+ // counts; the persisted snapshot must still reflect current graph data.
+ const snap = await kv.get<{
+ stats: { totalNodes: number; totalEdges: number };
+ topNodes: unknown[];
+ }>("mem:graph:snapshot", "current");
+ expect(snap).not.toBeNull();
+ expect(snap!.stats.totalNodes).toBe(4);
+ expect(snap!.stats.totalEdges).toBe(2);
+ expect(snap!.topNodes.length).toBeGreaterThan(0);
+ });
+
+ it("fails cleanly with a pointer when graph.json is absent", async () => {
+ const result = (await (sdk as any).trigger({
+ function_id: "mem::graph::import-graphify",
+ payload: { cwd: join(tmp, "nowhere") },
+ })) as { success: boolean; error: string };
+ expect(result.success).toBe(false);
+ expect(result.error).toContain("Run graphify first");
+ });
+
+ it("accepts an explicit path", async () => {
+ const alt = join(tmp, "custom.json");
+ writeFileSync(alt, JSON.stringify(FIXTURE));
+ const result = (await (sdk as any).trigger({
+ function_id: "mem::graph::import-graphify",
+ payload: { path: alt },
+ })) as { success: boolean; nodesImported: number };
+ expect(result.success).toBe(true);
+ expect(result.nodesImported).toBe(4);
+ });
+
+ it("rejects malformed JSON without writing anything", async () => {
+ writeFileSync(join(tmp, "graphify-out", "graph.json"), "{not json");
+ const result = (await (sdk as any).trigger({
+ function_id: "mem::graph::import-graphify",
+ payload: { cwd: tmp },
+ })) as { success: boolean };
+ expect(result.success).toBe(false);
+ expect(await kv.list(KV.graphNodes)).toHaveLength(0);
+ });
+});
diff --git a/test/remember-project-scope.test.ts b/test/remember-project-scope.test.ts
index bda699f5..31bbf43e 100644
--- a/test/remember-project-scope.test.ts
+++ b/test/remember-project-scope.test.ts
@@ -256,3 +256,81 @@ describe("mem::remember — cross-project dedup isolation", () => {
expect(unscoped.memory.supersedes).toContain(scoped.memory.id);
});
});
+
+describe("mem::remember — CJK dedup", () => {
+ beforeEach(() => {
+ getSearchIndex().clear();
+ setIndexPersistence(null);
+ });
+
+ afterEach(() => {
+ setIndexPersistence(null);
+ });
+
+ it("dedups two near-identical CJK memories (new one supersedes old)", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ const first = await sdk.trigger({
+ function_id: "mem::remember",
+ payload: {
+ content: "用户认证中间件必须先去除请求头里的 Bearer 前缀然后再校验令牌",
+ type: "pattern",
+ },
+ }) as { memory: { id: string } };
+
+ const second = await sdk.trigger({
+ function_id: "mem::remember",
+ payload: {
+ content: "用户认证中间件必须先去除请求头里的 Bearer 前缀然后校验令牌",
+ type: "pattern",
+ },
+ }) as { memory: { supersedes: string[] } };
+
+ expect(second.memory.supersedes).toContain(first.memory.id);
+
+ const original = await kv.get<{ isLatest: boolean }>("mem:memories", first.memory.id);
+ expect(original?.isLatest).toBe(false);
+ });
+
+ it("does NOT supersede two unrelated short CJK memories (北京 vs 上海)", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ const beijing = await sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content: "北京", type: "fact" },
+ }) as { memory: { id: string } };
+
+ const shanghai = await sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content: "上海", type: "fact" },
+ }) as { memory: { supersedes: string[] } };
+
+ // The old empty-set shortcut returned similarity 1 here and falsely
+ // chained "上海" as a new version of "北京".
+ expect(shanghai.memory.supersedes).toHaveLength(0);
+
+ const original = await kv.get<{ isLatest: boolean }>("mem:memories", beijing.memory.id);
+ expect(original?.isLatest).toBe(true);
+ });
+
+ it("preserves a trailing astral character in the title (no lone surrogate)", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ // 80th UTF-16 code unit falls inside a surrogate pair; the title must
+ // not end on a lone high surrogate.
+ const content = "x".repeat(79) + "😀 trailing";
+ const result = await sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content, type: "fact" },
+ }) as { memory: { title: string } };
+
+ expect(/[\uD800-\uDBFF]$/.test(result.memory.title)).toBe(false);
+ expect(result.memory.title.length).toBe(79);
+ });
+});
diff --git a/test/replay-import-key.test.ts b/test/replay-import-key.test.ts
index 9a4c6467..1c1c991f 100644
--- a/test/replay-import-key.test.ts
+++ b/test/replay-import-key.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeEach, vi } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -9,6 +9,13 @@ vi.mock("../src/logger.js", () => ({
import { registerReplayFunctions } from "../src/functions/replay.js";
import { KV } from "../src/state/schema.js";
+import {
+ getSearchIndex,
+ setVectorIndex,
+ setEmbeddingProvider,
+} from "../src/functions/search.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import type { EmbeddingProvider } from "../src/types.js";
function mockKV() {
const store = new Map<string, Map<string, unknown>>();
@@ -152,3 +159,90 @@ describe("import-jsonl re-key on parsed.sessionId (#775)", () => {
expect(sessionWrites.length).toBe(1);
});
});
+
+describe("import-jsonl indexes observations into BM25 AND vector", () => {
+ const mockEmbedder: EmbeddingProvider = {
+ name: "test",
+ dimensions: 3,
+ embed: async () => new Float32Array([0.1, 0.2, 0.3]),
+ embedBatch: async (texts: string[]) =>
+ texts.map(() => new Float32Array([0.1, 0.2, 0.3])),
+ };
+ let tmpRoot: string;
+ let vectorIndex: VectorIndex;
+
+ beforeEach(() => {
+ tmpRoot = mkdtempSync(join(tmpdir(), "replay-import-index-"));
+ getSearchIndex().clear();
+ vectorIndex = new VectorIndex();
+ setVectorIndex(vectorIndex);
+ setEmbeddingProvider(mockEmbedder);
+ });
+
+ afterEach(() => {
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ getSearchIndex().clear();
+ });
+
+ function writeFixture(sessionId: string) {
+ const dir = join(tmpRoot, "proj");
+ require("node:fs").mkdirSync(dir, { recursive: true });
+ const ts = "2026-04-17T10:00:00.000Z";
+ const lines = [
+ JSON.stringify({
+ type: "user",
+ uuid: "u1",
+ sessionId,
+ timestamp: ts,
+ cwd: tmpRoot,
+ message: { role: "user", content: [{ type: "text", text: "how do I fix the flaky retry" }] },
+ }),
+ JSON.stringify({
+ type: "assistant",
+ uuid: "a1",
+ sessionId,
+ timestamp: ts,
+ message: { role: "assistant", content: [{ type: "text", text: "await the fetch before asserting" }] },
+ }),
+ ];
+ writeFileSync(join(dir, `${sessionId}.jsonl`), lines.join("\n") + "\n");
+ }
+
+ it("populates the vector index (regression: replay used to BM25-add only, leaving imports unsearchable by meaning)", async () => {
+ writeFixture("sess-index");
+ const kv = mockKV();
+ const sdk = mockSdk(kv);
+ registerReplayFunctions(sdk, kv as never);
+
+ expect(vectorIndex.size).toBe(0);
+ expect(getSearchIndex().size).toBe(0);
+
+ const result = (await sdk.trigger("mem::replay::import-jsonl", {
+ path: tmpRoot,
+ })) as { success: boolean; imported?: number };
+
+ expect(result.success).toBe(true);
+ // Both lanes must be populated. The old code left vectorIndex at 0.
+ expect(vectorIndex.size).toBeGreaterThan(0);
+ expect(getSearchIndex().size).toBeGreaterThan(0);
+ expect(vectorIndex.size).toBe(getSearchIndex().size);
+ });
+
+ it("skips the vector lane cleanly when no embedding provider is configured (keyless install)", async () => {
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ writeFixture("sess-keyless");
+ const kv = mockKV();
+ const sdk = mockSdk(kv);
+ registerReplayFunctions(sdk, kv as never);
+
+ const result = (await sdk.trigger("mem::replay::import-jsonl", {
+ path: tmpRoot,
+ })) as { success: boolean };
+
+ // BM25 still works; no crash from the absent vector index.
+ expect(result.success).toBe(true);
+ expect(getSearchIndex().size).toBeGreaterThan(0);
+ });
+});
diff --git a/test/schema.test.ts b/test/schema.test.ts
index 2d3f65d2..94941c42 100644
--- a/test/schema.test.ts
+++ b/test/schema.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest'
-import { KV, STREAM, generateId } from '../src/state/schema.js'
+import { KV, STREAM, generateId, jaccardSimilarity } from '../src/state/schema.js'
describe('KV', () => {
it('has correct session scope', () => {
@@ -40,3 +40,85 @@ describe('generateId', () => {
expect(id.length).toBeGreaterThan(15)
})
})
+
+describe('jaccardSimilarity', () => {
+ it('returns 1 for identical ASCII strings', () => {
+ const s = 'always use express-jwt middleware for token validation'
+ expect(jaccardSimilarity(s, s)).toBe(1)
+ })
+
+ it('keeps ASCII word-level behavior', () => {
+ const a = 'always use express-jwt middleware for token validation'
+ const b = 'always use express-jwt middleware for request validation'
+ const score = jaccardSimilarity(a, b)
+ expect(score).toBeGreaterThan(0.5)
+ expect(score).toBeLessThan(1)
+ })
+
+ it('returns 0 for unrelated ASCII strings', () => {
+ expect(
+ jaccardSimilarity('the quick brown fox', 'lorem ipsum dolor sit'),
+ ).toBe(0)
+ })
+
+ it('returns 0 (never 1) when both token sets are empty and inputs differ', () => {
+ // Two short ASCII strings whose tokens are all filtered out by the
+ // length gate must not be treated as identical.
+ expect(jaccardSimilarity('a b', 'x y')).toBe(0)
+ })
+
+ it('supersedes identical short memories that tokenize to nothing', () => {
+ // Regression: words <=2 chars are dropped, so "AI" / "go" / "a b"
+ // produce empty token sets. Re-saving the exact same short memory must
+ // still be detected as a duplicate (score 1) via an exact-equality
+ // fallback, instead of leaking duplicate latest records.
+ expect(jaccardSimilarity('AI', 'AI')).toBe(1)
+ expect(jaccardSimilarity('go', 'go')).toBe(1)
+ // Unrelated short strings must still score 0, not falsely supersede.
+ expect(jaccardSimilarity('AI', 'ML')).toBe(0)
+ expect(jaccardSimilarity('go', 'AI')).toBe(0)
+ })
+
+ it('treats whitespace-only differences in short text as identical', () => {
+ // The exact-equality fallback collapses runs of whitespace and trims,
+ // so cosmetic spacing differences on an otherwise-empty-token memory
+ // still dedupe.
+ expect(jaccardSimilarity('a b', 'a b')).toBe(1)
+ expect(jaccardSimilarity(' AI ', 'AI')).toBe(1)
+ })
+
+ it('gives high similarity for near-identical CJK sentences', () => {
+ const a = '用户认证中间件必须先去除请求头里的 Bearer 前缀然后再校验令牌'
+ const b = '用户认证中间件必须先去除请求头里的 Bearer 前缀然后校验令牌'
+ expect(jaccardSimilarity(a, b)).toBeGreaterThan(0.7)
+ })
+
+ it('gives low similarity for unrelated short CJK strings', () => {
+ // "北京" vs "上海" — the old empty-set shortcut returned 1 here and
+ // falsely superseded an unrelated memory. Must be well below the
+ // 0.7 supersede threshold.
+ expect(jaccardSimilarity('北京', '上海')).toBeLessThan(0.7)
+ expect(jaccardSimilarity('北京', '上海')).toBe(0)
+ })
+
+ it('detects a duplicate for identical CJK strings', () => {
+ expect(jaccardSimilarity('设置认证中间件', '设置认证中间件')).toBe(1)
+ })
+
+ it('handles Japanese kana without whitespace', () => {
+ const a = 'トークンを検証する前に接頭辞を取り除く'
+ const b = 'トークンを検証する前に接頭辞を削除する'
+ expect(jaccardSimilarity(a, b)).toBeGreaterThan(0.4)
+ expect(jaccardSimilarity('東京', '大阪')).toBe(0)
+ })
+
+ it('NFC-normalizes before comparing', () => {
+ // Composed U+00E9 vs decomposed 'e' + U+0301 combining accent for
+ // "caf\u00e9" must compare equal even though the two strings differ
+ // byte-for-byte before normalization.
+ const composed = 'caf\u00e9 latte order'
+ const decomposed = 'cafe\u0301 latte order'
+ expect(composed).not.toBe(decomposed)
+ expect(jaccardSimilarity(composed, decomposed)).toBe(1)
+ })
+})
diff --git a/test/search.test.ts b/test/search.test.ts
index 7bc635ce..84b8134e 100644
--- a/test/search.test.ts
+++ b/test/search.test.ts
@@ -101,6 +101,10 @@ describe("mem::search", () => {
// Module-level SearchIndex singleton would leak across tests; reset.
getSearchIndex().clear();
+ // mem::search awaits a shared rebuild on a cold index; the explicit call
+ // here pre-populates the index deterministically so the query assertions
+ // below never depend on that cold-start path.
+ await rebuildIndex(kv as never);
});
it("returns full format by default", async () => {
diff --git a/test/snapshot-config.test.ts b/test/snapshot-config.test.ts
new file mode 100644
index 00000000..ee9949d6
--- /dev/null
+++ b/test/snapshot-config.test.ts
@@ -0,0 +1,96 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { loadSnapshotConfig, __resetEnvFileCache } from "../src/config.js";
+
+// loadSnapshotConfig reads getMergedEnv(), which merges the on-disk
+// ~/.agentmemory/.env under process.env. The tests below delete the
+// process.env keys, so a developer machine with SNAPSHOT_* in its real .env
+// would leak into the defaults; point HOME at an empty temp dir and reset
+// the env-file cache so the file layer is deterministic.
+//
+// Regression (P1): a zero/negative SNAPSHOT_INTERVAL flowed straight into
+// setInterval(fn, interval * 1000). Node clamps a non-positive delay to ~1ms,
+// so the git-snapshot timer would fire on nearly every event-loop tick,
+// saturating the worker with back-to-back full-state snapshots + commits.
+// Non-positive values must fall back to the documented 3600s default.
+
+const KEYS = ["SNAPSHOT_INTERVAL", "SNAPSHOT_ENABLED", "SNAPSHOT_DIR"] as const;
+const DEFAULT_INTERVAL = 3600;
+
+describe("loadSnapshotConfig interval validation", () => {
+ const saved: Record<string, string | undefined> = {};
+ let sandboxHome: string;
+ let savedHome: string | undefined;
+ let savedUserProfile: string | undefined;
+
+ beforeEach(() => {
+ sandboxHome = mkdtempSync(join(tmpdir(), "am-snapcfg-"));
+ savedHome = process.env["HOME"];
+ savedUserProfile = process.env["USERPROFILE"];
+ process.env["HOME"] = sandboxHome;
+ process.env["USERPROFILE"] = sandboxHome;
+ __resetEnvFileCache();
+ for (const k of KEYS) {
+ saved[k] = process.env[k];
+ delete process.env[k];
+ }
+ });
+
+ afterEach(() => {
+ for (const k of KEYS) {
+ if (saved[k] === undefined) delete process.env[k];
+ else process.env[k] = saved[k];
+ }
+ if (savedHome === undefined) delete process.env["HOME"];
+ else process.env["HOME"] = savedHome;
+ if (savedUserProfile === undefined) delete process.env["USERPROFILE"];
+ else process.env["USERPROFILE"] = savedUserProfile;
+ __resetEnvFileCache();
+ rmSync(sandboxHome, { recursive: true, force: true });
+ });
+
+ it("falls back to the default when interval is zero", () => {
+ process.env["SNAPSHOT_INTERVAL"] = "0";
+ expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL);
+ });
+
+ it("falls back to the default when interval is negative", () => {
+ process.env["SNAPSHOT_INTERVAL"] = "-5";
+ expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL);
+ });
+
+ it("falls back to the default when interval is non-numeric", () => {
+ process.env["SNAPSHOT_INTERVAL"] = "not-a-number";
+ expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL);
+ });
+
+ it("uses the default when interval is unset", () => {
+ expect(loadSnapshotConfig().interval).toBe(DEFAULT_INTERVAL);
+ });
+
+ it("accepts a valid positive interval unchanged", () => {
+ process.env["SNAPSHOT_INTERVAL"] = "120";
+ expect(loadSnapshotConfig().interval).toBe(120);
+ });
+
+ it("accepts the minimum floor of 1 second", () => {
+ process.env["SNAPSHOT_INTERVAL"] = "1";
+ expect(loadSnapshotConfig().interval).toBe(1);
+ });
+
+ it("never yields an interval that would clamp setInterval to sub-second", () => {
+ for (const bad of ["0", "-1", "-3600", "0.5"]) {
+ process.env["SNAPSHOT_INTERVAL"] = bad;
+ expect(loadSnapshotConfig().interval).toBeGreaterThanOrEqual(1);
+ }
+ });
+
+ it("reports enabled state from SNAPSHOT_ENABLED", () => {
+ process.env["SNAPSHOT_ENABLED"] = "true";
+ expect(loadSnapshotConfig().enabled).toBe(true);
+ process.env["SNAPSHOT_ENABLED"] = "false";
+ expect(loadSnapshotConfig().enabled).toBe(false);
+ });
+});
diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts
index 198774d2..6a052cd5 100644
--- a/test/snapshot.test.ts
+++ b/test/snapshot.test.ts
@@ -164,3 +164,64 @@ describe("Snapshot Functions", () => {
expect(audits.length).toBe(1);
});
});
+
+describe("snapshot-create reentrancy guard", () => {
+ // Regression (P2): mem::snapshot-create is triggered by the periodic timer,
+ // REST (api::snapshot-create), and MCP. Two runs writing state.json and
+ // committing in the same git repo at once race on the index lock. An
+ // overlapping call must be a no-op success while the first run finishes.
+ it("skips an overlapping call and releases the guard on completion", async () => {
+ let releaseFirst!: () => void;
+ const firstListGate = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+ let listCalls = 0;
+ const store = new Map<string, Map<string, unknown>>();
+ const gatedKv = {
+ get: async () => null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async () => {},
+ list: async <T>(scope: string): Promise<T[]> => {
+ listCalls++;
+ // Park the first snapshot inside its initial list() so a second
+ // snapshot-create observes the in-flight guard.
+ if (listCalls === 1) await firstListGate;
+ return (Array.from(store.get(scope)?.values() ?? []) as T[]) ?? [];
+ },
+ };
+ const localSdk = mockSdk();
+ registerSnapshotFunction(localSdk as never, gatedKv as never, "/tmp/reentrant");
+
+ // Start the first snapshot; it parks inside kv.list with the guard held.
+ const p1 = localSdk.trigger("mem::snapshot-create", { message: "first" });
+ await Promise.resolve();
+ await Promise.resolve();
+
+ // Overlapping call: must be rejected as already-in-progress, NOT run git.
+ const r2 = (await localSdk.trigger("mem::snapshot-create", {
+ message: "second",
+ })) as { success: boolean; message?: string; snapshot?: unknown };
+ expect(r2).toEqual({
+ success: true,
+ message: "Snapshot already in progress",
+ });
+ expect(r2.snapshot).toBeUndefined();
+
+ // Release the first run; it completes normally.
+ releaseFirst();
+ const r1 = (await p1) as { success: boolean; snapshot?: unknown };
+ expect(r1.success).toBe(true);
+ expect(r1.snapshot).toBeDefined();
+
+ // Guard is released: a fresh call runs the full body again.
+ const r3 = (await localSdk.trigger("mem::snapshot-create", {
+ message: "third",
+ })) as { success: boolean; snapshot?: unknown };
+ expect(r3.success).toBe(true);
+ expect(r3.snapshot).toBeDefined();
+ });
+});