#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
#!/usr/bin/env node
import { spawn, execFileSync } from "node:child_process";
import { createHash, randomBytes } from "node:crypto";
import {
copyFileSync,
chmodSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
writeFileSync,
appendFileSync,
openSync,
closeSync,
statSync,
} from "node:fs";
import http from "node:http";
import net from "node:net";
import { homedir, tmpdir, platform, arch } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { assertRecoveredVectors, hasPersistedVectorFixture, isVectorLogAcknowledged, isVectorRecoverySettled } from "./recovery.mjs";
const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
const FORBIDDEN_PORTS = new Set([3111, 3113, 4098, 4131, 4132, 4133, 49134]);
const STATE_FLUSH_WAIT_MS = 5000;
const SCENARIOS = [
["install", "packed artifact installs and reports its identity"],
["capture", "bundled hooks capture and search finds every marker"],
["mcp", "installed MCP entrypoints list tools and save/search through the server"],
["offline", "hooks run while the service is down and the spool is recovered on restart"],
["dedup", "a replayed host event after a force kill stays one observation"],
["deadletter", "an accepted capture that fails processing survives a restart and is visible as a dead letter"],
["vectors", "checkpointed and pending vectors survive a crash without re-embedding"],
["stopflush", "agentmemory stop then start loses nothing"],
["status", "viewer serves and /agentmemory/status explains every problem"],
["roundtrip", "export then import into a fresh home round-trips"],
];
function parseArgs(argv) {
const opts = { out: null, keep: false, skipBuild: false, tarball: null, mcpTarball: null, only: null, workerBinDir: null };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
const next = () => argv[++i];
if (a === "--out") opts.out = resolve(next());
else if (a === "--keep") opts.keep = true;
else if (a === "--skip-build") opts.skipBuild = true;
else if (a === "--tarball") opts.tarball = resolve(next());
else if (a === "--mcp-tarball") opts.mcpTarball = resolve(next());
else if (a === "--only") opts.only = new Set(next().split(","));
else if (a === "--worker-bin-dir" || a === "--workers-dir") opts.workerBinDir = resolve(next());
else if (a === "--help" || a === "-h") {
console.log(
"Usage: npm run release:gate -- [--tarball file.tgz] [--mcp-tarball file.tgz] [--skip-build] [--only a,b] [--out dir] [--keep] [--worker-bin-dir directory]\n" +
`Scenarios: ${SCENARIOS.map((s) => s[0]).join(", ")}`,
);
process.exit(0);
} else throw new Error(`unknown argument ${a}`);
}
return opts;
}
const opts = parseArgs(process.argv.slice(2));
const startedAt = Date.now();
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
const OUT = opts.out ?? join(tmpdir(), `agentmemory-release-gate-${stamp}`);
mkdirSync(OUT, { recursive: true });
const ROOT = mkdtempSync(join(tmpdir(), "amgate-"));
const LOG = join(OUT, "gate.log");
const CACHE = process.env["AGENTMEMORY_GATE_CACHE"] || join(homedir(), ".cache", "agentmemory-release-gate");
const NODE_BIN = dirname(process.execPath);
const ownedGroups = new Set();
const ownedChildren = new Set();
let cleanedUp = false;
function log(msg) {
const line = `[${((Date.now() - startedAt) / 1000).toFixed(1).padStart(6)}s] ${msg}`;
console.log(line);
appendFileSync(LOG, line + "\n");
}
function sleep(ms) {
return new Promise((r) => setTimeout(r, ms));
}
function alive(pid) {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function killGroup(pid, signal = "SIGKILL") {
if (!pid) return;
try {
process.kill(-pid, signal);
} catch {}
try {
process.kill(pid, signal);
} catch {}
}
function cleanup() {
if (cleanedUp) return;
cleanedUp = true;
for (const child of ownedChildren) {
try {
child.kill("SIGKILL");
} catch {}
}
for (const pid of ownedGroups) killGroup(pid);
if (!opts.keep) {
try {
rmSync(ROOT, { recursive: true, force: true });
} catch {}
}
}
process.on("exit", cleanup);
for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
process.on(sig, () => {
cleanup();
process.exit(130);
});
}
function run(cmd, args, { env = process.env, cwd = REPO, input, timeoutMs = 60_000, logFile } = {}) {
return new Promise((resolvePromise) => {
const t0 = Date.now();
const child = spawn(cmd, args, { env, cwd, stdio: ["pipe", "pipe", "pipe"], detached: true });
ownedChildren.add(child);
let stdout = "";
let stderr = "";
let timedOut = false;
child.stdout.on("data", (d) => (stdout += d));
child.stderr.on("data", (d) => (stderr += d));
const timer = setTimeout(() => {
timedOut = true;
killGroup(child.pid);
}, timeoutMs);
child.on("error", (err) => {
stderr += String(err);
});
child.on("close", (code, signal) => {
clearTimeout(timer);
ownedChildren.delete(child);
const result = { code, signal, stdout, stderr, ms: Date.now() - t0, timedOut };
if (logFile) appendFileSync(logFile, `$ ${cmd} ${args.join(" ")}\n${stdout}${stderr}\n[exit ${code ?? signal} in ${result.ms} ms]\n`);
resolvePromise(result);
});
if (input !== undefined) child.stdin.end(input);
else child.stdin.end();
});
}
async function mustRun(label, cmd, args, options) {
const r = await run(cmd, args, options);
if (r.code !== 0) {
throw new Error(`${label} failed (exit ${r.code ?? r.signal}${r.timedOut ? ", timed out" : ""}): ${(r.stderr || r.stdout).slice(-1500)}`);
}
return r;
}
const instanceHomes = new Map();
function secretFor(url) {
for (const [base, location] of instanceHomes) {
if (!url.startsWith(base)) continue;
for (const secretPath of [join(location.dataDir, "secret"), join(location.home, ".agentmemory", "secret")]) {
try {
return readFileSync(secretPath, "utf8").trim();
} catch {}
}
return "";
}
return "";
}
async function request(method, url, body, timeoutMs = 30_000) {
const headers = body === undefined ? { accept: "application/json" } : { "content-type": "application/json", accept: "application/json" };
const secret = secretFor(url);
if (secret) headers.authorization = `Bearer ${secret}`;
const res = await fetch(url, {
method,
headers,
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(timeoutMs),
});
const text = await res.text();
let json = null;
try {
json = JSON.parse(text);
} catch {}
return { status: res.status, json, text };
}
async function free(port) {
const server = net.createServer();
return await new Promise((resolve) => {
server.once("error", () => resolve(false));
server.listen(port, "127.0.0.1", () => server.close(() => resolve(true)));
});
}
async function poll(label, fn, { timeoutMs = 30_000, intervalMs = 250 } = {}) {
const deadline = Date.now() + timeoutMs;
let last;
for (;;) {
try {
const value = await fn();
if (value) return value;
last = "condition not met";
} catch (err) {
if (err && err.fatal) throw err;
last = err instanceof Error ? err.message : String(err);
}
if (Date.now() > deadline) throw new Error(`timed out after ${timeoutMs} ms waiting for ${label} (last: ${String(last).slice(0, 400)})`);
await sleep(intervalMs);
}
}
function assert(cond, message) {
if (!cond) throw new Error(message);
}
function portFree(port) {
return new Promise((resolvePromise) => {
const srv = net.createServer();
srv.once("error", () => resolvePromise(false));
srv.listen(port, "127.0.0.1", () => srv.close(() => resolvePromise(true)));
});
}
function portOpen(port) {
return new Promise((resolvePromise) => {
const sock = net.connect({ port, host: "127.0.0.1" });
sock.setTimeout(1000);
sock.once("connect", () => {
sock.destroy();
resolvePromise(true);
});
sock.once("error", () => resolvePromise(false));
sock.once("timeout", () => {
sock.destroy();
resolvePromise(false);
});
});
}
const reservedBases = new Set();
function instancePorts(base) {
return [base, base + 1, base + 2, base + 3, base + 6353, base + 46023];
}
async function pickBasePort() {
for (let i = 0; i < 200; i++) {
const base = 12000 + Math.floor(Math.random() * 7000);
if ([...reservedBases].some((b) => Math.abs(b - base) < 10)) continue;
const ports = instancePorts(base);
if (ports.some((p) => FORBIDDEN_PORTS.has(p))) continue;
let ok = true;
for (const p of ports) {
if (!(await portFree(p))) {
ok = false;
break;
}
}
if (ok) {
reservedBases.add(base);
return base;
}
}
throw new Error("no free port block found");
}
function vectorFor(text, dims) {
const out = new Array(dims);
let seed = createHash("sha256").update(String(text)).digest();
for (let i = 0; i < dims; i++) {
if (i % 32 === 0 && i > 0) seed = createHash("sha256").update(seed).digest();
out[i] = (seed[i % 32] - 128) / 128;
}
return out;
}
function startFakeEmbeddings(dims) {
const stats = { calls: 0, inputs: 0, chatCalls: 0 };
const server = http.createServer((req, res) => {
let body = "";
req.on("data", (c) => (body += c));
req.on("end", () => {
let payload = {};
try {
payload = JSON.parse(body || "{}");
} catch {}
if (req.url && req.url.endsWith("/embeddings")) {
const inputs = Array.isArray(payload.input) ? payload.input : [payload.input];
stats.calls++;
stats.inputs += inputs.length;
res.writeHead(200, { "content-type": "application/json" });
res.end(
JSON.stringify({
object: "list",
data: inputs.map((t, index) => ({ object: "embedding", index, embedding: vectorFor(t, dims) })),
model: payload.model || "release-gate-fake",
usage: { prompt_tokens: inputs.length, total_tokens: inputs.length },
}),
);
return;
}
stats.chatCalls++;
res.writeHead(500, { "content-type": "application/json" });
res.end(JSON.stringify({ error: { message: "release gate fake provider serves embeddings only" } }));
});
});
return new Promise((resolvePromise) => {
server.listen(0, "127.0.0.1", () => resolvePromise({ server, stats, port: server.address().port }));
});
}
function marker(tag) {
return `gate${tag}${randomBytes(4).toString("hex")}`.toLowerCase();
}
function cleanPath(prefix) {
return [join(prefix, "node_modules", ".bin"), NODE_BIN, "/usr/bin", "/bin", "/usr/sbin", "/sbin"].join(":");
}
class Gate {
constructor() {
this.prefix = join(ROOT, "prefix");
this.pkgDir = join(this.prefix, "node_modules", "@agentmemory", "agentmemory");
this.fake = null;
this.dims = 1536;
this.instances = [];
this.info = {};
}
bin(name) {
return join(this.prefix, "node_modules", ".bin", name);
}
baseEnv(home) {
const env = {
PATH: cleanPath(this.prefix),
HOME: home,
TMPDIR: process.env["TMPDIR"] || tmpdir(),
LANG: "C.UTF-8",
CI: "1",
NO_COLOR: "1",
};
return env;
}
hookCommands(event, toolName) {
const hooksFile = join(this.pkgDir, "plugin", "hooks", "hooks.json");
const config = JSON.parse(readFileSync(hooksFile, "utf-8"));
const entries = config.hooks?.[event] ?? [];
const commands = [];
for (const entry of entries) {
if (entry.matcher && toolName && !new RegExp(`^(?:${entry.matcher})$`).test(toolName)) continue;
if (entry.matcher && !toolName) continue;
for (const h of entry.hooks ?? []) if (h.type === "command") commands.push(h.command);
}
return commands;
}
}
const gate = new Gate();
class Instance {
constructor(name) {
this.name = name;
this.dir = join(ROOT, name);
this.home = join(this.dir, "home");
this.dataDir = join(this.dir, "data");
this.cwd = join(this.dir, "cwd");
this.project = join(this.dir, "projects", "gate-app");
for (const d of [this.home, this.cwd, this.project]) mkdirSync(d, { recursive: true });
this.logFile = join(OUT, `instance-${name}.log`);
this.hookLog = join(OUT, `hooks-${name}.log`);
this.port = 0;
this.cli = null;
this.cliExit = null;
this.enginePid = null;
this.composePid = null;
this.childGroups = new Set();
this.rssSnapshotLast = null;
this.extraEnv = {};
}
get base() {
return `http://127.0.0.1:${this.port}/agentmemory`;
}
env(extra = {}) {
return {
...gate.baseEnv(this.home),
OPENAI_API_KEY: "release-gate-fake-key",
OPENAI_API_KEY_FOR_LLM: "false",
EMBEDDING_PROVIDER: "openai",
OPENAI_BASE_URL: `http://127.0.0.1:${gate.fake.port}`,
OPENAI_EMBEDDING_DIMENSIONS: String(gate.dims),
AGENTMEMORY_URL: `http://127.0.0.1:${this.port}`,
AGENTMEMORY_DATA_DIR: this.dataDir,
...(this.extraEnv.AGENTMEMORY_DISABLE_COMPOSE_RESTART === "1" ? { AGENTMEMORY_DISABLE_COMPOSE_RESTART: "1" } : {}),
...(this.localWorkerBinDir() ? { AGENTMEMORY_WORKER_BIN_DIR: this.localWorkerBinDir() } : {}),
...extra,
};
}
enginePidPath() {
return join(this.home, ".agentmemory", "iii.pid");
}
composeStatePath() {
return join(this.home, ".agentmemory", "engine-state.json");
}
processGroups() {
let rows = [];
try {
rows = execFileSync("ps", ["-eo", "pid=,ppid=,pgid=,command="], { encoding: "utf8" })
.split(/\r?\n/).map((line) => {
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/);
return match ? { pid: Number(match[1]), ppid: Number(match[2]), pgid: Number(match[3]), args: match[4] } : null;
}).filter(Boolean);
} catch { return []; }
const state = (() => { try { return JSON.parse(readFileSync(this.composeStatePath(), "utf8")); } catch { return null; } })();
const owners = [state?.composePid, state?.enginePid, this.composePid, this.cli?.pid].filter((pid) => pid && alive(pid));
if (owners.length === 0) return [];
const descendants = new Set(owners);
let changed = true;
while (changed) {
changed = false;
for (const row of rows) {
if (!descendants.has(row.pid) && descendants.has(row.ppid)) { descendants.add(row.pid); changed = true; }
}
}
const ownGroup = Number(execFileSync("ps", ["-p", String(process.pid), "-o", "pgid="], { encoding: "utf8" }).trim());
const groups = [...new Set(rows.filter((row) => descendants.has(row.pid) && row.pgid !== ownGroup).map((row) => row.pgid))];
return groups;
}
localWorkerBinDir() {
return opts.workerBinDir || process.env.AGENTMEMORY_WORKER_BIN_DIR || null;
}
seedEngine() {
const version = gate.info.enginePin;
const cached = join(CACHE, `iii-${version}-${platform()}-${arch()}`);
const target = join(this.home, ".agentmemory", "bin", "iii");
if (existsSync(cached) && !existsSync(target)) {
mkdirSync(dirname(target), { recursive: true });
copyFileSync(cached, target);
chmodSync(target, 0o755);
}
}
cacheEngine() {
const version = gate.info.enginePin;
const source = join(this.home, ".agentmemory", "bin", "iii");
const cached = join(CACHE, `iii-${version}-${platform()}-${arch()}`);
if (existsSync(source) && !existsSync(cached)) {
try {
mkdirSync(CACHE, { recursive: true });
copyFileSync(source, cached);
chmodSync(cached, 0o755);
} catch {}
}
}
async start(extraEnv = {}) {
if (!this.port) this.port = await pickBasePort();
instanceHomes.set(`http://127.0.0.1:${this.port}/`, { home: this.home, dataDir: this.dataDir });
this.extraEnv = { ...(this.localWorkerBinDir() ? { AGENTMEMORY_WORKER_BIN_DIR: this.localWorkerBinDir() } : {}), ...extraEnv };
this.seedEngine();
for (const p of instancePorts(this.port)) {
await poll(`port ${p} to be free`, async () => !(await portOpen(p)), { timeoutMs: 30_000 });
}
const fd = openSync(this.logFile, "a");
appendFileSync(this.logFile, `\n==== start ${new Date().toISOString()} env=${JSON.stringify(extraEnv)}\n`);
const t0 = Date.now();
const child = spawn(gate.bin("agentmemory"), ["--port", String(this.port)], {
cwd: this.cwd,
env: this.env(extraEnv),
stdio: ["ignore", fd, fd],
detached: true,
});
closeSync(fd);
this.cli = child;
this.cliExit = null;
ownedGroups.add(child.pid);
child.on("exit", (code, signal) => {
this.cliExit = { code, signal };
});
await poll(
`${this.name} to answer /livez`,
async () => {
if (this.cliExit && this.cliExit.code !== 0) {
const err = new Error(`agentmemory exited (${this.cliExit.code ?? this.cliExit.signal}) before it was ready; see ${this.logFile}`);
err.fatal = true;
throw err;
}
const r = await request("GET", `${this.base}/livez`, undefined, 2000);
return r.status === 200;
},
{ timeoutMs: 420_000, intervalMs: 300 },
);
await poll(`${this.name} health`, async () => (await request("GET", `${this.base}/health`, undefined, 5000)).status === 200, { timeoutMs: 60_000 });
try {
const state = JSON.parse(readFileSync(join(this.home, ".agentmemory", "engine-state.json"), "utf8"));
if (state.kind === "compose") {
this.composePid = state.composePid ?? null;
this.enginePid = state.enginePid ?? null;
} else {
this.enginePid = parseInt(readFileSync(this.enginePidPath(), "utf-8").trim(), 10) || null;
}
} catch {
try { this.enginePid = parseInt(readFileSync(this.enginePidPath(), "utf-8").trim(), 10) || null; } catch {}
}
if (this.enginePid) ownedGroups.add(this.enginePid);
if (this.composePid) ownedGroups.add(this.composePid);
if (this.enginePid) ownedGroups.add(this.enginePid);
this.cacheEngine();
const ms = Date.now() - t0;
this.rssSnapshotLast = this.rssSnapshot();
log(`${this.name}: up on :${this.port} in ${ms} ms (worker pid ${child.pid}, engine pid ${this.enginePid})`);
log(`${this.name} startup RSS: ${JSON.stringify(this.rssSnapshotLast)}`);
await this.refreshContainerRssSnapshot();
return ms;
}
async waitDown() {
try {
const state = JSON.parse(readFileSync(this.composeStatePath(), "utf8"));
this.composePid = state.kind === "compose" ? state.composePid ?? this.composePid : this.composePid;
this.enginePid = state.kind === "compose" ? state.enginePid ?? this.enginePid : this.enginePid;
} catch {}
await poll(`${this.name} worker CLI to exit`, async () => this.cliExit !== null || !alive(this.cli.pid), { timeoutMs: 30_000 });
if (this.composePid) await poll(`${this.name} Compose daemon to exit`, async () => !alive(this.composePid), { timeoutMs: 30_000 });
if (this.enginePid) await poll(`${this.name} engine to exit`, async () => !alive(this.enginePid), { timeoutMs: 30_000 });
for (const port of instancePorts(this.port)) {
const label = ["REST", "viewer", "WebSocket", "stream", "metrics", "derived"][instancePorts(this.port).indexOf(port)] ?? "listener";
await poll(`${this.name} ${label} port ${port} listener to close`, async () => free(port), { timeoutMs: 30_000 });
}
}
async forceKill() {
// Capture direct Compose child process groups before killing the daemon;
// after SIGKILL they are reparented and cannot be recovered from its tree.
for (const pgid of this.processGroups()) this.childGroups.add(pgid);
for (const pgid of this.childGroups) killGroup(pgid);
if (this.enginePid) killGroup(this.enginePid);
if (this.composePid) killGroup(this.composePid);
killGroup(this.cli?.pid);
// Reap stragglers only within process groups proven to have been spawned by
// this Compose daemon. No PID is selected by the shared port number.
for (const pgid of this.childGroups) {
for (let attempt = 0; attempt < 20; attempt++) {
let exists = false;
try { process.kill(-pgid, 0); exists = true; } catch {}
if (!exists) break;
await sleep(100);
}
try { process.kill(-pgid, "SIGKILL"); } catch {}
}
await this.waitDown();
log(`${this.name}: force killed Compose, engine, and workers`);
}
async crashWorker() {
const worker = this.pid("worker");
assert(worker, "could not resolve the Compose-owned agentmemory worker PID from its state.json record");
try { process.kill(worker, "SIGKILL"); } catch {}
await poll("Compose to record the agentmemory worker failure", async () => {
const statePath = join(this.dataDir, "compose-state", `agentmemory-${this.port}`, "state.json");
try {
const composeState = JSON.parse(readFileSync(statePath, "utf8"));
return composeState.containers?.agentmemory?.status === "failed";
} catch { return false; }
}, { timeoutMs: 5_000, intervalMs: 100 });
}
async crashCompose() {
await this.forceKill();
}
configurationInfo() {
try {
const composeFile = join(this.dataDir, "compose", "worker-compose.yaml");
const lockFile = join(this.dataDir, "compose", "worker-compose.lock");
const yaml = readFileSync(composeFile, "utf8");
const lock = readFileSync(lockFile, "utf8");
const engineState = JSON.parse(readFileSync(this.composeStatePath(), "utf8"));
const engineConfigFiles = [];
const lockFiles = [];
const walk = (dir) => { for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
if (entry.isDirectory()) walk(path);
else if (entry.name === "engine-config.yaml") engineConfigFiles.push(path);
else if (entry.name.endsWith(".yaml") && path.includes("/packages/")) lockFiles.push(path);
} };
walk(join(this.dataDir, "compose-state"));
const engineConfigText = engineConfigFiles.map((path) => readFileSync(path, "utf8")).join("\n");
const engineConfig = engineConfigText.match(/name: configuration[\s\S]*?directory: ([^\n]+)/)?.[1]?.trim() ?? null;
const packages = Object.fromEntries(
[...lock.matchAll(/^ ([A-Za-z0-9._-]+):\n worker: [^\n]+\n requested: ([^\n]+)\n resolved:\n name: [^\n]+\n version: ([^\n]+)/gm)]
.map((match) => [match[1], { requested: match[2], version: match[3] }]),
);
const configResolve = lockFiles.map((path) => {
const text = readFileSync(path, "utf8");
const packageName = basename(dirname(path));
return { packageName, path, matchesConfiguration: /configuration/.test(text) };
});
const metadata = {
packageManifests: { state: "configuration@latest", http: "configuration@latest" },
registryLatestMetadataVersion: "0.24.5",
explicitComposeDependencyContainers: [...yaml.matchAll(/^ dependencies:\s*\n configuration: ([^\n]+)/gm)].map((match) => match[1]),
resolvedPackages: packages,
packageManifestsWithConfigurationDependencies: Object.keys(packages).filter((name) => ["state", "http"].includes(name)),
engineWorkerConfiguration: engineConfig ? { adapter: "fs", configDirectory: engineConfig, configured: true } : { adapter: "fs", configDirectory: null, configured: engineState.kind === "compose" },
dependencyKind: "engine worker, not a process/container",
registryBinaryDownloaded: false,
childProcessStarted: false,
required: true,
configurationRss: "No separate PID/RSS exists; configuration is loaded in-process inside iii-engine. Its RSS is the iii-engine entry in startupRss.",
pinSupported: "No per-dependency exact version selector is supported for engine-kind graph nodes in this Compose runtime. The active implementation is the engine-bundled configuration worker in the pinned iii 0.25.0, not the independent configuration@latest registry binary.",
explanation: "The published state/http manifests declare configuration: latest. The registry resolver returns dependency graph nodes in the same response; iii-compose::expand_graph omits nodes with kind=engine because they are supplied by the engine. Their dependency is satisfied by engine.workers.configuration (fs adapter), which Compose includes in engine-config.yaml. It is required by workers that use typed configuration, but is not downloaded/started as a child by Compose. Public registry info for configuration@latest currently reports 0.24.5; that is separate from the built-in configuration implementation in iii 0.25.0. It has no separate child RSS. No exact per-package dependency pin is available in this runtime; changing the engine pin is the control over the bundled implementation.",
};
writeFileSync(join(OUT, `configuration-${this.name}.json`), JSON.stringify(metadata, null, 2) + "\n");
return metadata;
} catch (error) {
return { error: String(error) };
}
}
async refreshContainerRssSnapshot() {
try {
const statePath = this.stateStoreFile();
const composeState = JSON.parse(readFileSync(statePath, "utf8"));
const port = this.port;
const rows = execFileSync("ps", ["-A", "-o", "pid=,rss=,command="], { encoding: "utf8" }).split(/\r?\n/);
const components = {};
for (const [name, record] of Object.entries(composeState.containers ?? {})) {
const pid = record.pid;
if (!pid || !alive(pid)) continue;
let processPid = pid;
let processArgs = "";
if (name === "agentmemory") {
const children = execFileSync("ps", ["-A", "-o", "pid=,ppid=,rss=,command="], { encoding: "utf8" }).trim();
const child = children.split(/\r?\n/).map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/)).find((match) => match && Number(match[2]) === Number(pid) && /node.*dist\/index\.mjs/.test(match[4]));
if (child) {
processPid = Number(child[1]);
processArgs = child[4];
}
}
const row = rows.map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(.*)$/)).find((match) => match && Number(match[1]) === Number(processPid));
if (!row) continue;
const rssKb = Number(row[2]);
components[name] = { pid: processPid, containerPid: pid, rssKb, rssMiB: Math.round(rssKb / 1024 * 10) / 10, args: processArgs || row[3] };
}
this.rssSnapshotLast = { ...this.rssSnapshotLast, ...components };
log(`${this.name} container RSS: ${JSON.stringify(components)} (project state ${statePath}; instance port ${port})`);
return this.rssSnapshotLast;
} catch (error) {
this.rssSnapshotLast = { ...this.rssSnapshotLast, error: String(error) };
return this.rssSnapshotLast;
}
}
stateStoreFile() {
const root = join(this.dataDir, "compose-state");
const files = [];
const walk = (dir) => {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
if (entry.isDirectory()) walk(path);
else if (entry.name === "state.json") files.push(path);
}
};
walk(root);
const instanceSuffix = join(`agentmemory-${this.port}`, "state.json");
const instanceState = files.find((file) => file.endsWith(instanceSuffix));
if (instanceState) return instanceState;
for (const file of files) {
const state = JSON.parse(readFileSync(file, "utf8"));
const composePath = state.compose_path ?? state.composePath ?? state.file;
if (composePath && resolve(composePath) === resolve(join(this.dataDir, "compose", "worker-compose.yaml"))) return file;
}
throw new Error(`Compose state.json for ${this.name} was not found under ${root}`);
}
rssSnapshot() {
try {
const composeState = JSON.parse(readFileSync(this.composeStatePath(), "utf8"));
const entries = readdirSync(join(this.dataDir, "compose-state"), { withFileTypes: true });
const statePath = entries.filter((entry) => entry.isDirectory()).map((entry) => join(this.dataDir, "compose-state", entry.name, `agentmemory-${this.port}`, "state.json")).find(existsSync);
const projectState = statePath ? JSON.parse(readFileSync(statePath, "utf8")) : null;
const components = {
"iii-compose": composeState.composePid,
"iii-engine": composeState.enginePid,
state: projectState?.containers?.state?.pid,
http: projectState?.containers?.http?.pid,
agentmemory: projectState?.containers?.agentmemory?.pid,
};
const result = {};
for (const [name, containerPid] of Object.entries(components)) {
if (!containerPid || !alive(containerPid)) continue;
let pid = containerPid;
let processArgs = "";
if (name === "agentmemory") {
const children = execFileSync("ps", ["-A", "-o", "pid=,ppid=,rss=,command="], { encoding: "utf8" }).trim();
const child = children.split(/\r?\n/).map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/)).find((match) => match && Number(match[2]) === Number(pid) && /node.*dist\/index\.mjs/.test(match[4]));
if (child) {
pid = Number(child[1]);
processArgs = child[4];
}
}
const fields = execFileSync("ps", ["-p", String(pid), "-o", "rss=,command="], { encoding: "utf8" }).trim().match(/^(\d+)\s+(.*)$/);
if (!fields) continue;
const rssKb = Number(fields[1]);
result[name] = { pid, containerPid, rssKb, rssMiB: Math.round(rssKb / 1024 * 10) / 10, args: processArgs || fields[2] || "" };
}
return result;
} catch (error) {
return { error: String(error) };
}
}
workerPid() {
try {
const files = [];
const stateRoot = join(this.dataDir, "compose-state");
const walk = (dir) => { for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
if (entry.isDirectory()) walk(path);
else if (entry.name === "state.json") files.push(path);
} };
walk(stateRoot);
const instanceState = files.map((path) => ({ path, state: JSON.parse(readFileSync(path, "utf8")) }))
.find(({ path }) => path.endsWith(join(`agentmemory-${this.port}`, "state.json")));
if (instanceState) return instanceState.state.containers?.agentmemory?.pid ?? null;
for (const path of files) {
const state = JSON.parse(readFileSync(path, "utf8"));
const composePath = state.compose_path ?? state.composePath ?? state.file;
if (composePath && resolve(composePath) === resolve(join(this.dataDir, "compose", "worker-compose.yaml"))) return state.containers?.agentmemory?.pid ?? null;
}
} catch {}
return null;
}
pid(name) {
if (name === "worker") return this.workerPid();
return null;
}
async stop() {
const t0 = Date.now();
const r = await run(gate.bin("agentmemory"), ["stop", "--port", String(this.port)], {
env: this.env(this.extraEnv),
cwd: this.cwd,
timeoutMs: 90_000,
logFile: this.logFile,
});
if (r.code !== 0) throw new Error(`agentmemory stop exited ${r.code ?? r.signal}: ${(r.stdout + r.stderr).slice(-800)}`);
await this.waitDown();
log(`${this.name}: stopped cleanly in ${Date.now() - t0} ms`);
return Date.now() - t0;
}
composeActive() {
return [this.composePid, this.enginePid].some((pid) => pid && alive(pid));
}
async shutdown() {
// The CLI can exit 0 while the detached Compose daemon keeps running.
if (!this.cli || (this.cliExit && !this.composeActive())) return;
try {
await this.stop();
} catch (err) {
log(`${this.name}: clean stop failed (${err.message}); killing`);
await this.forceKill().catch(() => {});
}
}
async hook(event, payload) {
const commands = gate.hookCommands(event, payload.tool_name);
assert(commands.length > 0, `no bundled ${event} hook matches ${payload.tool_name ?? "this event"}`);
const results = [];
for (const command of commands) {
const r = await run("sh", ["-c", command], {
env: {
...gate.baseEnv(this.home),
CLAUDE_PLUGIN_ROOT: join(gate.pkgDir, "plugin"),
CLAUDE_PROJECT_DIR: this.project,
AGENTMEMORY_URL: `http://127.0.0.1:${this.port}`,
AGENTMEMORY_DATA_DIR: this.dataDir,
},
cwd: this.project,
input: JSON.stringify({ hook_event_name: event, cwd: this.project, transcript_path: join(this.dir, "transcript.jsonl"), ...payload }),
timeoutMs: 20_000,
logFile: this.hookLog,
});
assert(r.code === 0, `${event} hook exited ${r.code ?? r.signal}${r.timedOut ? " (timed out)" : ""}: ${r.stderr.slice(-400)}`);
results.push(r);
}
return { ms: results.reduce((a, r) => a + r.ms, 0) };
}
toolEvent(sessionId, mark, extra = {}) {
return {
session_id: sessionId,
tool_name: "Bash",
tool_input: { command: `npm test -- ${mark}` },
tool_response: { stdout: `ran suite ${mark}: 12 passed`, stderr: "", interrupted: false },
tool_use_id: `toolu_${randomBytes(8).toString("hex")}`,
...extra,
};
}
async observations(sessionId) {
const r = await request("GET", `${this.base}/observations?sessionId=${encodeURIComponent(sessionId)}`);
assert(r.status === 200, `GET /observations returned ${r.status}`);
return r.json.observations ?? [];
}
async countMarker(sessionId, mark) {
const list = await this.observations(sessionId);
return list.filter((o) => JSON.stringify(o).includes(mark)).length;
}
async searchFinds(mark) {
const r = await request("POST", `${this.base}/search`, { query: mark, limit: 10 });
return r.status === 200 && r.text.includes(mark);
}
async status() {
const r = await request("GET", `${this.base}/status`, undefined, 30_000);
assert(r.status === 200, `GET /status returned ${r.status}`);
return r.json;
}
async capture(query = "") {
const r = await request("GET", `${this.base}/capture${query}`);
assert(r.status === 200, `GET /capture returned ${r.status}`);
return r.json;
}
async totals() {
const sessions = (await request("GET", `${this.base}/sessions`)).json?.sessions ?? [];
let observations = 0;
for (const s of sessions) observations += (await this.observations(s.id)).length;
const memories = ((await request("GET", `${this.base}/memories`)).json?.memories ?? []).length;
const st = await this.status();
return {
sessions: sessions.length,
observations,
memories,
bm25Documents: st.index?.bm25Documents ?? null,
vectorDocuments: st.index?.vectorDocuments ?? null,
};
}
}
async function mcpSession(command, args, env) {
const child = spawn(command, args, { env, stdio: ["pipe", "pipe", "pipe"], detached: true });
ownedChildren.add(child);
ownedGroups.add(child.pid);
let buf = "";
let stderr = "";
const pending = new Map();
child.stderr.on("data", (d) => (stderr += d));
child.stdout.on("data", (d) => {
buf += d;
let i;
while ((i = buf.indexOf("\n")) >= 0) {
const line = buf.slice(0, i).trim();
buf = buf.slice(i + 1);
if (!line) continue;
let msg;
try {
msg = JSON.parse(line);
} catch {
continue;
}
if (msg.id !== undefined && pending.has(msg.id)) {
pending.get(msg.id)(msg);
pending.delete(msg.id);
}
}
});
let nextId = 1;
const rpc = (method, params) => {
const id = nextId++;
child.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
return new Promise((resolvePromise, reject) => {
const timer = setTimeout(() => reject(new Error(`MCP ${method} timed out; stderr: ${stderr.slice(-400)}`)), 30_000);
pending.set(id, (msg) => {
clearTimeout(timer);
resolvePromise(msg);
});
});
};
const init = await rpc("initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "release-gate", version: "1" } });
child.stdin.write(JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) + "\n");
const close = () => {
try {
child.kill("SIGTERM");
} catch {}
ownedChildren.delete(child);
};
const call = async (name, argsObj) => {
const r = await rpc("tools/call", { name, arguments: argsObj });
const text = (r.result?.content ?? []).map((c) => c.text ?? "").join("\n");
return { isError: Boolean(r.error || r.result?.isError), text, error: r.error };
};
return { init, rpc, call, close, stderr: () => stderr };
}
const results = [];
async function scenario(name, fn) {
const meta = SCENARIOS.find((s) => s[0] === name);
if (opts.only && name !== "install" && !opts.only.has(name)) {
results.push({ name, title: meta[1], status: "skipped" });
return;
}
log(`---- ${name}: ${meta[1]}`);
const t0 = Date.now();
const details = {};
try {
await fn(details);
const ms = Date.now() - t0;
results.push({ name, title: meta[1], status: "pass", ms, details });
log(`PASS ${name} (${ms} ms)`);
} catch (err) {
const ms = Date.now() - t0;
const message = err instanceof Error ? err.message : String(err);
results.push({ name, title: meta[1], status: "fail", ms, error: message, details });
log(`FAIL ${name} (${ms} ms): ${message}`);
}
}
async function ensureUp(inst, extraEnv = {}) {
if (inst.cli && (await request("GET", `${inst.base}/livez`, undefined, 2000).then((r) => r.status === 200).catch(() => false))) return;
if (inst.cli && (!inst.cliExit || inst.composeActive())) await inst.forceKill().catch(() => {});
await inst.start(extraEnv);
}
async function buildAndPack() {
const packDir = join(ROOT, "pack");
mkdirSync(packDir, { recursive: true });
let tarball = opts.tarball;
let mcpTarball = opts.mcpTarball;
if (!tarball) {
if (!opts.skipBuild) {
log("building (npm run build)");
await mustRun("npm run build", "npm", ["run", "build"], { timeoutMs: 600_000, logFile: join(OUT, "build.log") });
}
log("packing @agentmemory/agentmemory");
const r = await mustRun("npm pack", "npm", ["pack", "--json", "--pack-destination", packDir], { timeoutMs: 300_000 });
tarball = join(packDir, JSON.parse(r.stdout)[0].filename);
}
if (!mcpTarball) {
const r = await mustRun("npm pack (mcp)", "npm", ["pack", "--json", "--pack-destination", packDir], { cwd: join(REPO, "packages", "mcp"), timeoutMs: 120_000 });
mcpTarball = join(packDir, JSON.parse(r.stdout)[0].filename);
}
return { tarball, mcpTarball };
}
async function install(details) {
const { tarball, mcpTarball } = await buildAndPack();
const sha = createHash("sha256").update(readFileSync(tarball)).digest("hex");
details.tarball = tarball;
details.tarballSha256 = sha;
details.tarballBytes = statSync(tarball).size;
mkdirSync(gate.prefix, { recursive: true });
writeFileSync(join(gate.prefix, "package.json"), JSON.stringify({ name: "agentmemory-release-gate-prefix", private: true }) + "\n");
const npmCache = execFileSync("npm", ["config", "get", "cache"], { encoding: "utf-8" }).trim();
const installHome = join(ROOT, "install-home");
mkdirSync(installHome, { recursive: true });
log(`installing ${tarball} into a clean prefix`);
const t0 = Date.now();
await mustRun(
"npm install of the packed tarballs",
"npm",
["install", "--no-audit", "--no-fund", "--no-package-lock", tarball, mcpTarball],
{
cwd: gate.prefix,
env: { ...gate.baseEnv(installHome), PATH: [NODE_BIN, "/usr/bin", "/bin", "/usr/sbin", "/sbin"].join(":"), npm_config_cache: npmCache },
timeoutMs: 600_000,
logFile: join(OUT, "install.log"),
},
);
details.installMs = Date.now() - t0;
const ls = await run("npm", ["ls", "--all", "--json"], { cwd: gate.prefix, env: { ...gate.baseEnv(installHome), npm_config_cache: npmCache }, timeoutMs: 120_000 });
writeFileSync(join(OUT, "resolved-dependencies.json"), ls.stdout);
const repoPkg = JSON.parse(readFileSync(join(REPO, "package.json"), "utf-8"));
const installed = JSON.parse(readFileSync(join(gate.pkgDir, "package.json"), "utf-8"));
assert(installed.name === "@agentmemory/agentmemory", `installed package name is ${installed.name}`);
details.version = installed.version;
if (!opts.tarball) assert(installed.version === repoPkg.version, `installed version ${installed.version} != repo ${repoPkg.version}`);
gate.info.enginePin = installed.dependencies["iii-sdk"];
gate.info.version = installed.version;
details.enginePin = gate.info.enginePin;
const cliPath = execFileSync("sh", ["-c", "command -v agentmemory"], { env: { PATH: cleanPath(gate.prefix) }, encoding: "utf-8" }).trim();
assert(cliPath === gate.bin("agentmemory"), `agentmemory on PATH resolves to ${cliPath}, not the installed prefix`);
const v = await mustRun("agentmemory --version", gate.bin("agentmemory"), ["--version"], { env: gate.baseEnv(installHome), cwd: installHome });
assert(v.stdout.trim() === installed.version, `agentmemory --version printed ${v.stdout.trim()}, expected ${installed.version}`);
for (const f of ["dist/cli.mjs", "dist/standalone.mjs", "dist/index.mjs", "dist/viewer/index.html", "plugin/hooks/hooks.json", "plugin/scripts/post-tool-use.mjs", "plugin/scripts/_capture.mjs"]) {
assert(existsSync(join(gate.pkgDir, f)), `packed artifact is missing ${f}`);
}
assert(existsSync(gate.bin("agentmemory-mcp")), "agentmemory-mcp bin is missing from the MCP shim package");
const a = instA;
details.firstBootMs = await a.start();
details.startupRss = a.rssSnapshotLast;
details.configurationDependency = a.configurationInfo();
const st = await a.status();
details.reportedVersion = st.service?.version;
details.reportedEnginePinned = st.service?.enginePinnedVersion;
assert(st.service?.version === installed.version, `/status reports version ${st.service?.version}, expected ${installed.version}`);
assert(String(st.service?.enginePinnedVersion ?? "").includes(gate.info.enginePin), `/status reports pinned engine ${st.service?.enginePinnedVersion}, expected ${gate.info.enginePin}`);
assert(st.provider?.embeddings && st.provider.embeddings !== "none", `embedding provider is ${st.provider?.embeddings}`);
details.embeddings = st.provider?.embeddings;
}
const instA = new Instance("main");
const instB = new Instance("vectors");
const instC = new Instance("restore");
const markers = {};
async function main() {
log(`release gate output: ${OUT}`);
log(`temp root: ${ROOT}`);
gate.fake = await startFakeEmbeddings(gate.dims);
if (opts.workerBinDir) {
assert(existsSync(resolve(opts.workerBinDir)), `--worker-bin-dir does not exist: ${opts.workerBinDir}`);
log(`local workers override: ${opts.workerBinDir}`);
}
log(`fake embedding provider on :${gate.fake.port}`);
await scenario("install", install);
if (!results[0] || results[0].status !== "pass") {
for (const [name, title] of SCENARIOS.slice(1)) results.push({ name, title, status: "fail", error: "install failed" });
return;
}
await scenario("capture", async (d) => {
await ensureUp(instA);
const sid = `gate-capture-${randomBytes(4).toString("hex")}`;
markers.capture = [marker("amber"), marker("birch"), marker("cedar")];
markers.prompt = marker("prompt");
markers.reply = marker("reply");
d.sessionId = sid;
const timings = [];
timings.push((await instA.hook("SessionStart", { session_id: sid, source: "startup" })).ms);
timings.push((await instA.hook("UserPromptSubmit", { session_id: sid, prompt: `Investigate the flaky parser test ${markers.prompt}` })).ms);
for (const m of markers.capture) timings.push((await instA.hook("PostToolUse", instA.toolEvent(sid, m))).ms);
timings.push((await instA.hook("Stop", { session_id: sid, stop_hook_active: false, last_assistant_message: `The parser flake came from CRLF handling ${markers.reply} in the tokenizer` })).ms);
d.hookMs = timings;
for (const m of [...markers.capture, markers.reply]) {
await poll(`search to find ${m}`, () => instA.searchFinds(m), { timeoutMs: 20_000 });
const n = await instA.countMarker(sid, m);
assert(n === 1, `marker ${m} is stored in ${n} observations, expected 1`);
}
const all = JSON.stringify(await instA.observations(sid));
assert(all.includes(markers.prompt), "the prompt marker is not in any stored observation");
d.observations = (await instA.observations(sid)).length;
});
await scenario("mcp", async (d) => {
await ensureUp(instA);
const env = {
...gate.baseEnv(instA.home),
AGENTMEMORY_URL: `http://127.0.0.1:${instA.port}`,
AGENTMEMORY_DATA_DIR: instA.dataDir,
AGENTMEMORY_SECRET: readFileSync(join(instA.dataDir, "secret"), "utf8").trim(),
};
const shim = await mcpSession(gate.bin("agentmemory-mcp"), [], env);
try {
const tools = await shim.rpc("tools/list", {});
const names = (tools.result?.tools ?? []).map((t) => t.name);
d.shimTools = names.length;
for (const t of ["memory_save", "memory_smart_search"]) assert(names.includes(t), `agentmemory-mcp does not list ${t}`);
assert(!shim.stderr().includes("LOCAL FALLBACK"), "agentmemory-mcp fell back to local mode instead of using the server");
const m = marker("mcp");
markers.mcp = m;
const saved = await shim.call("memory_save", { content: `Release gate MCP fact ${m}: the parser handles CRLF line endings`, type: "fact" });
assert(!saved.isError, `memory_save failed: ${saved.text || JSON.stringify(saved.error)}`);
await poll(`memory_smart_search to find ${m}`, async () => {
const r = await shim.call("memory_smart_search", { query: m, limit: 5 });
return !r.isError && r.text.includes(m);
}, { timeoutMs: 20_000, intervalMs: 500 });
const search = await poll(`server smart-search route to find ${m}`, async () => {
const result = await request("POST", `${instA.base}/smart-search`, { query: m, limit: 5 });
return result.status === 200 && result.text.includes(m) ? result : null;
}, { timeoutMs: 20_000, intervalMs: 500 });
d.mcpSearchStatus = search.status;
} finally {
shim.close();
}
const cli = await mcpSession(gate.bin("agentmemory"), ["mcp"], env);
try {
const tools = await cli.rpc("tools/list", {});
d.cliTools = (tools.result?.tools ?? []).length;
assert(d.cliTools === d.shimTools, `agentmemory mcp lists ${d.cliTools} tools, the shim lists ${d.shimTools}`);
} finally {
cli.close();
}
});
await scenario("offline", async (d) => {
await ensureUp(instA);
await instA.stop();
const sid = `gate-offline-${randomBytes(4).toString("hex")}`;
markers.offline = [marker("cobalt"), marker("indigo"), marker("violet")];
markers.offlineReply = marker("lapis");
const recovered = [...markers.offline, markers.offlineReply];
const hookMs = [];
for (const m of markers.offline) hookMs.push((await instA.hook("PostToolUse", instA.toolEvent(sid, m))).ms);
hookMs.push((await instA.hook("Stop", { session_id: sid, stop_hook_active: false, last_assistant_message: `Offline turn ended with ${markers.offlineReply} pending` })).ms);
d.hookMs = hookMs;
assert(Math.max(...hookMs) < 5000, `a hook took ${Math.max(...hookMs)} ms while the service was down`);
const cap = await mustRun("agentmemory capture --json", gate.bin("agentmemory"), ["capture", "--json"], { env: instA.env(), cwd: instA.cwd, timeoutMs: 30_000 });
const spool = JSON.parse(cap.stdout.slice(cap.stdout.indexOf("{"))).spool;
d.spooledWhileDown = spool.records;
assert(spool.records >= recovered.length, `spool holds ${spool.records} records, expected ${recovered.length}`);
d.restartMs = await instA.start();
for (const m of recovered) {
await poll(`recovered observation ${m}`, async () => (await instA.countMarker(sid, m)) >= 1, { timeoutMs: 30_000 });
await poll(`search to find ${m}`, () => instA.searchFinds(m), { timeoutMs: 20_000 });
}
for (const m of recovered) {
const n = await instA.countMarker(sid, m);
assert(n === 1, `offline marker ${m} is stored ${n} times, expected 1`);
}
const after = await instA.capture();
const left = (after.capture?.spool ?? []).reduce((a, s) => a + s.records, 0);
d.spoolAfterRestart = left;
assert(left === 0, `${left} records are still in the spool after restart`);
});
await scenario("dedup", async (d) => {
await ensureUp(instA);
const sid = `gate-dedup-${randomBytes(4).toString("hex")}`;
const m = marker("garnet");
const event = instA.toolEvent(sid, m);
await instA.hook("PostToolUse", event);
await poll(`first delivery of ${m}`, async () => (await instA.countMarker(sid, m)) === 1, { timeoutMs: 20_000 });
await sleep(STATE_FLUSH_WAIT_MS);
// This scenario proves cross-restart idempotency, so kill the whole
// Compose-owned process tree (app, Rust workers, and managed engine), then
// start the same persisted data on the same ports.
await instA.forceKill();
d.restartMs = await instA.start();
const before = (await instA.capture()).capture?.sinceStart?.duplicates ?? 0;
await instA.hook("PostToolUse", event);
await instA.hook("PostToolUse", event);
const dup = await poll("replays to be answered as duplicates", async () => {
const n = (await instA.capture()).capture?.sinceStart?.duplicates ?? 0;
return n - before >= 2 ? n - before : 0;
}, { timeoutMs: 20_000 });
d.duplicatesAnswered = dup;
assert((await instA.countMarker(sid, m)) === 1, `replayed event is stored ${await instA.countMarker(sid, m)} times, expected 1`);
await instA.hook("PostToolUse", { ...event, tool_use_id: `toolu_${randomBytes(8).toString("hex")}` });
await poll("a distinct event with identical content to be stored", async () => (await instA.countMarker(sid, m)) === 2, { timeoutMs: 20_000 });
});
await scenario("deadletter", async (d) => {
// Agentmemory capture inbox/dead-letter recovery is independent of queue workers.
await ensureUp(instA);
await instA.stop();
await instA.start({ MAX_OBS_PER_SESSION: "2" });
const sid = `gate-dead-${randomBytes(4).toString("hex")}`;
const ms = [marker("onyx"), marker("opal"), marker("quartz")];
for (const m of ms) await instA.hook("PostToolUse", instA.toolEvent(sid, m));
const dead = await poll("a dead letter for the failed capture", async () => {
const c = await instA.capture("?status=dead");
return (c.items ?? []).find((it) => String(it.preview ?? "").includes(ms[2]) && it.sessionId === sid);
}, { timeoutMs: 20_000 });
d.deadLetterError = dead.lastError;
assert((await instA.countMarker(sid, ms[2])) === 0, "the failed capture was stored anyway");
const st = await instA.status();
assert((st.problems ?? []).some((p) => p.code === "capture-dead-letters" && p.fix), "/status does not report the dead letter with a fix");
await sleep(STATE_FLUSH_WAIT_MS);
await instA.forceKill();
await instA.start({});
const still = await instA.capture("?status=dead");
assert((still.items ?? []).some((it) => it.eventId === dead.eventId), "the dead letter did not survive the restart");
const retry = await request("POST", `${instA.base}/capture/retry`, { eventId: dead.eventId });
assert(retry.status === 200, `POST /capture/retry returned ${retry.status}`);
d.retry = retry.json;
assert(retry.json?.recovered === 1, `retry recovered ${retry.json?.recovered}, expected 1`);
await poll(`recovered capture ${ms[2]}`, async () => (await instA.countMarker(sid, ms[2])) === 1, { timeoutMs: 20_000 });
await poll(`search to find ${ms[2]}`, () => instA.searchFinds(ms[2]), { timeoutMs: 20_000 });
const after = await instA.capture();
assert((after.capture?.inbox?.dead ?? 0) === 0, `${after.capture?.inbox?.dead} dead letters remain`);
const obs = await instA.observations(sid);
assert(obs.some((o) => o.id === dead.observationId), "the recovered observation does not keep the id assigned at acceptance");
});
await scenario("vectors", async (d) => {
await ensureUp(instB);
const sid = `gate-vectors-${randomBytes(4).toString("hex")}`;
const count = 20;
await instB.hook("PostToolUse", instB.toolEvent(sid, marker("vec0x")));
const t0 = Date.now();
const st = await poll("the first vector checkpoint", async () => {
const s = await instB.status();
const p = s.indexPersistence;
return s.index?.vectorDocuments === 1 && p?.vector?.lastSavedAt && !p.firstCheckpointPending && !p.saving && p.pendingChanges === 0 ? s : null;
}, { timeoutMs: 60_000, intervalMs: 500 });
d.firstCheckpointWaitMs = Date.now() - t0;
d.saveIntervalMs = st.indexPersistence.saveIntervalMs;
d.checkpointVectors = st.index.vectorDocuments;
const checkpointId = (await instB.observations(sid))[0]?.id;
assert(checkpointId, "the checkpoint observation is missing");
assert(d.saveIntervalMs > d.firstCheckpointWaitMs, "the checkpoint only happened at the regular save interval, so the first-checkpoint boundary was not exercised");
for (let i = 1; i < count; i++) await instB.hook("PostToolUse", instB.toolEvent(sid, marker(`vec${i}x`)));
const before = await poll("every pending vector write to be acknowledged", async () => {
const s = await instB.status();
return isVectorLogAcknowledged(s, count, d.checkpointVectors) ? s : null;
}, { timeoutMs: 60_000, intervalMs: 500 });
const vectorsBefore = before.index.vectorDocuments;
d.vectorsBefore = vectorsBefore;
d.pendingLogBeforeCrash = before.indexPersistence.pendingLog;
d.pendingLogAcknowledged = before.indexPersistence.pendingLogAcknowledged;
const pendingIds = (await instB.observations(sid)).map((observation) => observation.id).filter((id) => id !== checkpointId);
assert(pendingIds.length === count - 1, `${pendingIds.length} pending observations, expected ${count - 1}`);
await poll("checkpoint and pending vectors to be present on disk", async () =>
hasPersistedVectorFixture(instB.dataDir, sid, checkpointId, pendingIds, gate.dims),
{ timeoutMs: 60_000, intervalMs: 250 });
d.persistedVectorsBeforeCrash = pendingIds.length + 1;
const embedBefore = gate.fake.stats.inputs;
await instB.forceKill();
d.restartMs = await instB.start();
const after = await poll("startup index recovery to finish", async () => {
const s = await instB.status();
return isVectorRecoverySettled(s) ? s : null;
}, { timeoutMs: 30_000 });
d.vectorsAfter = after.index.vectorDocuments;
d.pendingVectorBackfill = after.index.pendingVectorBackfill;
d.vectorBackfillState = after.index.vectorBackfillState;
d.embeddingsAfterRestart = gate.fake.stats.inputs - embedBefore;
assertRecoveredVectors(after, vectorsBefore, d.embeddingsAfterRestart);
await instB.stop();
});
await scenario("stopflush", async (d) => {
await ensureUp(instA);
const sid = `gate-stop-${randomBytes(4).toString("hex")}`;
const m = marker("ember");
await instA.hook("PostToolUse", instA.toolEvent(sid, m));
await poll(`search to find ${m}`, () => instA.searchFinds(m), { timeoutMs: 20_000 });
const before = await instA.totals();
d.before = before;
d.stopMs = await instA.stop();
d.restartMs = await instA.start();
const after = await poll("indexes to match the pre-stop totals", async () => {
const t = await instA.totals();
return t.bm25Documents >= before.bm25Documents && t.vectorDocuments >= before.vectorDocuments ? t : null;
}, { timeoutMs: 30_000, intervalMs: 500 });
d.after = after;
for (const k of Object.keys(before)) assert(after[k] === before[k], `${k} was ${before[k]} before stop and ${after[k]} after start`);
assert((await instA.countMarker(sid, m)) === 1, "the observation captured right before stop is missing");
assert(await instA.searchFinds(m), "search lost the marker captured right before stop");
});
await scenario("status", async (d) => {
await ensureUp(instA);
const viewer = await request("GET", `http://127.0.0.1:${instA.port + 2}/`, undefined, 10_000);
assert(viewer.status === 200 && /<html/i.test(viewer.text), `viewer returned ${viewer.status}`);
d.viewerBytes = viewer.text.length;
const st = await instA.status();
d.status = st.status;
d.problems = (st.problems ?? []).map((p) => ({ level: p.level, code: p.code, message: p.message }));
const errors = (st.problems ?? []).filter((p) => p.level === "error");
assert(errors.length === 0, `status reports errors: ${errors.map((p) => `${p.code}: ${p.message}`).join("; ")}`);
const unexplained = (st.problems ?? []).filter((p) => !p.message || !p.fix);
assert(unexplained.length === 0, `problems without a message and a fix: ${unexplained.map((p) => p.code).join(", ")}`);
assert(["ok", "info", "warn"].includes(st.status), `status level is ${st.status}`);
const html = await request("GET", `${instA.base}/status?format=html`, undefined, 10_000).catch(() => null);
d.statusHtml = html?.status ?? null;
});
await scenario("roundtrip", async (d) => {
await ensureUp(instA);
const exp = await request("GET", `${instA.base}/export`, undefined, 60_000);
assert(exp.status === 200 && exp.json, `GET /export returned ${exp.status}`);
writeFileSync(join(ROOT, "export.json"), exp.text);
d.exportBytes = exp.text.length;
const source = await instA.totals();
d.source = source;
await instA.shutdown();
await ensureUp(instC);
const imp = await request("POST", `${instC.base}/import`, { exportData: exp.json, strategy: "merge" }, 120_000);
assert(imp.status === 200, `POST /import returned ${imp.status}: ${imp.text.slice(0, 300)}`);
d.import = imp.json;
const restored = await poll("restored totals to match the export", async () => {
const t = await instC.totals();
return t.sessions === source.sessions && t.observations === source.observations && t.memories === source.memories ? t : null;
}, { timeoutMs: 60_000, intervalMs: 1000 });
d.restored = restored;
const probe = [...(markers.capture ?? []), ...(markers.offline ?? []), markers.reply, markers.offlineReply].filter(Boolean);
for (const m of probe) await poll(`restored search to find ${m}`, () => instC.searchFinds(m), { timeoutMs: 30_000 });
if (markers.mcp) {
const mem = await request("GET", `${instC.base}/memories`);
assert(mem.text.includes(markers.mcp), "the MCP memory did not round-trip");
}
await instC.stop();
});
}
let fatal = null;
try {
await main();
} catch (err) {
fatal = err instanceof Error ? err.stack || err.message : String(err);
log(`gate aborted: ${fatal}`);
} finally {
for (const inst of [instA, instB, instC]) await inst.shutdown().catch(() => {});
if (gate.fake) gate.fake.server.close();
}
const failed = results.filter((r) => r.status === "fail");
const summary = {
ok: !fatal && failed.length === 0 && results.some((r) => r.status === "pass"),
package: "@agentmemory/agentmemory",
version: gate.info.version ?? null,
enginePin: gate.info.enginePin ?? null,
node: process.version,
platform: `${platform()}-${arch()}`,
durationMs: Date.now() - startedAt,
embeddings: gate.fake ? { requests: gate.fake.stats.calls, inputs: gate.fake.stats.inputs, chatRequests: gate.fake.stats.chatCalls } : null,
scenarios: results,
fatal,
output: OUT,
};
for (const inst of [instA, instB, instC]) {
try {
if (existsSync(inst.home)) {
const files = readdirSync(join(inst.home, ".agentmemory")).filter((f) => f.endsWith(".log"));
for (const f of files) copyFileSync(join(inst.home, ".agentmemory", f), join(OUT, `${inst.name}-${f}`));
}
} catch {}
}
writeFileSync(join(OUT, "summary.json"), JSON.stringify(summary, null, 2) + "\n");
console.log("");
for (const r of results) console.log(`${r.status.toUpperCase().padEnd(7)} ${r.name.padEnd(11)} ${String(r.ms ?? "-").padStart(7)} ms ${r.title}${r.error ? `\n ${r.error}` : ""}`);
console.log("");
console.log(JSON.stringify(summary, null, 2));
cleanup();
process.exit(summary.ok ? 0 : 1);