#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
import { spawn, execFile, execFileSync, type ChildProcess } from "node:child_process";
import { createHash } from "node:crypto";
import {
copyFileSync,
existsSync,
mkdirSync,
openSync,
closeSync,
readFileSync,
readdirSync,
rmSync,
statSync,
writeFileSync,
chmodSync,
} from "node:fs";
import { createServer, type Server, type ServerResponse } from "node:http";
import { createConnection } from "node:net";
import { cpus, homedir, platform, release, tmpdir, totalmem, arch } from "node:os";
import { basename, join, resolve } from "node:path";
import { performance } from "node:perf_hooks";
import { promisify } from "node:util";
import { pXX } from "./lib/percentiles.js";
import { CONCEPTS, NOUNS, buildToolCapture, mulberry32, type ToolCapture } from "./lib/corpus.js";
const execFileAsync = promisify(execFile);
const SCHEMA_VERSION = 1;
const PROJECT = "capture-costs";
type ProfileName = "keyless" | "embed" | "embed-llm";
interface Profile {
name: ProfileName;
description: string;
embeddings: boolean;
llm: boolean;
autoCompress: boolean;
}
const PROFILES: Record<ProfileName, Profile> = {
keyless: {
name: "keyless",
description: "No provider keys. BM25 only, zero-LLM synthetic compression.",
embeddings: false,
llm: false,
autoCompress: false,
},
embed: {
name: "embed",
description: "Deterministic fake OpenAI-compatible embeddings, LLM disabled, automatic compression off.",
embeddings: true,
llm: false,
autoCompress: false,
},
"embed-llm": {
name: "embed-llm",
description: "Fake embeddings plus a fake OpenAI-compatible chat model with AGENTMEMORY_AUTO_COMPRESS=true.",
embeddings: true,
llm: true,
autoCompress: true,
},
};
type ProviderMode = "fake" | "ollama";
type StateBackend = "file" | "redis";
type Scenario = "standard" | "crash-offline";
interface Config {
sizes: number[];
profiles: Profile[];
provider: ProviderMode;
upstreamUrl: string | null;
embedModel: string;
chatModel: string;
stateBackend: StateBackend;
redisUrl: string | null;
scenario: Scenario;
killAfterMs: number;
killSettleMs: number;
perSession: number;
hookSample: number;
concurrency: number;
outputMin: number;
outputMax: number;
seed: number;
dims: number;
contextBudget: number;
searchLimit: number;
evidenceSample: number;
restPort: number;
enginePort: number;
fakePort: number;
root: string;
outDir: string;
keep: boolean;
iiiBin: string;
distDir: string;
quiesceTimeoutMs: number;
readyTimeoutMs: number;
repeats: number;
budgetsPath: string | null;
writeBudgetsPath: string | null;
enforceBudgets: boolean;
}
function intList(raw: string | undefined, fallback: number[]): number[] {
if (!raw) return fallback;
const out = raw
.split(",")
.map((s) => parseInt(s.trim(), 10))
.filter((n) => Number.isFinite(n) && n > 0);
return out.length > 0 ? out : fallback;
}
function intEnv(name: string, fallback: number): number {
const n = parseInt(process.env[name] ?? "", 10);
return Number.isFinite(n) && n >= 0 ? n : fallback;
}
function iiiVersion(bin: string): string | null {
try {
const out = execFileSync(bin, ["--version"], { encoding: "utf8", timeout: 5000, stdio: ["ignore", "pipe", "ignore"] });
return out.match(/(\d+\.\d+\.\d+(?:[-+][\w.]+)?)/)?.[1] ?? null;
} catch {
return null;
}
}
function pinnedIiiVersion(distDir: string): string | null {
try {
const src = readFileSync(join(distDir, "cli.mjs"), "utf8");
return src.match(/IIPINNED_VERSION\s*=\s*[^"']*["'](\d+\.\d+\.\d+)["']/)?.[1] ?? null;
} catch {
return null;
}
}
function resolveIii(distDir: string): string {
const explicit = process.env["AGENTMEMORY_BENCH_III"];
if (explicit) return explicit;
const pinned = pinnedIiiVersion(distDir);
const candidates = [join(homedir(), ".agentmemory", "bin", "iii"), join(homedir(), ".local", "bin", "iii"), "/usr/local/bin/iii"];
for (const c of candidates) {
if (existsSync(c) && (!pinned || iiiVersion(c) === pinned)) return c;
}
throw new Error(`no iii binary matching the pinned version ${pinned ?? "?"}; set AGENTMEMORY_BENCH_III`);
}
function loadConfig(): Config {
const distDir = resolve(process.cwd(), "dist");
const names = (process.env["BENCH_PROFILES"] || "keyless,embed,embed-llm")
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const profiles = names.map((n) => {
const p = PROFILES[n as ProfileName];
if (!p) throw new Error(`unknown profile ${n}; valid: ${Object.keys(PROFILES).join(", ")}`);
return p;
});
const [outputMin, outputMax] = intList(process.env["BENCH_OUTPUT_BYTES"], [400, 4000]);
const restPort = intEnv("BENCH_REST_PORT", intEnv("BENCH_PORT", 4900));
const provider = (process.env["BENCH_PROVIDER"] || "fake") as ProviderMode;
if (provider !== "fake" && provider !== "ollama") throw new Error(`unknown BENCH_PROVIDER ${provider}; valid: fake, ollama`);
const stateBackend = (process.env["BENCH_STATE_BACKEND"] || "file") as StateBackend;
if (stateBackend !== "file" && stateBackend !== "redis") throw new Error(`unknown BENCH_STATE_BACKEND ${stateBackend}; valid: file, redis`);
const redisUrl = process.env["BENCH_REDIS_URL"] || null;
if (stateBackend === "redis" && !redisUrl) throw new Error("BENCH_STATE_BACKEND=redis needs BENCH_REDIS_URL, a Redis the bench may flush");
const scenario = (process.env["BENCH_SCENARIO"] || "standard") as Scenario;
if (scenario !== "standard" && scenario !== "crash-offline") throw new Error(`unknown BENCH_SCENARIO ${scenario}; valid: standard, crash-offline`);
return {
sizes: intList(process.env["BENCH_N"], [100, 1000]),
profiles,
provider,
upstreamUrl: provider === "ollama" ? (process.env["BENCH_OLLAMA_URL"] || "http://127.0.0.1:11434").replace(/\/+$/, "") : null,
embedModel: process.env["BENCH_EMBED_MODEL"] || (provider === "ollama" ? "nomic-embed-text" : "bench-fake-embedding"),
chatModel: process.env["BENCH_CHAT_MODEL"] || (provider === "ollama" ? "qwen3:4b-instruct-2507-q4_K_M" : "bench-fake-chat"),
stateBackend,
redisUrl,
scenario,
killAfterMs: intEnv("BENCH_KILL_AFTER_MS", 3000) || 3000,
killSettleMs: intEnv("BENCH_KILL_SETTLE_MS", 0),
perSession: intEnv("BENCH_OBS_PER_SESSION", 100) || 100,
hookSample: intEnv("BENCH_HOOK_SAMPLE", 100),
concurrency: intEnv("BENCH_CONCURRENCY", 8) || 8,
outputMin: outputMin ?? 400,
outputMax: outputMax ?? outputMin ?? 4000,
seed: intEnv("BENCH_SEED", 12648430),
dims: intEnv("BENCH_EMBED_DIMS", 768) || 768,
contextBudget: intEnv("BENCH_CONTEXT_BUDGET", 2000) || 2000,
searchLimit: intEnv("BENCH_SEARCH_LIMIT", 10) || 10,
evidenceSample: intEnv("BENCH_EVIDENCE_SAMPLE", 50) || 50,
restPort,
enginePort: intEnv("BENCH_ENGINE_PORT", 50100),
fakePort: intEnv("BENCH_FAKE_PORT", restPort + 50),
root: process.env["BENCH_ROOT"] || join(tmpdir(), `agentmemory-capture-costs-${Date.now().toString(36)}`),
outDir: process.env["BENCH_OUT_DIR"] || resolve(process.cwd(), "benchmark", "results"),
keep: process.env["BENCH_KEEP"] === "1",
iiiBin: resolveIii(distDir),
distDir,
quiesceTimeoutMs: intEnv("BENCH_QUIESCE_TIMEOUT_MS", 600000),
readyTimeoutMs: intEnv("BENCH_READY_TIMEOUT_MS", 600000),
repeats: intEnv("BENCH_REPEATS", 1) || 1,
budgetsPath:
process.env["BENCH_BUDGETS"] === "off"
? null
: resolve(process.cwd(), process.env["BENCH_BUDGETS"] || join("benchmark", "capture-costs-budgets.json")),
writeBudgetsPath: process.env["BENCH_WRITE_BUDGETS"] ? resolve(process.cwd(), process.env["BENCH_WRITE_BUDGETS"]) : null,
enforceBudgets: process.env["BENCH_ENFORCE_BUDGETS"] === "1",
};
}
interface Check {
profile: string;
observations: number;
repeat: number;
name: string;
value: number | null;
limit: number;
pass: boolean;
}
interface BudgetMetric {
path: string;
headroom: number;
floor: number;
kind: "storage" | "memory" | "latency" | "context";
}
const BUDGET_METRICS: Record<string, BudgetMetric> = {
diskGrowthPerObservationBytes: { path: "diskGrowthPerObservationBytes", headroom: 1.2, floor: 1024, kind: "storage" },
diskAfterCaptureBytes: { path: "diskAfterCapture.totalBytes", headroom: 1.2, floor: 65536, kind: "storage" },
diskIndexBytes: { path: "diskAfterCapture.byCategory.index", headroom: 1.25, floor: 16384, kind: "storage" },
diskDiagnosticBytes: { path: "diskAfterCapture.byCategory.diagnostic", headroom: 1.5, floor: 16384, kind: "storage" },
capturePeakRssKiB: { path: "capture.rss.peakTotalKiB", headroom: 1.25, floor: 65536, kind: "memory" },
idleRssKiB: { path: "idle.rssKiB.total", headroom: 1.25, floor: 65536, kind: "memory" },
recoveryIdleRssKiB: { path: "recovery.rss.idleTotalKiB", headroom: 1.25, floor: 65536, kind: "memory" },
hookP95Ms: { path: "capture.hookLatencyMs.p95", headroom: 2, floor: 250, kind: "latency" },
observeP95Ms: { path: "capture.observeLatencyMs.p95", headroom: 3, floor: 150, kind: "latency" },
recoveryReadyMs: { path: "recovery.readyMs", headroom: 2, floor: 3000, kind: "latency" },
recoveryIndexReadyMs: { path: "recovery.indexReadyMs", headroom: 2, floor: 3000, kind: "latency" },
searchFullBytesP50: { path: "agentVisibleContext.endpoints.search_full.bytes.p50", headroom: 1.2, floor: 4096, kind: "context" },
searchCompactTokensP50: { path: "agentVisibleContext.endpoints.search_compact.serverTokenEstimate.p50", headroom: 1.2, floor: 512, kind: "context" },
sessionStartInjectOnStdoutBytes: { path: "agentVisibleContext.hookStdout.session_start_inject_on.stdoutBytes", headroom: 1.2, floor: 4096, kind: "context" },
redisUsedMemoryBytes: { path: "redisAfterCapture.usedMemoryBytes", headroom: 1.2, floor: 4194304, kind: "storage" },
redisKeys: { path: "redisAfterCapture.keys", headroom: 1.2, floor: 64, kind: "storage" },
};
function variantOf(r: Record<string, unknown>): string {
return String(r["variant"] ?? r["profile"]);
}
function variantName(profile: Profile, cfg: Config): string {
let name = profile.name;
if (cfg.provider !== "fake" && profile.embeddings) name += `+${cfg.provider}`;
if (cfg.stateBackend !== "file") name += `@${cfg.stateBackend}`;
if (cfg.scenario !== "standard") name += `:${cfg.scenario}`;
return name;
}
interface BudgetFile {
budgets: Record<string, Record<string, Record<string, number>>>;
}
function numberAt(o: unknown, path: string): number | null {
const v = get(o, path);
return typeof v === "number" && Number.isFinite(v) ? v : null;
}
function checkInvariants(runs: Record<string, unknown>[]): Check[] {
const out: Check[] = [];
for (const r of runs) {
const n = r["observations"] as number;
const base = { profile: variantOf(r), observations: n, repeat: (r["repeat"] as number) ?? 1 };
const eq = (name: string, value: number | null, limit: number) => out.push({ ...base, name, value, limit, pass: value === limit });
const atMost = (name: string, value: number | null, limit: number) => out.push({ ...base, name, value, limit, pass: value !== null && value <= limit });
if (r["scenario"] === "crash-offline") {
const embeds = r["embeddings"] === true;
out.push({ ...base, name: "hooksSpooledWhileDown", value: numberAt(r, "crash.spoolRecordsWhileDown"), limit: 1, pass: (numberAt(r, "crash.spoolRecordsWhileDown") ?? 0) >= 1 });
eq("spoolDroppedRecords", numberAt(r, "crash.spoolStats.dropped"), 0);
eq("hookNonZeroExits", numberAt(r, "crash.hookNonZeroExits"), 0);
eq("logicalObservationsAfterRecovery", numberAt(r, "evidenceAfterRecovery.logicalObservations"), n);
eq("sourceTailRetainedAfterRecovery", numberAt(r, "evidenceAfterRecovery.sourceTailRetained"), numberAt(r, "evidenceAfterRecovery.sampled") ?? -1);
eq("spoolRecordsAfterRecovery", numberAt(r, "recovery.spoolRecordsAfter"), 0);
if (embeds) {
eq("killBeforeFirstVectorCheckpoint", numberAt(r, "crash.vectorCheckpointBeforeKill"), 0);
eq("vectorDocumentsAfterRecovery", numberAt(r, "evidenceAfterRecovery.vectorDocuments"), n);
atMost("recoveryRepeatEmbedInputs", numberAt(r, "recovery.provider.embedRepeatInputs"), n);
}
continue;
}
eq("logicalObservationsBeforeKill", numberAt(r, "evidenceBeforeKill.logicalObservations"), n);
eq("logicalObservationsAfterRecovery", numberAt(r, "evidenceAfterRecovery.logicalObservations"), n);
const sampled = numberAt(r, "evidenceBeforeKill.sampled") ?? -1;
eq("sourceTailRetainedBeforeKill", numberAt(r, "evidenceBeforeKill.sourceTailRetained"), sampled);
eq("sourceTailRetainedAfterRecovery", numberAt(r, "evidenceAfterRecovery.sourceTailRetained"), sampled);
eq("observeErrors", numberAt(r, "capture.observeErrors"), 0);
eq("hookStdoutBytesWithInjectionOff", numberAt(r, "capture.hookStdoutBytes.total"), 0);
eq("sessionStartStdoutBytesWithInjectionOff", numberAt(r, "agentVisibleContext.hookStdout.session_start_inject_off.stdoutBytes"), 0);
eq("preToolUseStdoutBytesWithInjectionOff", numberAt(r, "agentVisibleContext.hookStdout.pre_tool_use_inject_off.stdoutBytes"), 0);
const cap = numberAt(r, "agentVisibleContext.declaredContextBudgetTokens") ?? 0;
const ctxTokens = numberAt(r, "agentVisibleContext.endpoints.context.serverTokenEstimate.max");
const vecBefore = numberAt(r, "evidenceBeforeKill.vectorDocuments");
if (vecBefore !== null && vecBefore > 0) eq("vectorDocumentsAfterRecovery", numberAt(r, "evidenceAfterRecovery.vectorDocuments"), vecBefore);
out.push({ ...base, name: "contextTokensWithinDeclaredCap", value: ctxTokens, limit: cap, pass: ctxTokens !== null && ctxTokens <= cap });
}
return out;
}
function checkBudgets(runs: Record<string, unknown>[], file: BudgetFile): Check[] {
const out: Check[] = [];
for (const r of runs) {
const limits = file.budgets?.[variantOf(r)]?.[String(r["observations"])];
if (!limits) continue;
for (const [name, limit] of Object.entries(limits)) {
const metric = BUDGET_METRICS[name];
if (!metric) continue;
const value = numberAt(r, metric.path);
out.push({ profile: variantOf(r), observations: r["observations"] as number, repeat: (r["repeat"] as number) ?? 1, name, value, limit, pass: value !== null && value <= limit });
}
}
return out;
}
function deriveBudgets(runs: Record<string, unknown>[]): BudgetFile {
const budgets: BudgetFile["budgets"] = {};
for (const r of runs) {
if (r["error"] || (r["provider"] ?? "fake") !== "fake" || (r["scenario"] ?? "standard") !== "standard") continue;
const p = variantOf(r);
const n = String(r["observations"]);
const slot = ((budgets[p] ??= {})[n] ??= {});
for (const [name, metric] of Object.entries(BUDGET_METRICS)) {
const value = numberAt(r, metric.path);
if (value === null) continue;
const limit = Math.ceil(Math.max(value * metric.headroom, metric.floor));
slot[name] = Math.max(slot[name] ?? 0, limit);
}
}
return { budgets };
}
interface ProviderCounters {
embedRequests: number;
embedInputs: number;
embedRepeatInputs: number;
chatRequests: number;
chatRepeatPrompts: number;
chatPromptTokens: number;
chatCompletionTokens: number;
embedUpstreamMs: number;
chatUpstreamMs: number;
errors: number;
}
function zeroCounters(): ProviderCounters {
return {
embedRequests: 0,
embedInputs: 0,
embedRepeatInputs: 0,
chatRequests: 0,
chatRepeatPrompts: 0,
chatPromptTokens: 0,
chatCompletionTokens: 0,
embedUpstreamMs: 0,
chatUpstreamMs: 0,
errors: 0,
};
}
function diffCounters(a: ProviderCounters, b: ProviderCounters): ProviderCounters {
const out = zeroCounters();
for (const k of Object.keys(out) as (keyof ProviderCounters)[]) out[k] = b[k] - a[k];
return out;
}
class FakeProvider {
counters = zeroCounters();
inFlight = 0;
private seenInputs = new Set<string>();
private seenPrompts = new Set<string>();
private server: Server | null = null;
constructor(private port: number, private dims: number, private upstream: string | null = null) {}
reset(): void {
this.counters = zeroCounters();
this.seenInputs.clear();
this.seenPrompts.clear();
}
snapshot(): ProviderCounters {
return { ...this.counters };
}
private vector(text: string, dims: number): number[] {
const out = new Array<number>(dims);
let seed = createHash("sha256").update(text).digest();
let norm = 0;
for (let i = 0; i < dims; i++) {
if (i % 32 === 0 && i > 0) seed = createHash("sha256").update(seed).digest();
out[i] = (seed[i % 32]! - 128) / 128;
norm += out[i]! * out[i]!;
}
const scale = norm > 0 ? 1 / Math.sqrt(norm) : 1;
return out.map((v) => v * scale);
}
private chatReply(prompt: string): string {
const hash = createHash("sha256").update(prompt).digest("hex");
const words = prompt.match(/[a-z][a-z-]{3,}/g) ?? [];
const concepts = Array.from(new Set(words.filter((w) => (CONCEPTS as readonly string[]).includes(w)))).slice(0, 4);
const nouns = Array.from(new Set(words.filter((w) => (NOUNS as readonly string[]).includes(w)))).slice(0, 3);
const marker = prompt.match(/capmark\d+head/)?.[0] ?? `obs-${hash.slice(0, 8)}`;
return [
"<type>command_run</type>",
`<title>${marker} ${nouns.join(" ") || "tool output"}</title>`,
`<facts><fact>${marker} recorded ${nouns[0] ?? "work"}</fact><fact>digest ${hash.slice(0, 12)}</fact></facts>`,
`<narrative>${marker} captured a tool run touching ${nouns.join(", ") || "the workspace"} with ${concepts.join(", ") || "no named concepts"}.</narrative>`,
`<concepts>${concepts.map((c) => `<concept>${c}</concept>`).join("")}</concepts>`,
"<files></files>",
"<importance>5</importance>",
].join("\n");
}
private count(url: string, payload: Record<string, unknown>): { prompt: string } | null {
if (url.endsWith("/embeddings")) {
const inputs = (Array.isArray(payload["input"]) ? payload["input"] : [payload["input"]]).map(String);
this.counters.embedRequests++;
this.counters.embedInputs += inputs.length;
for (const t of inputs) {
if (this.seenInputs.has(t)) this.counters.embedRepeatInputs++;
else this.seenInputs.add(t);
}
return null;
}
const messages = Array.isArray(payload["messages"]) ? (payload["messages"] as { content?: unknown }[]) : [];
const prompt = messages.map((m) => (typeof m.content === "string" ? m.content : JSON.stringify(m.content))).join("\n");
this.counters.chatRequests++;
if (this.seenPrompts.has(prompt)) this.counters.chatRepeatPrompts++;
else this.seenPrompts.add(prompt);
return { prompt };
}
private async forward(url: string, raw: Buffer, payload: Record<string, unknown>, res: ServerResponse): Promise<void> {
const isEmbed = url.endsWith("/embeddings");
if (!isEmbed && !url.endsWith("/chat/completions")) {
this.counters.errors++;
res.writeHead(404, { "content-type": "application/json" });
res.end(JSON.stringify({ error: { message: `bench proxy: unsupported ${url}` } }));
return;
}
this.count(url, payload);
const t0 = performance.now();
this.inFlight++;
try {
const up = await fetch(`${this.upstream}${url}`, { method: "POST", headers: { "content-type": "application/json" }, body: raw.toString("utf8"), signal: AbortSignal.timeout(300000) });
const text = await up.text();
const ms = performance.now() - t0;
if (isEmbed) this.counters.embedUpstreamMs += ms;
else this.counters.chatUpstreamMs += ms;
if (!up.ok) this.counters.errors++;
if (!isEmbed && up.ok) {
try {
const usage = (JSON.parse(text) as { usage?: { prompt_tokens?: number; completion_tokens?: number } }).usage;
this.counters.chatPromptTokens += usage?.prompt_tokens ?? 0;
this.counters.chatCompletionTokens += usage?.completion_tokens ?? 0;
} catch {}
}
res.writeHead(up.status, { "content-type": up.headers.get("content-type") ?? "application/json" });
res.end(text);
} catch (err) {
this.counters.errors++;
res.writeHead(502, { "content-type": "application/json" });
res.end(JSON.stringify({ error: { message: `bench proxy: ${err instanceof Error ? err.message : String(err)}` } }));
} finally {
this.inFlight--;
}
}
start(): Promise<void> {
this.server = createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
let payload: Record<string, unknown> = {};
const raw = Buffer.concat(chunks);
try {
payload = JSON.parse(raw.toString("utf8") || "{}");
} catch {}
const url = req.url ?? "";
if (this.upstream) {
void this.forward(url, raw, payload, res);
return;
}
if (url.endsWith("/embeddings")) {
const inputs = (Array.isArray(payload["input"]) ? payload["input"] : [payload["input"]]).map(String);
const dims = typeof payload["dimensions"] === "number" ? (payload["dimensions"] as number) : this.dims;
this.count(url, payload);
const total = inputs.reduce((s, t) => s + Math.ceil(t.length / 4), 0);
res.writeHead(200, { "content-type": "application/json" });
res.end(
JSON.stringify({
object: "list",
data: inputs.map((t, index) => ({ object: "embedding", index, embedding: this.vector(t, dims) })),
model: String(payload["model"] ?? "fake-embedding"),
usage: { prompt_tokens: total, total_tokens: total },
}),
);
return;
}
if (url.endsWith("/chat/completions")) {
const { prompt } = this.count(url, payload)!;
const content = this.chatReply(prompt);
const promptTokens = Math.ceil(prompt.length / 4);
const completionTokens = Math.ceil(content.length / 4);
this.counters.chatPromptTokens += promptTokens;
this.counters.chatCompletionTokens += completionTokens;
res.writeHead(200, { "content-type": "application/json" });
res.end(
JSON.stringify({
id: `chatcmpl-${createHash("sha256").update(prompt).digest("hex").slice(0, 12)}`,
object: "chat.completion",
model: String(payload["model"] ?? "fake-chat"),
choices: [{ index: 0, finish_reason: "stop", message: { role: "assistant", content } }],
usage: { prompt_tokens: promptTokens, completion_tokens: completionTokens, total_tokens: promptTokens + completionTokens },
}),
);
return;
}
this.counters.errors++;
res.writeHead(404, { "content-type": "application/json" });
res.end(JSON.stringify({ error: { message: `fake provider: unsupported ${url}` } }));
});
});
return new Promise((ok, fail) => {
this.server!.once("error", fail);
this.server!.listen(this.port, "127.0.0.1", () => ok());
});
}
stop(): Promise<void> {
return new Promise((ok) => (this.server ? this.server.close(() => ok()) : ok()));
}
}
interface ProcRow {
pid: number;
ppid: number;
rssKiB: number;
cpuSec: number;
comm: string;
args: string;
}
function parseCpu(raw: string): number {
const [clock, ...rest] = raw.includes("-") ? raw.split("-").reverse() : [raw];
const days = rest.length > 0 ? parseInt(rest[0]!, 10) || 0 : 0;
const parts = clock!.split(":").map((p) => parseFloat(p) || 0);
let sec = 0;
for (const p of parts) sec = sec * 60 + p;
return sec + days * 86400;
}
async function psRows(): Promise<ProcRow[]> {
const { stdout } = await execFileAsync("ps", ["-A", "-o", "pid=,ppid=,rss=,time=,comm=,args="], { timeout: 10000, maxBuffer: 8 * 1024 * 1024 });
const rows: ProcRow[] = [];
for (const line of stdout.split("\n")) {
const m = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s*(.*)$/);
if (!m) continue;
rows.push({ pid: +m[1]!, ppid: +m[2]!, rssKiB: +m[3]!, cpuSec: parseCpu(m[4]!), comm: m[5]!, args: m[6]! });
}
return rows;
}
function treeOf(rows: ProcRow[], roots: number[]): ProcRow[] {
const byParent = new Map<number, ProcRow[]>();
for (const r of rows) {
const list = byParent.get(r.ppid) ?? [];
list.push(r);
byParent.set(r.ppid, list);
}
const seen = new Set<number>();
const out: ProcRow[] = [];
const stack = rows.filter((r) => roots.includes(r.pid));
while (stack.length > 0) {
const r = stack.pop()!;
if (seen.has(r.pid)) continue;
seen.add(r.pid);
out.push(r);
for (const c of byParent.get(r.pid) ?? []) stack.push(c);
}
return out;
}
type ProcessRole = "compose" | "engine" | "state" | "http" | "agentmemory" | "worker" | "other";
function roleOf(r: ProcRow): ProcessRole {
const name = basename(r.comm);
const args = r.args;
if (/\bcompose\s+--namespace\b/.test(args)) return "compose";
if (/^iii:e:/.test(name) || (/--config\s/.test(args) && /engine-config\.yaml/.test(args))) return "engine";
if (name === "state" || name === "iii-state" || /(?:^|\/)state(?:\s|$)/.test(args) || /\/state-\d[^/]*\/state(?:\s|$)/.test(args)) return "state";
if (name === "http" || name === "iii-http" || /(?:^|\/)http(?:\s|$)/.test(args) || /\/http-\d[^/]*\/http(?:\s|$)/.test(args)) return "http";
if ((name === "MainThread" || name === "node" || name === "sh") && /dist\/index\.mjs/.test(args)) return "agentmemory";
if (/^iii:c:/.test(name) || /\bcompose\s+--namespace\b/.test(args)) return "compose";
if (name === "iii" || name.startsWith("iii")) return "engine";
if (name === "node" || name.includes("node")) return "worker";
return "other";
}
interface TreeSample {
totalKiB: number;
engineKiB: number;
workerKiB: number;
processKiB: Record<ProcessRole, number>;
cpuSec: number;
pids: number[];
}
function summarizeTree(tree: ProcRow[]): TreeSample {
const processKiB: Record<ProcessRole, number> = { compose: 0, engine: 0, state: 0, http: 0, agentmemory: 0, worker: 0, other: 0 };
const s: TreeSample = { totalKiB: 0, engineKiB: 0, workerKiB: 0, processKiB, cpuSec: 0, pids: tree.map((r) => r.pid) };
for (const r of tree) {
s.totalKiB += r.rssKiB;
s.cpuSec += r.cpuSec;
const role = roleOf(r);
s.processKiB[role] += r.rssKiB;
if (role === "engine") s.engineKiB += r.rssKiB;
else if (role === "state" || role === "http" || role === "agentmemory" || role === "worker") s.workerKiB += r.rssKiB;
}
return s;
}
interface RssWindow {
samples: number;
peakTotalKiB: number;
peakEngineKiB: number;
peakWorkerKiB: number;
peakProcessKiB: Record<ProcessRole, number>;
meanTotalKiB: number;
cpuSecStart: number;
cpuSecEnd: number;
}
class RssSampler {
private timer: NodeJS.Timeout | null = null;
private busy = false;
private window: RssWindow | null = null;
private sum = 0;
constructor(private roots: () => number[], private intervalMs = 250) {}
async sampleOnce(): Promise<TreeSample> {
return summarizeTree(treeOf(await psRows(), this.roots()));
}
async begin(): Promise<void> {
const first = await this.sampleOnce();
this.sum = first.totalKiB;
this.window = {
samples: 1,
peakTotalKiB: first.totalKiB,
peakEngineKiB: first.engineKiB,
peakWorkerKiB: first.workerKiB,
peakProcessKiB: { ...first.processKiB },
meanTotalKiB: first.totalKiB,
cpuSecStart: first.cpuSec,
cpuSecEnd: first.cpuSec,
};
this.timer = setInterval(() => void this.tick(), this.intervalMs);
}
private async tick(): Promise<void> {
if (this.busy || !this.window) return;
this.busy = true;
try {
const s = await this.sampleOnce();
const w = this.window;
if (!w) return;
w.samples++;
this.sum += s.totalKiB;
w.peakTotalKiB = Math.max(w.peakTotalKiB, s.totalKiB);
w.peakEngineKiB = Math.max(w.peakEngineKiB, s.engineKiB);
w.peakWorkerKiB = Math.max(w.peakWorkerKiB, s.workerKiB);
for (const [role, value] of Object.entries(s.processKiB) as Array<[ProcessRole, number]>) {
w.peakProcessKiB[role] = Math.max(w.peakProcessKiB[role], value);
}
w.cpuSecEnd = Math.max(w.cpuSecEnd, s.cpuSec);
} catch {
} finally {
this.busy = false;
}
}
stop(): void {
if (this.timer) clearInterval(this.timer);
this.timer = null;
this.window = null;
}
async end(): Promise<RssWindow & { cpuSec: number }> {
if (this.timer) clearInterval(this.timer);
this.timer = null;
await this.tick();
const w = this.window!;
this.window = null;
w.meanTotalKiB = Math.round(this.sum / w.samples);
return { ...w, cpuSec: +(w.cpuSecEnd - w.cpuSecStart).toFixed(2) };
}
}
function sleep(ms: number): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}
function portOpen(port: number): Promise<boolean> {
return new Promise((ok) => {
const sock = createConnection({ port, host: "127.0.0.1" });
sock.setTimeout(500);
sock.once("connect", () => {
sock.destroy();
ok(true);
});
sock.once("timeout", () => {
sock.destroy();
ok(false);
});
sock.once("error", () => ok(false));
});
}
interface HttpResult {
status: number;
bytes: number;
ms: number;
body: unknown;
text: string;
}
let benchSecretFile: string | null = null;
function benchAuth(): Record<string, string> {
try {
const secret = benchSecretFile ? readFileSync(benchSecretFile, "utf8").trim() : "";
return secret ? { authorization: `Bearer ${secret}` } : {};
} catch {
return {};
}
}
async function http(base: string, method: string, path: string, body?: unknown, timeoutMs = 60000): Promise<HttpResult> {
const t0 = performance.now();
const res = await fetch(`${base}${path}`, {
method,
headers: { ...benchAuth(), ...(body === undefined ? { accept: "application/json" } : { "content-type": "application/json", accept: "application/json" }) },
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(timeoutMs),
});
const text = await res.text();
const ms = performance.now() - t0;
let parsed: unknown = text;
try {
parsed = JSON.parse(text);
} catch {}
return { status: res.status, bytes: Buffer.byteLength(text, "utf8"), ms, body: parsed, text };
}
function dist(values: number[]): { n: number; p50: number; p95: number; p99: number; max: number; mean: number } {
const sorted = [...values].sort((a, b) => a - b);
const r = (v: number) => (Number.isFinite(v) ? +v.toFixed(2) : NaN);
const mean = sorted.length ? sorted.reduce((s, v) => s + v, 0) / sorted.length : NaN;
return {
n: sorted.length,
p50: r(pXX(sorted, 50)),
p95: r(pXX(sorted, 95)),
p99: r(pXX(sorted, 99)),
max: r(pXX(sorted, 100)),
mean: r(mean),
};
}
type Category = "source" | "summary" | "index" | "queue" | "failed_delivery" | "diagnostic" | "config" | "stream" | "other";
function categorize(scope: string): Category {
if (/dlq|dead|failed/i.test(scope)) return "failed_delivery";
if (/queue/i.test(scope)) return "queue";
if (/^mem:(obs|enriched|image-refs|image-embeddings):?/.test(scope) || scope === "mem:sessions") return "source";
if (/^mem:(index|idx|emb|latent)(:|$)/.test(scope) || /^mem:graph:(name-index|edge-key|node-degree)$/.test(scope)) return "index";
if (/^mem:(audit|health|metrics|access|recent-searches|retention|diagnostics|signals|sentinels)(:|$)/.test(scope)) return "diagnostic";
if (/^mem:(config|state)(:|$)/.test(scope)) return "config";
if (scope.startsWith("mem:")) return "summary";
return "other";
}
function scopeFamily(scope: string): string {
return scope
.replace(/^mem:obs:.+$/, "mem:obs:*")
.replace(/^mem:enriched:.+$/, "mem:enriched:*")
.replace(/^mem:idx:obs:\d+$/, "mem:idx:obs:*")
.replace(/^mem:audit:\d{4}-\d{2}$/, "mem:audit:<month>")
.replace(/^(mem:index:bm25:vec:).+$/, "$1*")
.replace(/^mem:emb:.+$/, "mem:emb:*");
}
interface DiskReport {
totalBytes: number;
byCategory: Record<Category, number>;
byFamily: Record<string, number>;
files: number;
}
function walk(dir: string, visit: (path: string, size: number) => void): void {
if (!existsSync(dir)) return;
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const p = join(dir, entry.name);
if (entry.isDirectory()) walk(p, visit);
else if (entry.isFile()) {
try {
visit(p, statSync(p).size);
} catch {}
}
}
}
function measureDisk(dataDir: string): DiskReport {
const report: DiskReport = {
totalBytes: 0,
byCategory: { source: 0, summary: 0, index: 0, queue: 0, failed_delivery: 0, diagnostic: 0, config: 0, stream: 0, other: 0 },
byFamily: {},
files: 0,
};
walk(dataDir, (p, size) => {
report.totalBytes += size;
report.files++;
const rel = p.slice(dataDir.length + 1);
const [top, ...rest] = rel.split("/");
let scope = rel;
let category: Category = "other";
if (top === "state_store.db" && rest.length > 0) {
scope = decodeURIComponent(rest.join("/").replace(/\.bin$/, ""));
category = categorize(scope);
} else if (top === "stream_store") {
scope = `stream:${decodeURIComponent(rest.join("/").replace(/\.bin$/, ""))}`;
category = "stream";
} else if (/\.ya?ml$/.test(rel)) {
category = "config";
} else {
category = categorize(rel);
}
report.byCategory[category] += size;
const fam = scopeFamily(scope);
report.byFamily[fam] = (report.byFamily[fam] ?? 0) + size;
});
return report;
}
function diskDelta(a: DiskReport, b: DiskReport): DiskReport {
const out: DiskReport = { totalBytes: b.totalBytes - a.totalBytes, byCategory: { ...b.byCategory }, byFamily: {}, files: b.files - a.files };
for (const k of Object.keys(out.byCategory) as Category[]) out.byCategory[k] = b.byCategory[k] - a.byCategory[k];
for (const k of new Set([...Object.keys(a.byFamily), ...Object.keys(b.byFamily)])) {
out.byFamily[k] = (b.byFamily[k] ?? 0) - (a.byFamily[k] ?? 0);
}
return out;
}
async function settledDisk(dataDir: string, quietMs = 3000, timeoutMs = 60000): Promise<DiskReport> {
const start = Date.now();
let last = measureDisk(dataDir);
let stableSince = Date.now();
while (Date.now() - start < timeoutMs) {
await sleep(1000);
const now = measureDisk(dataDir);
if (now.totalBytes !== last.totalBytes || now.files !== last.files) {
last = now;
stableSince = Date.now();
} else if (Date.now() - stableSince >= quietMs) {
return now;
}
}
return last;
}
interface BootLog {
bm25Docs: number | null;
bm25RebuildMs: number | null;
vectorsLoaded: number | null;
vectorBackfillQueued: number | null;
vectorBackfillAwaitingOptIn: number | null;
pendingLogRecoveredVectors: number;
spoolRecovered: number;
spoolAlreadyStored: number;
inboxRecovered: number;
}
function parseBootLog(text: string): BootLog {
const bm = [...text.matchAll(/Rebuilt BM25 index from stored content \((\d+) docs in (\d+) ms\)/g)].pop();
const vec = [...text.matchAll(/Loaded persisted vector index \((\d+) vectors\)/g)].pop();
const back = [...text.matchAll(/(?:Backfilling|Re-embedding) (\d+) (?:missing )?vectors/g)].pop();
const optIn = [...text.matchAll(/Vector backfill needs (\d+) embeddings/g)].pop();
const replay = [...text.matchAll(/Recovered (\d+) vectors and \d+ removals written after the last index save/g)].pop();
const spool = [...text.matchAll(/Capture spool: (\d+) recovered, (\d+) already stored/g)].pop();
const inbox = [...text.matchAll(/Capture inbox: (\d+) of \d+ unfinished observations stored/g)].pop();
return {
bm25Docs: bm ? +bm[1]! : null,
bm25RebuildMs: bm ? +bm[2]! : null,
vectorsLoaded: vec ? +vec[1]! : 0,
vectorBackfillQueued: back ? +back[1]! : 0,
vectorBackfillAwaitingOptIn: optIn ? +optIn[1]! : 0,
pendingLogRecoveredVectors: replay ? +replay[1]! : 0,
spoolRecovered: spool ? +spool[1]! : 0,
spoolAlreadyStored: spool ? +spool[2]! : 0,
inboxRecovered: inbox ? +inbox[1]! : 0,
};
}
interface Instance {
child: ChildProcess;
logPath: string;
logOffset: number;
base: string;
}
class Harness {
private instance: Instance | null = null;
private knownPids = new Set<number>();
private lastLogPath: string | null = null;
readonly sampler: RssSampler;
readonly home: string;
readonly dataDir: string;
readonly cwd: string;
readonly base: string;
constructor(private cfg: Config, private profile: Profile, private runDir: string) {
this.home = join(runDir, "home");
benchSecretFile = join(this.home, ".agentmemory", "secret");
this.dataDir = join(runDir, "data");
this.cwd = join(runDir, PROJECT);
this.base = `http://127.0.0.1:${cfg.restPort}/agentmemory`;
this.sampler = new RssSampler(() => this.roots());
}
roots(): number[] {
return [...this.knownPids];
}
prepare(): void {
for (const d of [this.home, this.dataDir, this.cwd, join(this.home, ".agentmemory", "bin")]) mkdirSync(d, { recursive: true });
const target = join(this.home, ".agentmemory", "bin", "iii");
copyFileSync(this.cfg.iiiBin, target);
chmodSync(target, 0o755);
}
env(): NodeJS.ProcessEnv {
const keep = ["PATH", "TMPDIR", "LANG", "TERM", "SHELL", "USER", "LOGNAME"];
const env: NodeJS.ProcessEnv = {};
for (const k of keep) if (process.env[k] !== undefined) env[k] = process.env[k];
env["PATH"] = `${join(this.home, ".agentmemory", "bin")}:${process.env["PATH"] ?? ""}`;
Object.assign(env, {
HOME: this.home,
CI: "1",
NO_COLOR: "1",
XDG_CONFIG_HOME: join(this.home, ".config"),
XDG_DATA_HOME: join(this.home, ".local", "share"),
III_ENGINE_PORT: String(this.cfg.enginePort),
AGENTMEMORY_METRICS_PORT: String(this.cfg.restPort + 3),
AGENTMEMORY_PROJECT_NAME: PROJECT,
AGENTMEMORY_AUTO_COMPRESS: this.profile.autoCompress ? "true" : "false",
AGENTMEMORY_INJECT_CONTEXT: "false",
});
if (this.profile.embeddings) {
Object.assign(env, {
OPENAI_API_KEY: "bench-fake-key",
OPENAI_BASE_URL: `http://127.0.0.1:${this.cfg.fakePort}`,
OPENAI_EMBEDDING_DIMENSIONS: String(this.cfg.dims),
OPENAI_EMBEDDING_MODEL: this.cfg.embedModel,
OPENAI_API_KEY_FOR_LLM: this.profile.llm ? "true" : "false",
});
if (this.profile.llm) env["OPENAI_MODEL"] = this.cfg.chatModel;
}
if (this.cfg.stateBackend === "redis") {
env["AGENTMEMORY_STATE_BACKEND"] = "redis";
env["AGENTMEMORY_REDIS_URL"] = this.cfg.redisUrl!;
}
return env;
}
hookEnv(inject: boolean): NodeJS.ProcessEnv {
return {
PATH: process.env["PATH"],
HOME: this.home,
AGENTMEMORY_URL: `http://127.0.0.1:${this.cfg.restPort}`,
AGENTMEMORY_PROJECT_NAME: PROJECT,
AGENTMEMORY_INJECT_CONTEXT: inject ? "true" : "false",
};
}
async start(label: string): Promise<{ readyMs: number; indexReadyMs: number | null; boot: BootLog }> {
for (const port of [this.cfg.restPort, this.cfg.restPort + 2, this.cfg.restPort + 6353, this.cfg.enginePort]) {
if (await portOpen(port)) throw new Error(`port ${port} is in use before ${label}; refusing to share it`);
}
const logPath = join(this.runDir, `cli-${label}.log`);
this.lastLogPath = logPath;
const fd = openSync(logPath, "a");
const t0 = performance.now();
const child = spawn(
process.execPath,
[join(this.cfg.distDir, "cli.mjs"), "--port", String(this.cfg.restPort), "--data-dir", this.dataDir, "--verbose"],
{ cwd: this.cwd, env: this.env(), stdio: ["ignore", fd, fd] },
);
closeSync(fd);
this.knownPids.add(child.pid!);
this.instance = { child, logPath, logOffset: 0, base: this.base };
let readyMs = -1;
while (performance.now() - t0 < this.cfg.readyTimeoutMs) {
if (child.exitCode !== null) throw new Error(`cli exited with ${child.exitCode} during ${label}; see ${logPath}`);
try {
const r = await http(this.base, "GET", "/livez", undefined, 2000);
if (r.status === 200) {
readyMs = performance.now() - t0;
break;
}
} catch {}
await sleep(100);
}
if (readyMs < 0) throw new Error(`instance not ready within ${this.cfg.readyTimeoutMs} ms (${label}); see ${logPath}`);
let indexReadyMs: number | null = null;
while (performance.now() - t0 < this.cfg.readyTimeoutMs) {
const logs = `${readFileSync(logPath, "utf8")}\n${this.composeWorkerLogText()}`;
if (/Rebuilt BM25 index from stored content/.test(logs)) {
indexReadyMs = performance.now() - t0;
break;
}
try {
const status = await http(this.base, "GET", "/status", undefined, 2000);
if (status.status === 200) {
indexReadyMs = performance.now() - t0;
break;
}
} catch {}
await sleep(100);
}
await this.adoptTree();
return { readyMs: Math.round(readyMs), indexReadyMs: indexReadyMs === null ? null : Math.round(indexReadyMs), boot: parseBootLog(`${readFileSync(logPath, "utf8")}\n${this.composeWorkerLogText()}`) };
}
bootLog(): BootLog | null {
return this.lastLogPath && existsSync(this.lastLogPath) ? parseBootLog(`${readFileSync(this.lastLogPath, "utf8")}\n${this.composeWorkerLogText()}`) : null;
}
private composeWorkerLogText(): string {
const root = join(this.dataDir, "compose-state");
const logs: string[] = [];
const visit = (dir: string) => {
if (!existsSync(dir)) return;
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
if (entry.isDirectory()) visit(path);
else if (entry.name === "agentmemory.log") {
try {
logs.push(readFileSync(path, "utf8").split(/\r?\n/).map((line) => line.replace(/^(?:stdout|stderr)\t/, "")).join("\n"));
} catch {}
}
}
};
visit(root);
return logs.join("\n");
}
private spoolFiles(): string[] {
const files: string[] = [];
const visit = (dir: string, inSpool: boolean) => {
if (!existsSync(dir)) return;
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const p = join(dir, entry.name);
if (entry.isDirectory()) visit(p, inSpool || entry.name === "capture-spool");
else if (inSpool) files.push(p);
}
};
visit(this.home, false);
return files;
}
spoolRecords(): number {
return this.spoolFiles()
.filter((p) => p.endsWith(".jsonl"))
.reduce((n, p) => n + readFileSync(p, "utf8").split("\n").filter((l) => l.trim()).length, 0);
}
spoolStats(): Record<string, number> {
const out: Record<string, number> = {};
for (const p of this.spoolFiles().filter((f) => f.endsWith(".stats.json"))) {
try {
for (const [k, v] of Object.entries(JSON.parse(readFileSync(p, "utf8")) as Record<string, unknown>)) {
if (typeof v === "number") out[k] = (out[k] ?? 0) + v;
}
} catch {}
}
return out;
}
async adoptTree(): Promise<void> {
const rows = await psRows();
for (const r of treeOf(rows, this.roots())) this.knownPids.add(r.pid);
const pidfile = join(this.home, ".agentmemory", "iii.pid");
if (existsSync(pidfile)) {
const pid = parseInt(readFileSync(pidfile, "utf8").trim(), 10);
if (Number.isFinite(pid) && rows.some((r) => r.pid === pid && roleOf(r) === "engine")) this.knownPids.add(pid);
}
}
private async alive(): Promise<number[]> {
const rows = await psRows();
const live = new Set(rows.map((r) => r.pid));
return this.roots().filter((p) => live.has(p));
}
private async waitPortsFree(timeoutMs: number): Promise<void> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
const busy = await Promise.all([this.cfg.restPort, this.cfg.restPort + 2, this.cfg.restPort + 6353, this.cfg.enginePort].map(portOpen));
if (!busy.some(Boolean)) return;
await sleep(250);
}
}
async forceKill(): Promise<void> {
await this.adoptTree();
for (const pid of await this.alive()) {
try {
process.kill(pid, "SIGKILL");
} catch {}
}
await sleep(500);
this.knownPids.clear();
this.instance = null;
await this.waitPortsFree(15000);
}
async stop(): Promise<void> {
await this.adoptTree();
const cli = this.instance?.child.pid;
if (cli) {
try {
process.kill(cli, "SIGTERM");
} catch {}
}
const deadline = Date.now() + 20000;
while (Date.now() < deadline) {
const left = await this.alive();
if (left.length === 0) break;
if (Date.now() > deadline - 12000) {
for (const pid of left) {
try {
process.kill(pid, "SIGTERM");
} catch {}
}
}
await sleep(500);
}
for (const pid of await this.alive()) {
try {
process.kill(pid, "SIGKILL");
} catch {}
}
this.knownPids.clear();
this.instance = null;
await this.waitPortsFree(15000);
}
}
interface HookRun {
ms: number;
stdoutBytes: number;
stderrBytes: number;
exitCode: number | null;
}
function runHook(script: string, payload: unknown, env: NodeJS.ProcessEnv, timeoutMs = 15000, cwd?: string): Promise<HookRun> {
return new Promise((ok) => {
const t0 = performance.now();
const child = spawn(process.execPath, [script], { env, cwd, stdio: ["pipe", "pipe", "pipe"] });
let out = 0;
let err = 0;
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
child.stdout.on("data", (c: Buffer) => (out += c.length));
child.stderr.on("data", (c: Buffer) => (err += c.length));
child.on("close", (code) => {
clearTimeout(timer);
ok({ ms: performance.now() - t0, stdoutBytes: out, stderrBytes: err, exitCode: code });
});
child.stdin.end(JSON.stringify(payload));
});
}
function sessionIdFor(profile: Profile, n: number, j: number): string {
return `cc-${profile.name}-${n}-s${j}`;
}
function observeBody(sessionId: string, cwd: string, cap: ToolCapture) {
return {
hookType: "post_tool_use",
sessionId,
project: PROJECT,
cwd,
timestamp: new Date().toISOString(),
data: { tool_name: cap.toolName, tool_input: cap.toolInput, tool_output: cap.toolOutput },
};
}
function hookBody(sessionId: string, cwd: string, cap: ToolCapture) {
return {
session_id: sessionId,
transcript_path: "/dev/null",
cwd,
hook_event_name: "PostToolUse",
tool_name: cap.toolName,
tool_input: cap.toolInput,
tool_response: cap.toolOutput,
};
}
async function drive(concurrency: number, total: number, fn: (i: number) => Promise<void>): Promise<void> {
let next = 0;
await Promise.all(
Array.from({ length: Math.min(concurrency, Math.max(total, 1)) }, async () => {
while (true) {
const i = next++;
if (i >= total) return;
await fn(i);
}
}),
);
}
interface Evidence {
logicalObservations: number;
bm25Documents: number | null;
vectorDocuments: number | null;
pendingVectorBackfill: number | null;
sessionsListed: number;
sampled: number;
markerSearchHits: number;
sourceTailRetained: number;
sourceHeadRetained: number;
}
async function collectEvidence(h: Harness, profile: Profile, n: number, perSession: number, caps: Map<number, ToolCapture>, sample: number[]): Promise<Evidence> {
const sessions = Math.ceil(n / perSession);
let logical = 0;
let listed = 0;
for (let j = 0; j < sessions; j++) {
const r = await http(h.base, "GET", `/observations?sessionId=${encodeURIComponent(sessionIdFor(profile, n, j))}&limit=1`, undefined, 120000);
const body = r.body as { total?: number };
if (r.status === 200) listed++;
logical += typeof body?.total === "number" ? body.total : 0;
}
let hits = 0;
let tail = 0;
let head = 0;
const replayCache = new Map<number, string>();
for (const i of sample) {
const cap = caps.get(i)!;
const s = await http(h.base, "POST", "/search", { query: cap.headMarker, limit: 5 }, 120000);
if (s.text.includes(cap.headMarker)) hits++;
const j = Math.floor(i / perSession);
if (!replayCache.has(j)) {
const rl = await http(h.base, "GET", `/replay/load?sessionId=${encodeURIComponent(sessionIdFor(profile, n, j))}`, undefined, 120000);
replayCache.set(j, rl.text);
}
const text = replayCache.get(j)!;
if (text.includes(cap.tailMarker)) tail++;
if (text.includes(cap.headMarker)) head++;
}
const status = await http(h.base, "GET", "/status", undefined, 120000);
const index = (status.body as { index?: Record<string, unknown> })?.index ?? {};
const num = (v: unknown) => (typeof v === "number" ? v : null);
return {
logicalObservations: logical,
bm25Documents: num(index["bm25Documents"]),
vectorDocuments: num(index["vectorDocuments"]),
pendingVectorBackfill: num(index["pendingVectorBackfill"]),
sessionsListed: listed, sampled: sample.length, markerSearchHits: hits, sourceTailRetained: tail, sourceHeadRetained: head };
}
async function quiesce(h: Harness, fake: FakeProvider, timeoutMs: number): Promise<{ ms: number; disk: DiskReport }> {
const t0 = performance.now();
let last = JSON.stringify(fake.snapshot());
let stableSince = Date.now();
while (performance.now() - t0 < timeoutMs) {
await sleep(1000);
const now = JSON.stringify(fake.snapshot());
if (now !== last || fake.inFlight > 0) {
last = now;
stableSince = Date.now();
} else if (Date.now() - stableSince >= 3000) break;
}
const disk = await settledDisk(h.dataDir, 3000, Math.max(10000, timeoutMs - (performance.now() - t0)));
return { ms: Math.round(performance.now() - t0), disk };
}
function tokensOf(body: unknown): number | null {
if (!body || typeof body !== "object") return null;
const b = body as Record<string, unknown>;
if (typeof b["tokens_used"] === "number") return b["tokens_used"] as number;
if (typeof b["tokens"] === "number") return b["tokens"] as number;
if (typeof b["tokensUsed"] === "number") return b["tokensUsed"] as number;
return null;
}
async function measureContext(h: Harness, profile: Profile, n: number, caps: Map<number, ToolCapture>, cfg: Config) {
const rng = mulberry32(cfg.seed ^ 0x51ed27);
const queries = Array.from({ length: 8 }, () => `${CONCEPTS[Math.floor(rng() * CONCEPTS.length)]} ${NOUNS[Math.floor(rng() * NOUNS.length)]}`);
const rows: Record<string, { bytes: number[]; tokens: number[]; ms: number[]; results: number[] }> = {};
const add = (k: string, r: HttpResult) => {
const row = (rows[k] ??= { bytes: [], tokens: [], ms: [], results: [] });
row.bytes.push(r.bytes);
const t = tokensOf(r.body);
if (t !== null) row.tokens.push(t);
row.ms.push(r.ms);
const results = (r.body as { results?: unknown[] })?.results;
row.results.push(Array.isArray(results) ? results.length : 0);
};
for (const q of queries) {
add("search_full", await http(h.base, "POST", "/search", { query: q, limit: cfg.searchLimit }, 120000));
add("search_compact", await http(h.base, "POST", "/search", { query: q, limit: cfg.searchLimit, format: "compact" }, 120000));
add("search_narrative", await http(h.base, "POST", "/search", { query: q, limit: cfg.searchLimit, format: "narrative" }, 120000));
add("smart_search", await http(h.base, "POST", "/smart-search", { query: q, limit: cfg.searchLimit }, 120000));
}
const ctx = await http(h.base, "POST", "/context", { sessionId: sessionIdFor(profile, n, 0), project: PROJECT, budget: cfg.contextBudget }, 120000);
add("context", ctx);
const summarize = (k: string) => {
const r = rows[k]!;
return {
calls: r.bytes.length,
bytes: dist(r.bytes),
serverTokenEstimate: r.tokens.length ? dist(r.tokens) : "unknown",
latencyMs: dist(r.ms),
results: dist(r.results),
};
};
const hookDir = join(cfg.distDir, "hooks");
const firstRead = [...caps.values()].find((c) => c.toolName === "Read");
const preTool = firstRead
? { session_id: sessionIdFor(profile, n, 0), cwd: h.cwd, hook_event_name: "PreToolUse", tool_name: "Read", tool_input: firstRead.toolInput }
: { session_id: sessionIdFor(profile, n, 0), cwd: h.cwd, hook_event_name: "PreToolUse", tool_name: "Read", tool_input: { file_path: "src/cache/index.ts" } };
const hooks: Record<string, HookRun> = {};
for (const inject of [false, true]) {
const tag = inject ? "on" : "off";
hooks[`session_start_inject_${tag}`] = await runHook(
join(hookDir, "session-start.mjs"),
{ session_id: `cc-${profile.name}-${n}-ctx-${tag}`, cwd: h.cwd, hook_event_name: "SessionStart", source: "startup" },
h.hookEnv(inject),
);
hooks[`pre_tool_use_inject_${tag}`] = await runHook(join(hookDir, "pre-tool-use.mjs"), preTool, h.hookEnv(inject));
}
return {
declaredContextBudgetTokens: cfg.contextBudget,
searchLimit: cfg.searchLimit,
queries,
endpoints: Object.fromEntries(Object.keys(rows).map((k) => [k, summarize(k)])),
hookStdout: Object.fromEntries(Object.entries(hooks).map(([k, v]) => [k, { stdoutBytes: v.stdoutBytes, ms: Math.round(v.ms), exitCode: v.exitCode }])),
};
}
async function redisCli(url: string, args: string[]): Promise<string> {
const { stdout } = await execFileAsync("redis-cli", ["-u", url, ...args], { timeout: 60000, maxBuffer: 16 * 1024 * 1024 });
return stdout;
}
async function redisStats(url: string): Promise<{ usedMemoryBytes: number | null; keys: number | null; serverRssKiB: number | null }> {
const info = await redisCli(url, ["INFO"]);
const field = (k: string) => {
const m = info.match(new RegExp(`^${k}:(\\d+)`, "m"));
return m ? +m[1]! : null;
};
const keys = parseInt((await redisCli(url, ["DBSIZE"])).trim(), 10);
const pid = field("process_id");
const rows = pid === null ? [] : await psRows();
return { usedMemoryBytes: field("used_memory"), keys: Number.isFinite(keys) ? keys : null, serverRssKiB: rows.find((r) => r.pid === pid)?.rssKiB ?? null };
}
async function providerProcessRssKiB(): Promise<number> {
return (await psRows()).filter((r) => basename(r.comm).startsWith("ollama")).reduce((s, r) => s + r.rssKiB, 0);
}
async function prepareRun(cfg: Config, profile: Profile, n: number): Promise<{ runDir: string; variant: string; out: Record<string, unknown> }> {
const variant = variantName(profile, cfg);
const runDir = join(cfg.root, `${variant.replace(/[^a-z0-9-]+/gi, "_")}-${n}`);
rmSync(runDir, { recursive: true, force: true });
mkdirSync(runDir, { recursive: true });
if (cfg.stateBackend === "redis") await redisCli(cfg.redisUrl!, ["FLUSHALL"]);
const out: Record<string, unknown> = {
profile: profile.name,
variant,
profileDescription: profile.description,
observations: n,
provider: cfg.provider,
stateBackend: cfg.stateBackend,
scenario: cfg.scenario,
embeddings: profile.embeddings,
};
return { runDir, variant, out };
}
async function waitDrained(h: Harness, fake: FakeProvider, embeddings: boolean, n: number, timeoutMs: number): Promise<void> {
const t0 = Date.now();
while (Date.now() - t0 < timeoutMs) {
const status = await http(h.base, "GET", "/status", undefined, 120000).catch(() => null);
const index = (status?.body as { index?: Record<string, unknown> } | undefined)?.index ?? {};
const vectors = typeof index["vectorDocuments"] === "number" ? (index["vectorDocuments"] as number) : 0;
if (h.spoolRecords() === 0 && (!embeddings || vectors >= n)) break;
await sleep(1000);
}
await quiesce(h, fake, Math.max(10000, timeoutMs - (Date.now() - t0)));
}
async function runCrashOffline(cfg: Config, profile: Profile, n: number, fake: FakeProvider) {
const { runDir, variant, out } = await prepareRun(cfg, profile, n);
const h = new Harness(cfg, profile, runDir);
h.prepare();
fake.reset();
const log = (msg: string) => process.stderr.write(`[capture-costs] ${variant} n=${n} ${msg}\n`);
try {
log("cold start");
out["coldStart"] = await h.start("cold");
const rng = mulberry32(cfg.seed);
const caps = new Map<number, ToolCapture>();
for (let i = 0; i < n; i++) {
const size = cfg.outputMin + Math.floor(rng() * Math.max(1, cfg.outputMax - cfg.outputMin));
caps.set(i, buildToolCapture(rng, i, size));
}
const sessions = Math.ceil(n / cfg.perSession);
for (let j = 0; j < sessions; j++) {
await http(h.base, "POST", "/session/start", { sessionId: sessionIdFor(profile, n, j), project: PROJECT, cwd: h.cwd });
}
const hookScript = join(cfg.distDir, "hooks", "post-tool-use.mjs");
const killCount = Math.max(1, Math.floor(n / 2));
let done = 0;
let inFlight = 0;
let killedAtMs: number | null = null;
let gate: Promise<void> | null = null;
let openGate = () => {};
const online: number[] = [];
const offline: number[] = [];
let nonZero = 0;
const t0 = performance.now();
log(`capture through hooks, kill after ${killCount} hooks or ${cfg.killAfterMs} ms, settle ${cfg.killSettleMs} ms`);
const killer = (async () => {
while (done < killCount && performance.now() - t0 < cfg.killAfterMs) await sleep(10);
if (cfg.killSettleMs > 0) {
gate = new Promise<void>((ok) => (openGate = ok));
while (inFlight > 0) await sleep(10);
await sleep(cfg.killSettleMs);
}
const status = await http(h.base, "GET", "/status", undefined, 5000).catch(() => null);
const persistence = (status?.body as { indexPersistence?: { vector?: { lastSavedAt?: string | null } | null } } | undefined)?.indexPersistence ?? null;
const embedded = fake.snapshot();
const completed = done;
killedAtMs = performance.now() - t0;
await h.forceKill();
openGate();
const accepted = (readFileSync(join(runDir, "cli-cold.log"), "utf8").match(/Observation captured/g) ?? []).length;
out["crash"] = {
killAfterMs: Math.round(killedAtMs),
settleMs: cfg.killSettleMs,
hooksCompletedBeforeKill: completed,
observationsAcceptedBeforeKill: accepted,
embeddedInputsBeforeKill: embedded.embedInputs,
chatRequestsBeforeKill: embedded.chatRequests,
vectorCheckpointBeforeKill: persistence?.vector?.lastSavedAt ? 1 : 0,
indexPersistenceBeforeKill: persistence,
};
})();
await drive(cfg.concurrency, n, async (i) => {
if (gate) await gate;
inFlight++;
const start = performance.now() - t0;
const r = await runHook(hookScript, hookBody(sessionIdFor(profile, n, Math.floor(i / cfg.perSession)), h.cwd, caps.get(i)!), h.hookEnv(false), 15000, h.cwd);
(killedAtMs === null || start < killedAtMs ? online : offline).push(r.ms);
if (r.exitCode !== 0) nonZero++;
done++;
inFlight--;
});
await killer;
const spooled = h.spoolRecords();
Object.assign(out["crash"] as Record<string, unknown>, {
captureWallMs: Math.round(performance.now() - t0),
hookLatencyOnlineMs: dist(online),
hookLatencyOfflineMs: dist(offline),
hookNonZeroExits: nonZero,
spoolRecordsWhileDown: spooled,
spoolStats: h.spoolStats(),
});
log(`${spooled} observations spooled while down; restart`);
const recBefore = fake.snapshot();
const recT0 = performance.now();
const rec = await h.start("recovery");
await h.sampler.begin();
await waitDrained(h, fake, profile.embeddings, n, cfg.quiesceTimeoutMs);
const recRss = await h.sampler.end();
const recProvider = diffCounters(recBefore, fake.snapshot());
out["recovery"] = {
...rec,
boot: h.bootLog() ?? rec.boot,
settledMs: Math.round(performance.now() - recT0),
spoolRecordsAfter: h.spoolRecords(),
spoolStatsAfter: h.spoolStats(),
provider: recProvider,
rss: { peakTotalKiB: recRss.peakTotalKiB },
cpuSec: recRss.cpuSec,
};
const sampleIdx = Array.from({ length: Math.min(cfg.evidenceSample, n) }, (_, k) => Math.floor((k * n) / Math.min(cfg.evidenceSample, n)));
log("evidence after recovery");
out["evidenceAfterRecovery"] = await collectEvidence(h, profile, n, cfg.perSession, caps, sampleIdx);
await h.stop();
} catch (err) {
out["error"] = err instanceof Error ? err.message : String(err);
log(`error: ${out["error"]}`);
} finally {
h.sampler.stop();
await h.stop().catch(() => {});
if (!cfg.keep) rmSync(runDir, { recursive: true, force: true });
}
return out;
}
function kib(n: number): number {
return Math.round(n);
}
async function runOne(cfg: Config, profile: Profile, n: number, fake: FakeProvider) {
const { runDir, variant, out } = await prepareRun(cfg, profile, n);
const h = new Harness(cfg, profile, runDir);
h.prepare();
fake.reset();
const log = (msg: string) => process.stderr.write(`[capture-costs] ${variant} n=${n} ${msg}\n`);
try {
log("cold start");
const cold = await h.start("cold");
const baseline = await settledDisk(h.dataDir, 3000, 30000);
const idleEmpty = await h.sampler.sampleOnce();
out["coldStart"] = { ...cold, rssKiB: { total: idleEmpty.totalKiB, engine: idleEmpty.engineKiB, worker: idleEmpty.workerKiB, processes: idleEmpty.processKiB } };
out["diskFixed"] = baseline;
const rng = mulberry32(cfg.seed);
const caps = new Map<number, ToolCapture>();
for (let i = 0; i < n; i++) {
const size = cfg.outputMin + Math.floor(rng() * Math.max(1, cfg.outputMax - cfg.outputMin));
caps.set(i, buildToolCapture(rng, i, size));
}
const sessions = Math.ceil(n / cfg.perSession);
const hookDir = join(cfg.distDir, "hooks");
const before = fake.snapshot();
log(`capture ${n} observations in ${sessions} sessions`);
await h.sampler.begin();
const sessionStart = await runHook(
join(hookDir, "session-start.mjs"),
{ session_id: sessionIdFor(profile, n, 0), cwd: h.cwd, hook_event_name: "SessionStart", source: "startup" },
h.hookEnv(false),
);
for (let j = 1; j < sessions; j++) {
await http(h.base, "POST", "/session/start", { sessionId: sessionIdFor(profile, n, j), project: PROJECT, cwd: h.cwd });
}
const hookCount = Math.min(cfg.hookSample, n);
const hookRuns: HookRun[] = [];
const captureT0 = performance.now();
for (let i = 0; i < hookCount; i++) {
const cap = caps.get(i)!;
hookRuns.push(await runHook(join(hookDir, "post-tool-use.mjs"), hookBody(sessionIdFor(profile, n, Math.floor(i / cfg.perSession)), h.cwd, cap), h.hookEnv(false)));
}
const observeMs: number[] = [];
let observeErrors = 0;
let deduplicated = 0;
const errorSamples: string[] = [];
await drive(cfg.concurrency, n - hookCount, async (k) => {
const i = hookCount + k;
const cap = caps.get(i)!;
try {
const r = await http(h.base, "POST", "/observe", observeBody(sessionIdFor(profile, n, Math.floor(i / cfg.perSession)), h.cwd, cap), 60000);
observeMs.push(r.ms);
if (r.status >= 400) {
observeErrors++;
if (errorSamples.length < 5) errorSamples.push(`HTTP ${r.status}: ${r.text.slice(0, 160)}`);
}
if ((r.body as { deduplicated?: boolean })?.deduplicated) deduplicated++;
} catch (err) {
observeErrors++;
if (errorSamples.length < 5) errorSamples.push(err instanceof Error ? err.message : String(err));
}
});
const captureWallMs = performance.now() - captureT0;
const q = await quiesce(h, fake, cfg.quiesceTimeoutMs);
const captureRss = await h.sampler.end();
const afterCapture = fake.snapshot();
out["capture"] = {
wallMs: Math.round(captureWallMs),
quiesceMs: q.ms,
hookSample: hookCount,
sessionStartHook: { ms: Math.round(sessionStart.ms), stdoutBytes: sessionStart.stdoutBytes, exitCode: sessionStart.exitCode },
hookLatencyMs: dist(hookRuns.map((r) => r.ms)),
hookStdoutBytes: { total: hookRuns.reduce((s, r) => s + r.stdoutBytes, 0), max: Math.max(0, ...hookRuns.map((r) => r.stdoutBytes)) },
hookStderrBytes: hookRuns.reduce((s, r) => s + r.stderrBytes, 0),
hookNonZeroExits: hookRuns.filter((r) => r.exitCode !== 0).length,
observeConcurrency: cfg.concurrency,
observeLatencyMs: dist(observeMs),
observeErrors,
observeErrorSamples: errorSamples,
deduplicatedResponses: deduplicated,
rss: { peakTotalKiB: kib(captureRss.peakTotalKiB), peakEngineKiB: kib(captureRss.peakEngineKiB), peakWorkerKiB: kib(captureRss.peakWorkerKiB), peakProcessKiB: captureRss.peakProcessKiB, meanTotalKiB: captureRss.meanTotalKiB, samples: captureRss.samples },
cpuSec: captureRss.cpuSec,
provider: diffCounters(before, afterCapture),
};
log("idle");
await sleep(5000);
const idle = await h.sampler.sampleOnce();
out["idle"] = { rssKiB: { total: idle.totalKiB, engine: idle.engineKiB, worker: idle.workerKiB, processes: idle.processKiB } };
out["diskAfterCapture"] = q.disk;
out["diskGrowth"] = diskDelta(baseline, q.disk);
out["diskGrowthPerObservationBytes"] = n > 0 ? Math.round(diskDelta(baseline, q.disk).totalBytes / n) : null;
if (cfg.stateBackend === "redis") out["redisAfterCapture"] = await redisStats(cfg.redisUrl!);
if (cfg.provider !== "fake") out["providerProcessRssKiB"] = await providerProcessRssKiB();
const sampleIdx = Array.from({ length: Math.min(cfg.evidenceSample, n) }, (_, k) => Math.floor((k * n) / Math.min(cfg.evidenceSample, n)));
log("evidence before kill");
out["evidenceBeforeKill"] = await collectEvidence(h, profile, n, cfg.perSession, caps, sampleIdx);
log("context cost");
const ctxBefore = fake.snapshot();
out["agentVisibleContext"] = await measureContext(h, profile, n, caps, cfg);
out["agentVisibleContextProvider"] = diffCounters(ctxBefore, fake.snapshot());
log("force kill and recover");
await h.forceKill();
const recBefore = fake.snapshot();
const recT0 = performance.now();
const rec = await h.start("recovery");
await h.sampler.begin();
const rq = await quiesce(h, fake, cfg.quiesceTimeoutMs);
const recRss = await h.sampler.end();
const recProvider = diffCounters(recBefore, fake.snapshot());
await sleep(3000);
const recIdle = await h.sampler.sampleOnce();
out["recovery"] = {
...rec,
bootAfterSettle: h.bootLog(),
settledMs: Math.round(performance.now() - recT0),
reEmbedRequests: recProvider.embedRequests,
reEmbeddedInputs: recProvider.embedInputs,
provider: recProvider,
rss: { peakTotalKiB: recRss.peakTotalKiB, idleTotalKiB: recIdle.totalKiB, idleEngineKiB: recIdle.engineKiB, idleWorkerKiB: recIdle.workerKiB, peakProcessKiB: recRss.peakProcessKiB, idleProcessKiB: recIdle.processKiB },
cpuSec: recRss.cpuSec,
disk: rq.disk,
};
log("evidence after recovery");
out["evidenceAfterRecovery"] = await collectEvidence(h, profile, n, cfg.perSession, caps, sampleIdx);
log("graceful stop and warm start");
await h.stop();
const warmBefore = fake.snapshot();
const warm = await h.start("warm");
await quiesce(h, fake, Math.min(cfg.quiesceTimeoutMs, 120000));
const warmIdle = await h.sampler.sampleOnce();
out["warmStart"] = { ...warm, provider: diffCounters(warmBefore, fake.snapshot()), rssKiB: { total: warmIdle.totalKiB, engine: warmIdle.engineKiB, worker: warmIdle.workerKiB, processes: warmIdle.processKiB } };
await h.stop();
out["diskFinal"] = await settledDisk(h.dataDir, 2000, 20000);
} catch (err) {
out["error"] = err instanceof Error ? err.message : String(err);
log(`error: ${out["error"]}`);
} finally {
h.sampler.stop();
await h.stop().catch(() => {});
if (!cfg.keep) rmSync(runDir, { recursive: true, force: true });
}
return out;
}
function gitSha(): string {
try {
return execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 }).trim();
} catch {
return "unknown";
}
}
function fmtBytes(n: unknown): string {
if (typeof n !== "number" || !Number.isFinite(n)) return "n/a";
if (Math.abs(n) >= 1024 * 1024) return `${(n / 1024 / 1024).toFixed(2)} MiB`;
if (Math.abs(n) >= 1024) return `${(n / 1024).toFixed(1)} KiB`;
return `${n} B`;
}
function fmtKiB(n: unknown): string {
return typeof n === "number" ? `${(n / 1024).toFixed(0)} MiB` : "n/a";
}
function get(o: unknown, path: string): unknown {
return path.split(".").reduce<unknown>((acc, k) => (acc && typeof acc === "object" ? (acc as Record<string, unknown>)[k] : undefined), o);
}
function markdown(report: Record<string, unknown>): string {
const repeated = ((report["config"] as { repeats?: number })?.repeats ?? 1) > 1;
const all = (report["runs"] as Record<string, unknown>[]).map(
(r): Record<string, unknown> => ({ ...r, profile: repeated ? `${variantOf(r)} #${r["repeat"]}` : variantOf(r) }),
);
const runs = all.filter((r) => (r["scenario"] ?? "standard") === "standard");
const crashes = all.filter((r) => r["scenario"] === "crash-offline");
const lines: string[] = [];
const env = report["environment"] as Record<string, unknown>;
lines.push(`Commit \`${report["commit"]}\`, ${env["platform"]}, ${env["cpu"]}, ${fmtBytes(env["memoryBytes"])} RAM, Node ${env["node"]}, iii ${env["iii"]}.`, "");
lines.push("### Disk retention", "");
lines.push("| profile | N | logical obs | fixed | after capture | growth/obs | source | summary | index | diagnostic | config | stream | queue | failed delivery |");
lines.push("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|");
for (const r of runs) {
const g = (k: string) => fmtBytes(get(r, `diskAfterCapture.byCategory.${k}`));
lines.push(`| ${r["profile"]} | ${r["observations"]} | ${get(r, "evidenceBeforeKill.logicalObservations") ?? "n/a"} | ${fmtBytes(get(r, "diskFixed.totalBytes"))} | ${fmtBytes(get(r, "diskAfterCapture.totalBytes"))} | ${fmtBytes(r["diskGrowthPerObservationBytes"])} | ${g("source")} | ${g("summary")} | ${g("index")} | ${g("diagnostic")} | ${g("config")} | ${g("stream")} | ${g("queue")} | ${g("failed_delivery")} |`);
}
lines.push("", "### Resident memory and CPU (iii Compose process tree)", "");
lines.push("| profile | N | empty idle | capture peak | engine peak | Compose | state | http | agentmemory | worker peak | idle after capture | after recovery | capture CPU s | recovery CPU s |");
lines.push("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|");
for (const r of runs) {
const role = (name: ProcessRole) => fmtKiB(get(r, `capture.rss.peakProcessKiB.${name}`));
lines.push(`| ${r["profile"]} | ${r["observations"]} | ${fmtKiB(get(r, "coldStart.rssKiB.total"))} | ${fmtKiB(get(r, "capture.rss.peakTotalKiB"))} | ${fmtKiB(get(r, "capture.rss.peakEngineKiB"))} | ${role("compose")} | ${role("state")} | ${role("http")} | ${role("agentmemory")} | ${fmtKiB(get(r, "capture.rss.peakWorkerKiB"))} | ${fmtKiB(get(r, "idle.rssKiB.total"))} | ${fmtKiB(get(r, "recovery.rss.idleTotalKiB"))} | ${get(r, "capture.cpuSec") ?? "n/a"} | ${get(r, "recovery.cpuSec") ?? "n/a"} |`);
}
lines.push("", "### Capture latency", "");
lines.push("| profile | N | hook p50 ms | hook p95 ms | hook stdout B | observe p50 ms | observe p95 ms | observe p99 ms | errors | capture wall ms | quiesce ms |");
lines.push("|---|---|---|---|---|---|---|---|---|---|---|");
for (const r of runs) {
lines.push(`| ${r["profile"]} | ${r["observations"]} | ${get(r, "capture.hookLatencyMs.p50")} | ${get(r, "capture.hookLatencyMs.p95")} | ${get(r, "capture.hookStdoutBytes.total")} | ${get(r, "capture.observeLatencyMs.p50")} | ${get(r, "capture.observeLatencyMs.p95")} | ${get(r, "capture.observeLatencyMs.p99")} | ${get(r, "capture.observeErrors")} | ${get(r, "capture.wallMs")} | ${get(r, "capture.quiesceMs")} |`);
}
lines.push("", "### Startup and recovery", "");
lines.push("| profile | N | cold ready ms | warm ready ms | recovery ready ms | recovery BM25 rebuild ms (docs) | vectors loaded | vector docs before / after | backfill queued / awaiting opt-in | re-embed requests (inputs) | logical obs before / after | marker hits before / after | source tail before / after |");
lines.push("|---|---|---|---|---|---|---|---|---|---|---|---|---|");
for (const r of runs) {
lines.push(`| ${r["profile"]} | ${r["observations"]} | ${get(r, "coldStart.readyMs")} | ${get(r, "warmStart.readyMs")} | ${get(r, "recovery.readyMs")} | ${get(r, "recovery.boot.bm25RebuildMs")} (${get(r, "recovery.boot.bm25Docs")}) | ${get(r, "recovery.boot.vectorsLoaded")} | ${get(r, "evidenceBeforeKill.vectorDocuments")} / ${get(r, "evidenceAfterRecovery.vectorDocuments")} | ${get(r, "recovery.boot.vectorBackfillQueued")} / ${get(r, "recovery.boot.vectorBackfillAwaitingOptIn")} | ${get(r, "recovery.reEmbedRequests")} (${get(r, "recovery.reEmbeddedInputs")}) | ${get(r, "evidenceBeforeKill.logicalObservations")} / ${get(r, "evidenceAfterRecovery.logicalObservations")} | ${get(r, "evidenceBeforeKill.markerSearchHits")}/${get(r, "evidenceBeforeKill.sampled")} / ${get(r, "evidenceAfterRecovery.markerSearchHits")}/${get(r, "evidenceAfterRecovery.sampled")} | ${get(r, "evidenceBeforeKill.sourceTailRetained")} / ${get(r, "evidenceAfterRecovery.sourceTailRetained")} |`);
}
lines.push("", "### Provider calls (OpenAI-compatible endpoint seen by the bench)", "");
lines.push("| profile | N | capture embed req (inputs, repeats) | capture chat req (repeats) | chat prompt tok | chat completion tok | upstream embed s | upstream chat s | provider errors | context-phase embed req | recovery embed req | warm-start embed req |");
lines.push("|---|---|---|---|---|---|---|---|---|---|---|---|");
for (const r of runs) {
const p = (k: string) => get(r, `capture.provider.${k}`);
const secs = (k: string) => (typeof p(k) === "number" ? ((p(k) as number) / 1000).toFixed(1) : "n/a");
lines.push(`| ${r["profile"]} | ${r["observations"]} | ${p("embedRequests")} (${p("embedInputs")}, ${p("embedRepeatInputs")}) | ${p("chatRequests")} (${p("chatRepeatPrompts")}) | ${p("chatPromptTokens")} | ${p("chatCompletionTokens")} | ${secs("embedUpstreamMs")} | ${secs("chatUpstreamMs")} | ${p("errors")} | ${get(r, "agentVisibleContextProvider.embedRequests")} | ${get(r, "recovery.provider.embedRequests")} | ${get(r, "warmStart.provider.embedRequests")} |`);
}
const redisRuns = runs.filter((r) => r["redisAfterCapture"]);
if (redisRuns.length > 0) {
lines.push("", "### Redis state store after capture", "");
lines.push("| profile | N | used memory | keys | redis-server RSS |", "|---|---|---|---|---|");
for (const r of redisRuns) lines.push(`| ${r["profile"]} | ${r["observations"]} | ${fmtBytes(get(r, "redisAfterCapture.usedMemoryBytes"))} | ${get(r, "redisAfterCapture.keys")} | ${fmtKiB(get(r, "redisAfterCapture.serverRssKiB"))} |`);
}
if (crashes.length > 0) {
lines.push("", "### Force-kill during capture with the service down", "");
lines.push("| profile | N | killed at ms | hooks before kill | embedded before kill | checkpoint before kill | spooled while down | offline hook p95 ms | spool recovered (dup) | pending-log vectors | recovery embed inputs (repeats) | logical obs after | vectors after | spool left |");
lines.push("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|");
for (const r of crashes) {
lines.push(`| ${r["profile"]} | ${r["observations"]} | ${get(r, "crash.killAfterMs")} | ${get(r, "crash.hooksCompletedBeforeKill")} | ${get(r, "crash.embeddedInputsBeforeKill")} | ${get(r, "crash.vectorCheckpointBeforeKill") ? "yes" : "no"} | ${get(r, "crash.spoolRecordsWhileDown")} | ${get(r, "crash.hookLatencyOfflineMs.p95")} | ${get(r, "recovery.boot.spoolRecovered")} (${get(r, "recovery.boot.spoolAlreadyStored")}) | ${get(r, "recovery.boot.pendingLogRecoveredVectors")} | ${get(r, "recovery.provider.embedInputs")} (${get(r, "recovery.provider.embedRepeatInputs")}) | ${get(r, "evidenceAfterRecovery.logicalObservations")} | ${get(r, "evidenceAfterRecovery.vectorDocuments")} | ${get(r, "recovery.spoolRecordsAfter")} |`);
}
}
lines.push("", "### Agent-visible context (server token estimate, not provider billing)", "");
lines.push("| profile | N | search full B p50 (tok) | compact B p50 (tok) | narrative B p50 (tok) | smart-search B p50 | context B (tok) | SessionStart stdout off / on | PreToolUse stdout off / on |");
lines.push("|---|---|---|---|---|---|---|---|---|");
for (const r of runs) {
const e = (k: string, f: string) => get(r, `agentVisibleContext.endpoints.${k}.${f}`);
const tok = (k: string) => {
const t = e(k, "serverTokenEstimate");
return typeof t === "object" && t ? (t as { p50: number }).p50 : "unknown";
};
const hs = (k: string) => get(r, `agentVisibleContext.hookStdout.${k}.stdoutBytes`);
lines.push(`| ${r["profile"]} | ${r["observations"]} | ${e("search_full", "bytes.p50")} (${tok("search_full")}) | ${e("search_compact", "bytes.p50")} (${tok("search_compact")}) | ${e("search_narrative", "bytes.p50")} (${tok("search_narrative")}) | ${e("smart_search", "bytes.p50")} | ${e("context", "bytes.p50")} (${tok("context")}) | ${hs("session_start_inject_off")} / ${hs("session_start_inject_on")} | ${hs("pre_tool_use_inject_off")} / ${hs("pre_tool_use_inject_on")} |`);
}
const checks: [string, Check[]][] = [["Invariants (evidence completeness and default-off injection)", (report["invariants"] as Check[]) ?? []], ["Budgets", ((report["budgets"] as { checks?: Check[] } | undefined)?.checks ?? [])]];
for (const [title, list] of checks) {
if (list.length === 0) continue;
const failed = list.filter((c) => !c.pass);
lines.push("", `### ${title}`, "", `${list.length - failed.length}/${list.length} checks pass.`);
if (failed.length > 0) {
lines.push("", "| profile | N | repeat | check | value | limit |", "|---|---|---|---|---|---|");
for (const c of failed) lines.push(`| ${c.profile} | ${c.observations} | ${c.repeat} | ${c.name} | ${c.value} | ${c.limit} |`);
}
}
const errors = all.filter((r) => r["error"]);
if (errors.length > 0) {
lines.push("", "### Errors", "");
for (const r of errors) lines.push(`- ${r["profile"]} N=${r["observations"]}: ${r["error"]}`);
}
return lines.join("\n") + "\n";
}
function mergeReports(paths: string[], outDir: string, budgetsPath: string | null, writeBudgetsPath: string | null): void {
const reports = paths.map((p) => JSON.parse(readFileSync(p, "utf8")) as Record<string, unknown>);
const first = reports[0]!;
const runs = reports.flatMap((r) => r["runs"] as Record<string, unknown>[]);
const commits = new Set(reports.map((r) => r["commit"]));
if (commits.size > 1) throw new Error(`refusing to merge reports from different commits: ${[...commits].join(", ")}`);
const merged: Record<string, unknown> = {
...first,
finished: reports.map((r) => String(r["finished"])).sort().pop(),
config: { ...(first["config"] as object), sizes: [...new Set(runs.map((r) => r["observations"] as number))], repeats: Math.max(...reports.map((r) => ((r["config"] as { repeats?: number })?.repeats ?? 1))) },
mergedFrom: paths.map((p) => basename(p)),
runs,
};
merged["invariants"] = checkInvariants(runs);
if (budgetsPath && existsSync(budgetsPath)) {
merged["budgets"] = { file: basename(budgetsPath), checks: checkBudgets(runs, JSON.parse(readFileSync(budgetsPath, "utf8")) as BudgetFile) };
}
if (writeBudgetsPath) writeFileSync(writeBudgetsPath, JSON.stringify(deriveBudgets(runs), null, 2) + "\n");
const short = String(first["commit"]).slice(0, 7);
writeFileSync(join(outDir, `capture-costs-${short}.json`), JSON.stringify(merged, null, 2) + "\n");
writeFileSync(join(outDir, `capture-costs-${short}.md`), markdown(merged));
process.stdout.write(markdown(merged));
}
async function main(): Promise<void> {
if (process.env["BENCH_MERGE"]) {
const outDir = process.env["BENCH_OUT_DIR"] || resolve(process.cwd(), "benchmark", "results");
mkdirSync(outDir, { recursive: true });
const budgets = process.env["BENCH_BUDGETS"] === "off" ? null : resolve(process.cwd(), process.env["BENCH_BUDGETS"] || join("benchmark", "capture-costs-budgets.json"));
const write = process.env["BENCH_WRITE_BUDGETS"] ? resolve(process.cwd(), process.env["BENCH_WRITE_BUDGETS"]) : null;
mergeReports(process.env["BENCH_MERGE"].split(",").map((p) => resolve(process.cwd(), p.trim())), outDir, budgets, write);
return;
}
const cfg = loadConfig();
if (!existsSync(join(cfg.distDir, "cli.mjs")) || !existsSync(join(cfg.distDir, "hooks", "post-tool-use.mjs"))) {
throw new Error("dist/ is missing; run npm run build first");
}
mkdirSync(cfg.root, { recursive: true });
mkdirSync(cfg.outDir, { recursive: true });
const fake = new FakeProvider(cfg.fakePort, cfg.dims, cfg.upstreamUrl);
await fake.start();
const commit = gitSha();
const started = new Date().toISOString();
const runs: Record<string, unknown>[] = [];
try {
for (const n of cfg.sizes) {
for (let rep = 1; rep <= cfg.repeats; rep++) {
for (const profile of cfg.profiles) {
const t0 = performance.now();
const r = cfg.scenario === "crash-offline" ? await runCrashOffline(cfg, profile, n, fake) : await runOne(cfg, profile, n, fake);
r["repeat"] = rep;
r["runWallMs"] = Math.round(performance.now() - t0);
runs.push(r);
}
}
}
} finally {
await fake.stop();
if (!cfg.keep) rmSync(cfg.root, { recursive: true, force: true });
}
const report: Record<string, unknown> = {
schema_version: SCHEMA_VERSION,
commit,
started,
finished: new Date().toISOString(),
environment: {
platform: `${platform()} ${release()} ${arch()}`,
cpu: `${cpus()[0]?.model ?? "unknown"} x${cpus().length}`,
memoryBytes: totalmem(),
node: process.version,
iii: iiiVersion(cfg.iiiBin),
stateBackend: cfg.stateBackend,
...(cfg.stateBackend === "redis" ? { redis: (await redisCli(cfg.redisUrl!, ["INFO", "server"])).match(/^redis_version:(\S+)/m)?.[1] ?? null } : {}),
},
config: {
sizes: cfg.sizes,
profiles: cfg.profiles.map((p) => ({ name: p.name, description: p.description })),
observationsPerSession: cfg.perSession,
hookSample: cfg.hookSample,
observeConcurrency: cfg.concurrency,
toolOutputBytes: [cfg.outputMin, cfg.outputMax],
seed: cfg.seed,
embeddingDimensions: cfg.dims,
contextBudgetTokens: cfg.contextBudget,
searchLimit: cfg.searchLimit,
evidenceSample: cfg.evidenceSample,
repeats: cfg.repeats,
scenario: cfg.scenario,
provider: cfg.provider === "fake" ? { mode: "fake" } : { mode: cfg.provider, url: cfg.upstreamUrl, embeddingModel: cfg.embedModel, chatModel: cfg.chatModel },
...(cfg.scenario === "crash-offline" ? { killAfterMs: cfg.killAfterMs, killSettleMs: cfg.killSettleMs } : {}),
ports: { rest: cfg.restPort, viewer: cfg.restPort + 2, metrics: cfg.restPort + 6353, engine: cfg.enginePort, fakeProvider: cfg.fakePort },
},
runs,
};
report["invariants"] = checkInvariants(runs);
if (cfg.budgetsPath && existsSync(cfg.budgetsPath)) {
report["budgets"] = { file: cfg.budgetsPath, checks: checkBudgets(runs, JSON.parse(readFileSync(cfg.budgetsPath, "utf8")) as BudgetFile) };
}
if (cfg.writeBudgetsPath) {
writeFileSync(cfg.writeBudgetsPath, JSON.stringify(deriveBudgets(runs), null, 2) + "\n");
process.stderr.write(`[capture-costs] wrote budgets to ${cfg.writeBudgetsPath}\n`);
}
const short = commit === "unknown" ? `nogit-${Date.now().toString(36)}` : commit.slice(0, 7);
const jsonPath = join(cfg.outDir, `capture-costs-${short}.json`);
const mdPath = join(cfg.outDir, `capture-costs-${short}.md`);
writeFileSync(jsonPath, JSON.stringify(report, null, 2) + "\n");
writeFileSync(mdPath, markdown(report));
process.stdout.write(markdown(report));
process.stderr.write(`[capture-costs] wrote ${jsonPath} and ${mdPath}\n`);
const failedInvariants = (report["invariants"] as Check[]).some((c) => !c.pass);
const budgetChecks = ((report["budgets"] as { checks?: Check[] } | undefined)?.checks ?? []);
const failedBudgets = budgetChecks.some((c) => !c.pass);
if (runs.some((r) => r["error"]) || failedInvariants) process.exitCode = 1;
if (failedBudgets && cfg.enforceBudgets) process.exitCode = 2;
}
main().catch((err) => {
process.stderr.write(`[capture-costs] ${err instanceof Error ? err.stack ?? err.message : String(err)}\n`);
process.exit(1);
});