#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix(capture): durable observation acceptance, restart-safe dedup and offline spool (#1460)
Rohit Ghumare60b88a2a5dc4
diff --git a/.env.example b/.env.exampleindex fcc077a4..d5376654 100644--- a/.env.example+++ b/.env.example@@ -29,6 +29,16 @@# AGENTMEMORY_CAPTURE_DENY= # Extra tool names or globs to skip, added to the defaults (memory_*, toolsearch, listmcpresources, fetchmcpresource)# AGENTMEMORY_CAPTURE_OUTPUT_MAX=8000 # Max characters of tool output stored per captured observation# AGENTMEMORY_PRE_COMPACT_BUDGET=1500 # Token budget for context injected by the PreCompact hook; 0 disables it+# AGENTMEMORY_CAPTURE_MAX_ATTEMPTS=5 # Attempts before a capture becomes a dead letter+# AGENTMEMORY_CAPTURE_RETRY_INTERVAL_MS=10000 # Retry sweep interval and backoff base+# AGENTMEMORY_CAPTURE_DEDUP_HOURS=168 # How long stored event ids block replays+# AGENTMEMORY_CAPTURE_EVENTS_MAX=100000 # Max stored event ids+# AGENTMEMORY_CAPTURE_INBOX_MAX=10000 # Max unprocessed events before 503+# AGENTMEMORY_CAPTURE_DEAD_MAX=1000 # Max dead letters kept+# AGENTMEMORY_CAPTURE_SPOOL=false # Turn off the hook offline spool+# AGENTMEMORY_CAPTURE_SPOOL_DIR= # Spool folder (default <data dir>/capture-spool)+# AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES=5242880 # Spool size limit+# AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS=168 # Spooled records older than this are dropped# The detection order is OPENAI_API_KEY → MINIMAX_API_KEY → ANTHROPIC_API_KEY# → GEMINI_API_KEY → OPENROUTER_API_KEY → noop.@@ -140,7 +150,7 @@# AGENTMEMORY_AUDIT_MIGRATE_MAX_BYTES=33554432 # Largest legacy mem:audit scope file the one-time migration will read locally. Default 32 MiB; bigger or unreadable stores are left frozen.# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately.# AGENTMEMORY_VECTOR_BUCKET_SIZE=500 # Vectors held per storage bucket before a new bucket opens; a save only rewrites buckets it changed.-# AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Cap on vectors embedded per boot to backfill ones missing since the last save.+# AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Batch size for re-embedding vectors missing since the last save (continues across restarts); also caps the whole-store opt-in per boot.# AGENTMEMORY_VECTOR_BACKFILL=all # Required to backfill an entire store's vectors at once (no prior vector save, or a stale legacy manifest); unset means that case waits for an explicit opt-in.# AGENTMEMORY_VIEWER_STREAM_MAX=500 # Max items kept in the persisted "viewer" live-stream group; older entries are pruned in batches. Floored at 200 (the viewer's reload window); negative values fall back to the defaultdiff --git a/AGENTS.md b/AGENTS.mdindex e65ccef4..2e318fee 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -119,7 +119,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import).## Current Stats (v0.9.29)- 54 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all)-- 135 REST endpoints+- 138 REST endpoints- 6 MCP resources, 3 MCP prompts- 12 hooks, 17 skills- 260+ iii functionsdiff --git a/README.md b/README.mdindex f182581e..1c22f8af 100644--- a/README.md+++ b/README.md@@ -1680,7 +1680,7 @@ Create `~/.agentmemory/.env`:<h2 id="api"><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/section-api.svg"><img src="assets/tags/section-api.svg" alt="API" height="32" /></picture></h2>-135 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer <secret>`, and mesh sync endpoints require an explicitly set `AGENTMEMORY_SECRET` on both peers.+138 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer <secret>`, and mesh sync endpoints require an explicitly set `AGENTMEMORY_SECRET` on both peers.**Authentication is on by default.** When `AGENTMEMORY_SECRET` is not set (in the shell or in `~/.agentmemory/.env`), the server generates a random secret on first start and stores it in `~/.agentmemory/secret` with mode `0600`. Every bundled client reads it from there when it talks to a local server: the CLI, the viewer, the hooks under `plugin/scripts`, the MCP server and the `@agentmemory/mcp` shim, the configs written by `agentmemory connect`, and the bundled OpenCode, Pi, OpenClaw, Hermes and filesystem-watcher integrations. The stored secret is only sent to loopback URLs (`localhost`, `127.0.0.0/8`, `::1`). An explicit `AGENTMEMORY_SECRET` always wins, and remote clients still need it set. Docker and the `deploy/` entrypoints already generate and export their own secret. To call the API by hand:@@ -1702,7 +1702,10 @@ curl -H "Authorization: Bearer $(cat ~/.agentmemory/secret)" http://localhost:31| `GET` | `/agentmemory/health` | Health check (always public) || `POST` | `/agentmemory/session/start` | Start session + get context || `POST` | `/agentmemory/session/end` | End session |-| `POST` | `/agentmemory/observe` | Capture observation |+| `POST` | `/agentmemory/observe` | Capture observation (see capture delivery below) |+| `GET` | `/agentmemory/capture` | Capture inbox, dead letters and offline spool |+| `POST` | `/agentmemory/capture/retry` | Retry dead-letter captures |+| `POST` | `/agentmemory/capture/drain` | Send the local offline spool now || `POST` | `/agentmemory/smart-search` | Hybrid search || `POST` | `/agentmemory/context` | Generate context || `POST` | `/agentmemory/remember` | Save to long-term memory |@@ -1720,6 +1723,20 @@ Full endpoint list: [`src/triggers/api.ts`](src/triggers/api.ts)</details>+**Capture delivery.** Hooks send each observation once to `POST /agentmemory/observe` with an `eventId`. It is the host's own id for the call when the payload has one (for example Claude Code's `tool_use_id`), otherwise a hash of the session, hook type, tool name, input, output and host timestamp. The server writes the event to a capture inbox in the state store, stores the observation, then removes the inbox entry. The status code says what happened:++| Status | `status` field | Meaning |+|---|---|---|+| `201` | `accepted` | Stored. `observationId` is the new observation. |+| `202` | `accepted` (`state: "retrying"`) | Accepted, but storing failed. The server retries it, also after a restart. |+| `200` | `duplicate` | This `eventId` was already accepted. `observationId` is the existing observation; nothing new is stored. |+| `400` / `422` | `rejected` | Invalid payload, or storing failed for good (the event is kept as a dead letter). |+| `503` | `rejected` (`retryable: true`) | The inbox is full (`AGENTMEMORY_CAPTURE_INBOX_MAX`). Hooks spool the event and send it later. |++Failed events are retried every `AGENTMEMORY_CAPTURE_RETRY_INTERVAL_MS` (10 s) with doubling backoff, up to `AGENTMEMORY_CAPTURE_MAX_ATTEMPTS` (5). Events that still fail stay in the inbox as dead letters, are listed on `/agentmemory/status` and the viewer Health page, and can be retried with `POST /agentmemory/capture/retry` (`{"eventId": "..."}` or `{"all": true}`). Accepted event ids are remembered for `AGENTMEMORY_CAPTURE_DEDUP_HOURS` (168 hours, at most `AGENTMEMORY_CAPTURE_EVENTS_MAX` ids), so a hook replayed after a timeout or a restart is stored once, while two separate tool calls with their own host ids are stored twice even when their content is identical. When an observation is deleted (forget, session delete, eviction, auto-forget or an import that replaces the store), its event is marked as deleted before the observation is removed, so a replay of that event inside the same window is answered as a duplicate and stores nothing. The state store writes to disk every 2 seconds, so an answered event can still be only in memory for a moment. To cover that, every `2xx` answer also carries the server's `bootId` (new at every start), `acceptedAt` and `durableAfterMs` (the save interval plus 1.5 s on the file store, 1.5 s on redis, where persistence is the operator's setting). Hooks keep the event in the local spool until that window has passed and delete it on a later call without another request. If the `bootId` has changed by then, the server restarted, so the hook sends the event again with the same `eventId`; an event that did reach the disk is not stored twice. The server also sends such events itself at start and every retry interval, so a restart loses nothing even when no hook runs afterwards. Older hooks ignore the extra fields, and new hooks against an older server discard the event on `2xx` as before.++When the server is down, does not answer in time or returns a 5xx, the hook appends the observation to a local spool file, `<data dir>/capture-spool/<host>-<port>.jsonl` (override the folder with `AGENTMEMORY_CAPTURE_SPOOL_DIR`). The file is private to your user (mode 600), secrets are redacted the same way the server redacts them, it holds at most `AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES` (5 MiB) and drops entries older than `AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS` (168). When it is full, new entries are dropped and counted, and `/agentmemory/status` reports it. The hook still exits 0 within its time limit and adds no request when the server is healthy. The spool is sent at the next start and by the first hook that reaches the server again, in a background process so the agent does not wait. Event ids make this safe: an observation that did arrive before a timeout is not stored twice. `npx @agentmemory/agentmemory capture` shows the spool and the server inbox, `--drain` sends the spool now, and `GET /agentmemory/capture` returns the same as JSON. Set `AGENTMEMORY_CAPTURE_SPOOL=false` to turn the spool off.+**Compacting graph provenance.** Each knowledge graph node and edge keeps the ids of the newest 32 observations it came from. Stores written before that cap can hold thousands of ids per hot node, which makes graph search and the viewer slow or drops the worker. agentmemory fixes this by itself: on the first start after upgrading it trims every node, edge, superseded edge (the temporal graph history) and the cached snapshot to the cap in the background, in small slices with a pause between them, so search, capture and the viewer keep working. It saves its progress, resumes after a restart and never runs again once it has finished. `/agentmemory/status` and the viewer Health page show it as pending, running (with the current scope and position), done or failed. Set `AGENTMEMORY_GRAPH_COMPACT_ON_BOOT=false` to turn it off.To run it by hand, call `POST /agentmemory/graph/compact`. It walks the name and edge-key indexes instead of listing every node and edge, and is safe to re-run. When it trims ids it writes a `graph_compact` audit entry.diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjsnew file mode 100644index 00000000..d6d80934--- /dev/null+++ b/plugin/scripts/_capture.mjs@@ -0,0 +1,885 @@+import { spawn } from "node:child_process";+import { chmodSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs";+import { createHash } from "node:crypto";+import { homedir, platform } from "node:os";+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";+//#region src/secret-store.ts+const SECRET_KEY = "AGENTMEMORY_SECRET";+function agentmemoryHomeDir() {+ return join(homedir(), ".agentmemory");+}+function secretFilePath() {+ return join(agentmemoryHomeDir(), "secret");+}+function usable(value) {+ if (typeof value !== "string") return "";+ const trimmed = value.trim();+ if (!trimmed) return "";+ if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";+ return trimmed;+}+function unquote(value) {+ const quote = value[0];+ if ((quote === "\"" || quote === "'") && value.length > 1) {+ const close = value.indexOf(quote, 1);+ if (close !== -1) return value.slice(1, close);+ }+ const hash = value.indexOf(" #");+ return hash === -1 ? value : value.slice(0, hash).trim();+}+function readEnvFileSecret() {+ let content;+ try {+ content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");+ } catch {+ return "";+ }+ if (typeof content !== "string") return "";+ let found = "";+ for (const line of content.split("\n")) {+ const trimmed = line.trim();+ if (trimmed.startsWith("#")) continue;+ const eq = trimmed.indexOf("=");+ if (eq === -1) continue;+ if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;+ found = usable(unquote(trimmed.slice(eq + 1).trim()));+ }+ return found;+}+function readStoredSecret() {+ try {+ return usable(readFileSync(secretFilePath(), "utf-8"));+ } catch {+ return "";+ }+}+function isLoopbackUrl(url) {+ let hostname;+ try {+ hostname = new URL(url).hostname.toLowerCase();+ } catch {+ return false;+ }+ const bare = hostname.replace(/^\[|\]$/g, "");+ return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);+}+function resolveClientSecret(baseUrl, env = process.env) {+ const fromEnv = usable(env[SECRET_KEY]);+ if (fromEnv) return fromEnv;+ if (!isLoopbackUrl(baseUrl)) return "";+ return readEnvFileSecret() || readStoredSecret();+}+//#endregion+//#region src/capture/event-id.ts+const HOST_ID_FIELDS = [+ "event_id",+ "eventId",+ "tool_use_id",+ "toolUseId",+ "tool_call_id",+ "toolCallId",+ "call_id",+ "callId",+ "prompt_id",+ "promptId",+ "message_id",+ "messageId"+];+const HOST_TIME_FIELDS = [+ "timestamp",+ "ts",+ "created_at",+ "createdAt",+ "event_time"+];+function stableStringify(value) {+ if (value === void 0) return "null";+ if (value === null || typeof value !== "object") {+ const encoded = JSON.stringify(value);+ return encoded === void 0 ? "null" : encoded;+ }+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;+ const obj = value;+ return `{${Object.keys(obj).filter((k) => obj[k] !== void 0).sort().map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(",")}}`;+}+function hostScalar(host, fields) {+ for (const field of fields) {+ const v = host[field];+ if (typeof v === "string" && v.trim()) return v.trim();+ if (typeof v === "number" && Number.isFinite(v)) return String(v);+ }+}+function digest(parts) {+ return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 32);+}+function deriveEventId(hookType, sessionId, host, content) {+ const source = host && typeof host === "object" ? host : {};+ const hostId = hostScalar(source, HOST_ID_FIELDS);+ if (hostId) return `evh_${digest([+ "host",+ sessionId,+ hookType,+ hostId+ ])}`;+ return `evc_${digest([+ "content",+ sessionId,+ hookType,+ hostScalar(source, HOST_TIME_FIELDS) ?? "",+ stableStringify(content)+ ])}`;+}+//#endregion+//#region src/cli-data-dir.ts+function argValue(args, name) {+ const equals = `${name}=`;+ const inline = args.find((arg) => arg.startsWith(equals));+ if (inline) return inline.slice(equals.length);+ const idx = args.indexOf(name);+ if (idx !== -1) return args[idx + 1];+}+function expandHome(pathValue, home) {+ if (pathValue === "~") return home;+ if (pathValue.startsWith("~/") || pathValue.startsWith("~\\")) return join(home, pathValue.slice(2));+ return pathValue;+}+function absoluteDataDir(pathValue, cwd, home) {+ const expanded = expandHome(pathValue, home);+ return isAbsolute(expanded) ? expanded : resolve(cwd, expanded);+}+function platformDefaultDataDir(env, home, nodePlatform, useXdg = true) {+ if (nodePlatform === "darwin") return join(home, "Library", "Application Support", "agentmemory");+ if (nodePlatform === "win32") {+ const appData = env["APPDATA"];+ return appData ? join(appData, "agentmemory") : join(home, ".agentmemory");+ }+ const xdgDataHome = env["XDG_DATA_HOME"];+ if (useXdg && xdgDataHome && isAbsolute(xdgDataHome)) return join(xdgDataHome, "agentmemory");+ return join(home, ".local", "share", "agentmemory");+}+function nearestGitParent(pathValue) {+ let current = pathValue;+ while (true) {+ if (existsSync(join(current, ".git"))) return current;+ const parent = dirname(current);+ if (parent === current) return void 0;+ current = parent;+ }+}+function isWithin(parent, child) {+ const pathFromParent = relative(parent, child);+ return pathFromParent === "" || pathFromParent !== ".." && !pathFromParent.startsWith(`..${sep}`) && !isAbsolute(pathFromParent);+}+function resolveDataDir(options = {}) {+ const args = options.args ?? process.argv.slice(2);+ const env = options.env ?? process.env;+ const cwd = options.cwd ?? process.cwd();+ const home = options.home ?? homedir();+ const nodePlatform = options.platform ?? platform();+ const parsedInstance = parseInt(argValue(args, "--instance") ?? "0", 10);+ const instance = parsedInstance > 0 && parsedInstance <= 50 ? parsedInstance : 0;+ const forInstance = (dataDir) => instance ? join(dataDir, `instance-${instance}`) : dataDir;+ const flagValue = argValue(args, "--data-dir");+ if (flagValue) return {+ dataDir: forInstance(absoluteDataDir(flagValue, cwd, home)),+ source: "flag"+ };+ const envValue = env["AGENTMEMORY_DATA_DIR"];+ if (envValue) return {+ dataDir: forInstance(absoluteDataDir(envValue, cwd, home)),+ source: "env"+ };+ const legacyDir = resolve(cwd, "data");+ const hasLegacyStore = existsSync(join(legacyDir, "state_store.db")) || existsSync(join(legacyDir, "iii-config.yaml"));+ if (instance === 0 && hasLegacyStore) return {+ dataDir: legacyDir,+ source: "default"+ };+ const defaultDir = platformDefaultDataDir(env, home, nodePlatform);+ const gitParent = nearestGitParent(cwd);+ if (gitParent && isWithin(gitParent, defaultDir)) {+ const relocated = platformDefaultDataDir(env, home, nodePlatform, false);+ if (relocated !== defaultDir) return {+ dataDir: forInstance(relocated),+ source: "default",+ relocatedFrom: defaultDir+ };+ }+ return {+ dataDir: forInstance(defaultDir),+ source: "default"+ };+}+//#endregion+//#region src/functions/privacy.ts+const PRIVATE_TAG_RE = /<private>[\s\S]*?<\/private>/gi;+const SECRET_PATTERN_SOURCES = [+ /(?:api[_-]?key|secret|token|password|credential|auth)[\s]*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi,+ /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi,+ /sk-proj-[A-Za-z0-9\-_]{20,}/g,+ /(?:sk|pk|rk|ak)-[A-Za-z0-9][A-Za-z0-9\-_]{19,}/g,+ /sk-ant-[A-Za-z0-9\-_]{20,}/g,+ /gh[pus]_[A-Za-z0-9]{36,}/g,+ /github_pat_[A-Za-z0-9_]{22,}/g,+ /xoxb-[A-Za-z0-9\-]+/g,+ /AKIA[0-9A-Z]{16}/g,+ /AIza[A-Za-z0-9\-_]{35}/g,+ /eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/g,+ /npm_[A-Za-z0-9]{36}/g,+ /glpat-[A-Za-z0-9\-_]{20,}/g,+ /dop_v1_[A-Za-z0-9]{64}/g+];+const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----|$)/g;+const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@:"'<>]*:[^\s/?#@"'<>]+@/gi;+function stripPrivateData(input) {+ let result = input.replace(PRIVATE_TAG_RE, "[REDACTED]");+ result = result.replace(new RegExp(PRIVATE_KEY_BLOCK_RE.source, PRIVATE_KEY_BLOCK_RE.flags), "[REDACTED_SECRET]");+ result = result.replace(new RegExp(URL_CREDENTIALS_RE.source, URL_CREDENTIALS_RE.flags), "$1[REDACTED_SECRET]@");+ for (const source of SECRET_PATTERN_SOURCES) {+ const pattern = new RegExp(source.source, source.flags);+ result = result.replace(pattern, "[REDACTED_SECRET]");+ }+ return result;+}+//#endregion+//#region src/capture/spool.ts+const DEFAULT_MAX_BYTES = 5 * 1024 * 1024;+const MIN_MAX_BYTES = 64 * 1024;+const DEFAULT_MAX_AGE_HOURS = 168;+const LOCK_STALE_MS = 1e4;+const DRAIN_LOCK_STALE_MS = 12e4;+const ORPHAN_CLAIM_MS = 12e4;+const LOCK_WAIT_MS = 500;+const IMAGE_PLACEHOLDER = "[image dropped from capture spool]";+const BOOT_ID_RE = /^[A-Za-z0-9]{8,64}$/;+const SENT_FILE_RE = /^([A-Za-z0-9]{8,64})-(\d+)\.jsonl$/;+const SENT_BUCKET_MS = 1e3;+const MAX_DURABLE_AFTER_MS = 10 * 6e4;+function emptyStats() {+ return {+ spooled: 0,+ dropped: 0,+ droppedBytes: 0,+ expired: 0,+ delivered: 0,+ duplicates: 0,+ rejected: 0+ };+}+function positiveInt(raw, fallback) {+ if (!raw || !/^\d+$/.test(raw.trim())) return fallback;+ const n = parseInt(raw.trim(), 10);+ return n > 0 ? n : fallback;+}+function spoolPolicy(env = process.env) {+ const maxBytes = Math.max(MIN_MAX_BYTES, positiveInt(env["AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES"], DEFAULT_MAX_BYTES));+ const maxAgeHours = positiveInt(env["AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS"], DEFAULT_MAX_AGE_HOURS);+ return {+ enabled: env["AGENTMEMORY_CAPTURE_SPOOL"] !== "false",+ maxBytes,+ maxAgeMs: maxAgeHours * 36e5,+ maxRecordBytes: Math.min(256 * 1024, Math.floor(maxBytes / 4))+ };+}+function spoolDir(env = process.env) {+ const explicit = env["AGENTMEMORY_CAPTURE_SPOOL_DIR"];+ if (explicit && explicit.trim()) return resolve(explicit.trim());+ return join(resolveDataDir({+ args: [],+ env+ }).dataDir, "capture-spool");+}+const LOOPBACK = new Set([+ "localhost",+ "127.0.0.1",+ "::1",+ "[::1]",+ "0.0.0.0"+]);+function spoolTargetName(url) {+ try {+ const u = new URL(url);+ return `${LOOPBACK.has(u.hostname) ? "local" : u.hostname.toLowerCase().replace(/[^a-z0-9.-]/g, "_")}-${u.port || (u.protocol === "https:" ? "443" : "80")}`;+ } catch {+ return "local-3111";+ }+}+function spoolPaths(url, dir = spoolDir()) {+ const name = spoolTargetName(url);+ return {+ dir,+ name,+ file: join(dir, `${name}.jsonl`),+ lock: join(dir, `${name}.lock`),+ drainLock: join(dir, `${name}.drain.lock`),+ stats: join(dir, `${name}.stats.json`)+ };+}+function ensureDir(dir) {+ mkdirSync(dir, {+ recursive: true,+ mode: 448+ });+ try {+ chmodSync(dir, 448);+ } catch {}+}+function sleepSync(ms) {+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);+}+function tryLock(path, staleMs) {+ try {+ const fd = openSync(path, "wx", 384);+ writeSync(fd, `${process.pid}\n`);+ closeSync(fd);+ return true;+ } catch (err) {+ if (err.code !== "EEXIST") return false;+ try {+ if (Date.now() - statSync(path).mtimeMs > staleMs) unlinkSync(path);+ } catch {}+ return false;+ }+}+function releaseLock(path) {+ try {+ unlinkSync(path);+ } catch {}+}+function withLock(path, fn, waitMs = LOCK_WAIT_MS) {+ const deadline = Date.now() + waitMs;+ let locked = tryLock(path, LOCK_STALE_MS);+ while (!locked && Date.now() < deadline) {+ sleepSync(5);+ locked = tryLock(path, LOCK_STALE_MS);+ }+ try {+ return fn(locked);+ } finally {+ if (locked) releaseLock(path);+ }+}+function readSpoolStats(paths) {+ try {+ const parsed = JSON.parse(readFileSync(paths.stats, "utf-8"));+ return {+ ...emptyStats(),+ ...parsed+ };+ } catch {+ return emptyStats();+ }+}+function writeStats(paths, patch) {+ try {+ const stats = readSpoolStats(paths);+ patch(stats);+ const tmp = `${paths.stats}.${process.pid}.tmp`;+ writeFileSync(tmp, JSON.stringify(stats), { mode: 384 });+ renameSync(tmp, paths.stats);+ } catch {}+}+function fileSize(path) {+ try {+ return statSync(path).size;+ } catch {+ return 0;+ }+}+function sanitizeBody(body) {+ try {+ return JSON.parse(stripPrivateData(JSON.stringify(body)));+ } catch {+ return body;+ }+}+function withoutImage(body) {+ const data = body["data"];+ if (!data || typeof data !== "object" || Array.isArray(data)) return body;+ const d = data;+ if (d["image_data"] === void 0) return body;+ return {+ ...body,+ data: {+ ...d,+ image_data: IMAGE_PLACEHOLDER+ }+ };+}+function parseLines(text) {+ const records = [];+ let corrupt = 0;+ for (const line of text.split("\n")) {+ if (!line.trim()) continue;+ try {+ const rec = JSON.parse(line);+ if (rec && typeof rec.eventId === "string" && rec.body && typeof rec.body === "object") records.push(rec);+ else corrupt++;+ } catch {+ corrupt++;+ }+ }+ return {+ records,+ corrupt+ };+}+function isExpired(rec, policy, now) {+ const at = Date.parse(rec.spooledAt);+ return Number.isFinite(at) && now - at > policy.maxAgeMs;+}+function serialize(records) {+ return records.map((r) => JSON.stringify(r)).join("\n") + (records.length ? "\n" : "");+}+function rewriteLocked(paths, records) {+ const tmp = `${paths.file}.${process.pid}.tmp`;+ const fd = openSync(tmp, "w", 384);+ try {+ writeSync(fd, serialize(records));+ fsyncSync(fd);+ } finally {+ closeSync(fd);+ }+ renameSync(tmp, paths.file);+}+function pruneExpiredLocked(paths, policy) {+ if (!existsSync(paths.file)) return 0;+ const { records } = parseLines(readFileSync(paths.file, "utf-8"));+ const now = Date.now();+ const kept = records.filter((r) => !isExpired(r, policy, now));+ const expired = records.length - kept.length;+ if (expired > 0) rewriteLocked(paths, kept);+ return expired;+}+function appendSpool(url, eventId, body, reason, options = {}) {+ const policy = options.policy ?? spoolPolicy();+ if (!policy.enabled) return {+ spooled: false,+ dropped: "disabled"+ };+ const paths = spoolPaths(url, options.dir);+ const note = (dropReason, bytes) => {+ writeStats(paths, (s) => {+ s.dropped++;+ s.droppedBytes += bytes;+ s.lastDropAt = (/* @__PURE__ */ new Date()).toISOString();+ s.lastDropReason = dropReason;+ });+ return {+ spooled: false,+ dropped: dropReason+ };+ };+ try {+ ensureDir(paths.dir);+ const record = {+ v: 1,+ eventId,+ spooledAt: (/* @__PURE__ */ new Date()).toISOString(),+ reason,+ attempts: options.attempts ?? 0,+ body: sanitizeBody(body)+ };+ let line = JSON.stringify(record) + "\n";+ if (Buffer.byteLength(line) > policy.maxRecordBytes) {+ record.body = withoutImage(record.body);+ line = JSON.stringify(record) + "\n";+ }+ const bytes = Buffer.byteLength(line);+ if (bytes > policy.maxRecordBytes) return note("too-large", bytes);+ return withLock(paths.lock, () => {+ let expired = 0;+ if (fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy);+ const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes;+ if (over > 0) evictSent(paths, over);+ if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) {+ if (expired) writeStats(paths, (s) => void (s.expired += expired));+ return note("full", bytes);+ }+ const fd = openSync(paths.file, "a", 384);+ try {+ writeSync(fd, line);+ fsyncSync(fd);+ } finally {+ closeSync(fd);+ }+ writeStats(paths, (s) => {+ s.spooled++;+ s.expired += expired;+ s.lastSpooledAt = record.spooledAt;+ s.lastSpoolReason = reason;+ });+ return { spooled: true };+ });+ } catch {+ return {+ spooled: false,+ dropped: "error"+ };+ }+}+function listSent(paths) {+ const segments = [];+ let entries;+ try {+ entries = readdirSync(paths.dir);+ } catch {+ return segments;+ }+ const prefix = `${paths.name}.sent-`;+ for (const entry of entries) {+ if (!entry.startsWith(prefix)) continue;+ const match = entry.slice(prefix.length).match(SENT_FILE_RE);+ if (!match) continue;+ const file = join(paths.dir, entry);+ segments.push({+ file,+ bootId: match[1],+ dueAt: Number(match[2]),+ bytes: fileSize(file)+ });+ }+ return segments;+}+function sentBytes(paths) {+ return listSent(paths).reduce((n, s) => n + s.bytes, 0);+}+function evictSent(paths, need) {+ let freed = 0;+ for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) {+ if (freed >= need) return;+ try {+ unlinkSync(seg.file);+ freed += seg.bytes;+ } catch {}+ }+}+function parseSentMark(body) {+ if (!body || typeof body !== "object" || Array.isArray(body)) return null;+ const bootId = body["bootId"];+ const after = body["durableAfterMs"];+ if (typeof bootId !== "string" || !BOOT_ID_RE.test(bootId)) return null;+ if (typeof after !== "number" || !Number.isFinite(after) || after < 0) return null;+ return {+ bootId,+ durableAfterMs: Math.min(Math.ceil(after), MAX_DURABLE_AFTER_MS)+ };+}+function retainSent(url, eventId, body, mark, options = {}) {+ const policy = options.policy ?? spoolPolicy();+ if (!policy.enabled || !BOOT_ID_RE.test(mark.bootId)) return false;+ const paths = spoolPaths(url, options.dir);+ const now = options.now ?? Date.now();+ try {+ ensureDir(paths.dir);+ const record = {+ v: 1,+ eventId,+ spooledAt: new Date(now).toISOString(),+ reason: "sent",+ attempts: 0,+ body: sanitizeBody(body),+ bootId: mark.bootId+ };+ let line = JSON.stringify(record) + "\n";+ if (Buffer.byteLength(line) > policy.maxRecordBytes) {+ record.body = withoutImage(record.body);+ line = JSON.stringify(record) + "\n";+ }+ const bytes = Buffer.byteLength(line);+ if (bytes > policy.maxRecordBytes) return false;+ const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS;+ const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`);+ return withLock(paths.lock, () => {+ if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false;+ const fd = openSync(file, "a", 384);+ try {+ writeSync(fd, line);+ } finally {+ closeSync(fd);+ }+ return true;+ });+ } catch {+ return false;+ }+}+function reconcileSent(url, bootId, options = {}) {+ const result = {+ released: 0,+ requeued: 0+ };+ if (!(options.policy ?? spoolPolicy()).enabled) return result;+ const paths = spoolPaths(url, options.dir);+ const now = options.now ?? Date.now();+ for (const seg of listSent(paths)) {+ if (seg.bootId === bootId) {+ if (seg.dueAt > now) continue;+ try {+ unlinkSync(seg.file);+ result.released++;+ } catch {}+ continue;+ }+ const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${now}-${result.requeued}.jsonl`);+ try {+ renameSync(seg.file, target);+ } catch {+ continue;+ }+ try {+ utimesSync(target, 0, 0);+ } catch {}+ result.requeued++;+ }+ return result;+}+function claimFiles(paths) {+ const claimed = [];+ try {+ const prefix = `${paths.name}.draining-`;+ for (const entry of readdirSync(paths.dir)) {+ if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue;+ const full = join(paths.dir, entry);+ try {+ if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full);+ } catch {}+ }+ } catch {}+ if (fileSize(paths.file) > 0) {+ const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`);+ withLock(paths.lock, () => {+ try {+ renameSync(paths.file, target);+ claimed.push(target);+ } catch {}+ });+ }+ return claimed;+}+function requeueLocked(paths, leftover) {+ if (leftover.length === 0) return;+ const existing = existsSync(paths.file) ? parseLines(readFileSync(paths.file, "utf-8")).records : [];+ rewriteLocked(paths, [...leftover, ...existing]);+}+function spoolHasRecords(url, dir) {+ return fileSize(spoolPaths(url, dir).file) > 0;+}+function drainInProgress(url, dir) {+ const paths = spoolPaths(url, dir);+ try {+ return Date.now() - statSync(paths.drainLock).mtimeMs < DRAIN_LOCK_STALE_MS;+ } catch {+ return false;+ }+}+async function drainSpool(url, send, options = {}) {+ const policy = options.policy ?? spoolPolicy();+ const result = {+ claimed: 0,+ delivered: 0,+ duplicates: 0,+ rejected: 0,+ expired: 0,+ corrupt: 0,+ remaining: 0+ };+ const paths = spoolPaths(url, options.dir);+ if (!existsSync(paths.dir)) return {+ ...result,+ skipped: "empty"+ };+ try {+ ensureDir(paths.dir);+ } catch {}+ if (!tryLock(paths.drainLock, DRAIN_LOCK_STALE_MS)) return {+ ...result,+ skipped: "locked"+ };+ try {+ const files = claimFiles(paths);+ if (files.length === 0) return {+ ...result,+ skipped: "empty"+ };+ const records = [];+ for (const file of files) try {+ const parsed = parseLines(readFileSync(file, "utf-8"));+ records.push(...parsed.records);+ result.corrupt += parsed.corrupt;+ } catch {}+ result.claimed = records.length;+ const seen = /* @__PURE__ */ new Set();+ const now = Date.now();+ const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY;+ const maxRecords = options.maxRecords ?? Number.POSITIVE_INFINITY;+ const leftover = [];+ let stopped = false;+ let processed = 0;+ for (const rec of records) {+ if (seen.has(rec.eventId)) {+ result.duplicates++;+ continue;+ }+ seen.add(rec.eventId);+ if (isExpired(rec, policy, now)) {+ result.expired++;+ continue;+ }+ if (stopped || processed >= maxRecords || Date.now() > deadline) {+ leftover.push(rec);+ continue;+ }+ processed++;+ let outcome;+ try {+ outcome = await send(rec);+ } catch (err) {+ outcome = "retry";+ result.error = err instanceof Error ? err.message : String(err);+ }+ if (outcome === "delivered") result.delivered++;+ else if (outcome === "duplicate") result.duplicates++;+ else if (outcome === "rejected") result.rejected++;+ else {+ stopped = true;+ leftover.push({+ ...rec,+ attempts: (rec.attempts ?? 0) + 1+ });+ }+ }+ result.remaining = leftover.length;+ withLock(paths.lock, () => requeueLocked(paths, leftover), 2e3);+ for (const file of files) try {+ unlinkSync(file);+ } catch {}+ writeStats(paths, (s) => {+ s.delivered += result.delivered;+ s.duplicates += result.duplicates;+ s.rejected += result.rejected;+ s.expired += result.expired;+ s.lastDrainAt = (/* @__PURE__ */ new Date()).toISOString();+ s.lastDrainDelivered = result.delivered;+ s.lastDrainDuplicates = result.duplicates;+ s.lastDrainRejected = result.rejected;+ s.lastDrainRemaining = result.remaining;+ if (result.error) s.lastDrainError = result.error;+ else delete s.lastDrainError;+ });+ return result;+ } finally {+ releaseLock(paths.drainLock);+ }+}+//#endregion+//#region src/hooks/_capture.ts+const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";+const SECRET = resolveClientSecret(REST_URL);+const DRAIN_CHILD_ENV = "AGENTMEMORY_CAPTURE_DRAIN_CHILD";+const DRAIN_MAX_RECORDS = 500;+const DRAIN_DEADLINE_MS = 2e4;+const DRAIN_REQUEST_TIMEOUT_MS = 5e3;+function authHeaders() {+ const h = { "Content-Type": "application/json" };+ if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;+ return h;+}+function withEventId(body, host, content) {+ return {+ ...body,+ eventId: deriveEventId(body.hookType, body.sessionId, host, content ?? body.data)+ };+}+function classify(status) {+ if (status >= 200 && status < 300) return status === 200 ? "duplicate" : "delivered";+ if (status === 408 || status === 429 || status >= 500) return "retry";+ return "rejected";+}+async function post(body, timeoutMs) {+ const res = await fetch(`${REST_URL}/agentmemory/observe`, {+ method: "POST",+ headers: authHeaders(),+ body: JSON.stringify(body),+ signal: AbortSignal.timeout(timeoutMs)+ });+ const text = await res.text().catch(() => "");+ const outcome = classify(res.status);+ if (outcome !== "delivered" && outcome !== "duplicate") return {+ outcome,+ mark: null+ };+ try {+ return {+ outcome,+ mark: parseSentMark(JSON.parse(text))+ };+ } catch {+ return {+ outcome,+ mark: null+ };+ }+}+function keepUntilDurable(eventId, body, mark) {+ const { requeued } = reconcileSent(REST_URL, mark.bootId);+ retainSent(REST_URL, eventId, body, mark);+ return requeued;+}+function startDrainChild() {+ if (process.env[DRAIN_CHILD_ENV] === "1") return;+ if (!process.argv[1] || drainInProgress(REST_URL)) return;+ try {+ const child = spawn(process.execPath, [process.argv[1]], {+ detached: true,+ stdio: "ignore",+ windowsHide: true,+ env: {+ ...process.env,+ [DRAIN_CHILD_ENV]: "1"+ }+ });+ child.on("error", () => {});+ child.unref();+ } catch {}+}+async function captureObservation(body, timeoutMs) {+ const eventId = body.eventId ?? deriveEventId(body.hookType, body.sessionId, null, body.data);+ const payload = {+ ...body,+ eventId+ };+ let reason;+ try {+ const { outcome, mark } = await post(payload, timeoutMs);+ if (outcome !== "retry") {+ if ((mark ? keepUntilDurable(eventId, payload, mark) : 0) > 0 || spoolHasRecords(REST_URL)) startDrainChild();+ return outcome;+ }+ reason = "server-error";+ } catch (err) {+ const name = err instanceof Error ? err.name : "";+ reason = name === "TimeoutError" || name === "AbortError" ? "timeout" : "unreachable";+ }+ return appendSpool(REST_URL, eventId, payload, reason).spooled ? "spooled" : "dropped";+}+function isDrainChild() {+ return process.env[DRAIN_CHILD_ENV] === "1";+}+async function runDrainChild() {+ await drainSpool(REST_URL, async (record) => {+ const { outcome, mark } = await post({+ ...record.body,+ eventId: record.eventId+ }, DRAIN_REQUEST_TIMEOUT_MS);+ if (mark) retainSent(REST_URL, record.eventId, record.body, mark);+ return outcome;+ }, {+ maxRecords: DRAIN_MAX_RECORDS,+ deadlineMs: DRAIN_DEADLINE_MS+ }).catch(() => void 0);+}+//#endregion+export { REST_URL, authHeaders, captureObservation, isDrainChild, runDrainChild, withEventId };++//# sourceMappingURL=_capture.mjs.map\ No newline at end of filediff --git a/plugin/scripts/notification.mjs b/plugin/scripts/notification.mjsindex e8711f66..c474a226 100755--- a/plugin/scripts/notification.mjs+++ b/plugin/scripts/notification.mjs@@ -1,75 +1,7 @@#!/usr/bin/env node-import { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_project.tsfunction resolveProject(cwd) {const explicit = process.env["AGENTMEMORY_PROJECT_NAME"];@@ -106,14 +38,10 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 2e3;+const EXIT_CAP_MS = 2500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -133,24 +61,19 @@ async function main() {].find((v) => typeof v === "string" && v.length > 0);const sessionId = typeof rawSessionId === "string" ? rawSessionId : "unknown";const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "notification",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: (/* @__PURE__ */ new Date()).toISOString(),- data: {- notification_type: notificationType,- title: data.title,- message: data.message- }- }),- signal: AbortSignal.timeout(2e3)- }).catch(() => {});- setTimeout(() => process.exit(0), 2e3).unref();+ captureObservation(withEventId({+ hookType: "notification",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),+ data: {+ notification_type: notificationType,+ title: data.title,+ message: data.message+ }+ }, data), OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));//#endregiondiff --git a/plugin/scripts/post-tool-failure.mjs b/plugin/scripts/post-tool-failure.mjsindex 62a7b2f0..c3ddc2ab 100755--- a/plugin/scripts/post-tool-failure.mjs+++ b/plugin/scripts/post-tool-failure.mjs@@ -1,75 +1,7 @@#!/usr/bin/env node-import { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_capture-filter.tsconst DEFAULT_DENY_PATTERNS = ["memory_*",@@ -147,14 +79,10 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 3e3;+const EXIT_CAP_MS = 3500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -172,24 +100,19 @@ async function main() {const toolInput = data.tool_input ?? data.toolArgs;const error = data.error ?? data.errorMessage;const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "post_tool_failure",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: (/* @__PURE__ */ new Date()).toISOString(),- data: {- tool_name: toolName,- tool_input: typeof toolInput === "string" ? toolInput.slice(0, 4e3) : JSON.stringify(toolInput ?? "").slice(0, 4e3),- error: typeof error === "string" ? error.slice(0, 4e3) : JSON.stringify(error ?? "").slice(0, 4e3)- }- }),- signal: AbortSignal.timeout(3e3)- }).catch(() => {});- setTimeout(() => process.exit(0), 3e3).unref();+ captureObservation(withEventId({+ hookType: "post_tool_failure",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),+ data: {+ tool_name: toolName,+ tool_input: typeof toolInput === "string" ? toolInput.slice(0, 4e3) : JSON.stringify(toolInput ?? "").slice(0, 4e3),+ error: typeof error === "string" ? error.slice(0, 4e3) : JSON.stringify(error ?? "").slice(0, 4e3)+ }+ }, data), OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));//#endregiondiff --git a/plugin/scripts/post-tool-use.mjs b/plugin/scripts/post-tool-use.mjsindex f9607354..585dea9b 100755--- a/plugin/scripts/post-tool-use.mjs+++ b/plugin/scripts/post-tool-use.mjs@@ -1,75 +1,7 @@#!/usr/bin/env node-import { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_capture-filter.tsconst DEFAULT_DENY_PATTERNS = ["memory_*",@@ -168,14 +100,10 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 3e3;+const EXIT_CAP_MS = 3500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -192,26 +120,26 @@ async function main() {const toolInput = data.tool_input ?? data.toolArgs;const { imageData, cleanOutput } = extractImageData(toolOutput(data));const cwd = hookCwd(data) || process.cwd();- const outputMax = captureOutputMax();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "post_tool_use",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: (/* @__PURE__ */ new Date()).toISOString(),- data: {- tool_name: toolName,- tool_input: toolInput,- tool_output: truncateCaptureOutput(cleanOutput, outputMax),- ...imageData ? { image_data: imageData } : {}- }- }),- signal: AbortSignal.timeout(3e3)- }).catch(() => {});- setTimeout(() => process.exit(0), 3e3).unref();+ const toolOutputText = truncateCaptureOutput(cleanOutput, captureOutputMax());+ captureObservation(withEventId({+ hookType: "post_tool_use",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),+ data: {+ tool_name: toolName,+ tool_input: toolInput,+ tool_output: toolOutputText,+ ...imageData ? { image_data: imageData } : {}+ }+ }, data, {+ tool_name: toolName,+ tool_input: toolInput,+ tool_output: toolOutputText,+ image_data: imageData+ }), OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}function toolOutput(data) {if (data.tool_response !== void 0) return data.tool_response;diff --git a/plugin/scripts/prompt-submit.mjs b/plugin/scripts/prompt-submit.mjsindex 80ac600e..03061e77 100755--- a/plugin/scripts/prompt-submit.mjs+++ b/plugin/scripts/prompt-submit.mjs@@ -1,75 +1,7 @@#!/usr/bin/env node-import { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_project.tsfunction resolveProject(cwd) {const explicit = process.env["AGENTMEMORY_PROJECT_NAME"];@@ -106,14 +38,10 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 3e3;+const EXIT_CAP_MS = 3500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -126,20 +54,15 @@ async function main() {if (isSdkChildContext(data)) return;const sessionId = data.session_id || data.sessionId || data.conversation_id || "unknown";const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "prompt_submit",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: (/* @__PURE__ */ new Date()).toISOString(),- data: { prompt: data.prompt ?? data.userPrompt }- }),- signal: AbortSignal.timeout(3e3)- }).catch(() => {});- setTimeout(() => process.exit(0), 3e3).unref();+ captureObservation(withEventId({+ hookType: "prompt_submit",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),+ data: { prompt: data.prompt ?? data.userPrompt }+ }, data), OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));//#endregiondiff --git a/plugin/scripts/session-end.mjs b/plugin/scripts/session-end.mjsindex 6199d0c3..d9be519d 100755--- a/plugin/scripts/session-end.mjs+++ b/plugin/scripts/session-end.mjs@@ -1,75 +1,8 @@#!/usr/bin/env nodeimport { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { REST_URL, authHeaders, captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_project.tsfunction resolveProject(cwd) {const explicit = process.env["AGENTMEMORY_PROJECT_NAME"];@@ -106,13 +39,6 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}function extractTranscriptPrompts(data) {const path = data.transcript_path;if (typeof path !== "string" || !path.endsWith(".jsonl")) return [];@@ -143,6 +69,7 @@ function extractTranscriptPrompts(data) {return prompts;}async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -159,19 +86,19 @@ async function main() {const cwd = hookCwd(data) || process.cwd();const project = resolveProject(cwd);const timestamp = (/* @__PURE__ */ new Date()).toISOString();- await Promise.allSettled(transcriptPrompts.map((prompt) => fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "prompt_submit",- sessionId,- project,- cwd,- timestamp,- data: { prompt }- }),- signal: AbortSignal.timeout(3e3)- })));+ await Promise.allSettled(transcriptPrompts.map((prompt, index) => captureObservation(withEventId({+ hookType: "prompt_submit",+ sessionId,+ project,+ cwd,+ timestamp,+ data: { prompt }+ }, {}, {+ source: "transcript",+ transcript: data.transcript_path,+ index,+ prompt+ }), 3e3)));}fetch(`${REST_URL}/agentmemory/session/end`, {method: "POST",diff --git a/plugin/scripts/subagent-start.mjs b/plugin/scripts/subagent-start.mjsindex 91c796f0..01f33785 100755--- a/plugin/scripts/subagent-start.mjs+++ b/plugin/scripts/subagent-start.mjs@@ -1,75 +1,7 @@#!/usr/bin/env node-import { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_project.tsfunction resolveProject(cwd) {const explicit = process.env["AGENTMEMORY_PROJECT_NAME"];@@ -106,15 +38,10 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-const TIMEOUT_MS = 800;-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 800;+const EXIT_CAP_MS = 1300;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -129,23 +56,18 @@ async function main() {const agentId = data.agent_id || data.agentName;const agentType = data.agent_type || data.agentDisplayName || data.agentName;const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "subagent_start",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: (/* @__PURE__ */ new Date()).toISOString(),- data: {- agent_id: agentId,- agent_type: agentType- }- }),- signal: AbortSignal.timeout(TIMEOUT_MS)- }).catch(() => {});- setTimeout(() => process.exit(0), TIMEOUT_MS).unref();+ captureObservation(withEventId({+ hookType: "subagent_start",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),+ data: {+ agent_id: agentId,+ agent_type: agentType+ }+ }, data), OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));//#endregiondiff --git a/plugin/scripts/subagent-stop.mjs b/plugin/scripts/subagent-stop.mjsindex f59d67de..8e0797b4 100755--- a/plugin/scripts/subagent-stop.mjs+++ b/plugin/scripts/subagent-stop.mjs@@ -1,75 +1,7 @@#!/usr/bin/env node-import { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_project.tsfunction resolveProject(cwd) {const explicit = process.env["AGENTMEMORY_PROJECT_NAME"];@@ -106,14 +38,10 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 2e3;+const EXIT_CAP_MS = 2500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -129,24 +57,19 @@ async function main() {const agentType = data.agent_type || data.agentDisplayName || data.agentName;const lastMsg = typeof data.last_assistant_message === "string" ? data.last_assistant_message.slice(0, 4e3) : "";const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "subagent_stop",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: (/* @__PURE__ */ new Date()).toISOString(),- data: {- agent_id: agentId,- agent_type: agentType,- last_message: lastMsg- }- }),- signal: AbortSignal.timeout(2e3)- }).catch(() => {});- setTimeout(() => process.exit(0), 2e3).unref();+ captureObservation(withEventId({+ hookType: "subagent_stop",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),+ data: {+ agent_id: agentId,+ agent_type: agentType,+ last_message: lastMsg+ }+ }, data), OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));//#endregiondiff --git a/plugin/scripts/task-completed.mjs b/plugin/scripts/task-completed.mjsindex 55568e4f..b5fc0fca 100755--- a/plugin/scripts/task-completed.mjs+++ b/plugin/scripts/task-completed.mjs@@ -1,75 +1,7 @@#!/usr/bin/env node-import { readFileSync } from "node:fs";-import { homedir } from "node:os";-import { basename, join } from "node:path";import { execSync } from "node:child_process";-//#region src/secret-store.ts-const SECRET_KEY = "AGENTMEMORY_SECRET";-function agentmemoryHomeDir() {- return join(homedir(), ".agentmemory");-}-function secretFilePath() {- return join(agentmemoryHomeDir(), "secret");-}-function usable(value) {- if (typeof value !== "string") return "";- const trimmed = value.trim();- if (!trimmed) return "";- if (trimmed.startsWith("${") && trimmed.endsWith("}")) return "";- return trimmed;-}-function unquote(value) {- const quote = value[0];- if ((quote === "\"" || quote === "'") && value.length > 1) {- const close = value.indexOf(quote, 1);- if (close !== -1) return value.slice(1, close);- }- const hash = value.indexOf(" #");- return hash === -1 ? value : value.slice(0, hash).trim();-}-function readEnvFileSecret() {- let content;- try {- content = readFileSync(join(agentmemoryHomeDir(), ".env"), "utf-8");- } catch {- return "";- }- if (typeof content !== "string") return "";- let found = "";- for (const line of content.split("\n")) {- const trimmed = line.trim();- if (trimmed.startsWith("#")) continue;- const eq = trimmed.indexOf("=");- if (eq === -1) continue;- if (trimmed.slice(0, eq).replace(/^export\s+/, "").trim() !== SECRET_KEY) continue;- found = usable(unquote(trimmed.slice(eq + 1).trim()));- }- return found;-}-function readStoredSecret() {- try {- return usable(readFileSync(secretFilePath(), "utf-8"));- } catch {- return "";- }-}-function isLoopbackUrl(url) {- let hostname;- try {- hostname = new URL(url).hostname.toLowerCase();- } catch {- return false;- }- const bare = hostname.replace(/^\[|\]$/g, "");- return bare === "localhost" || bare === "::1" || /^127(?:\.\d{1,3}){3}$/.test(bare);-}-function resolveClientSecret(baseUrl, env = process.env) {- const fromEnv = usable(env[SECRET_KEY]);- if (fromEnv) return fromEnv;- if (!isLoopbackUrl(baseUrl)) return "";- return readEnvFileSecret() || readStoredSecret();-}-//#endregion+import { basename } from "node:path";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.mjs";//#region src/hooks/_project.tsfunction resolveProject(cwd) {const explicit = process.env["AGENTMEMORY_PROJECT_NAME"];@@ -106,14 +38,10 @@ function isSdkChildContext(payload) {if (!payload || typeof payload !== "object") return false;return payload.entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-function authHeaders() {- const h = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 2e3;+const EXIT_CAP_MS = 2500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) input += chunk;let data;@@ -126,26 +54,21 @@ async function main() {if (isSdkChildContext(data)) return;const sessionId = data.session_id || data.sessionId || data.conversation_id || "unknown";const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "task_completed",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: (/* @__PURE__ */ new Date()).toISOString(),- data: {- task_id: data.task_id,- task_subject: data.task_subject,- task_description: typeof data.task_description === "string" ? data.task_description.slice(0, 2e3) : "",- teammate_name: data.teammate_name,- team_name: data.team_name- }- }),- signal: AbortSignal.timeout(2e3)- }).catch(() => {});- setTimeout(() => process.exit(0), 2e3).unref();+ captureObservation(withEventId({+ hookType: "task_completed",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),+ data: {+ task_id: data.task_id,+ task_subject: data.task_subject,+ task_description: typeof data.task_description === "string" ? data.task_description.slice(0, 2e3) : "",+ teammate_name: data.teammate_name,+ team_name: data.team_name+ }+ }, data), OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));//#endregiondiff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.mdindex fd8f5e38..27e5e8e1 100644--- a/plugin/skills/agentmemory-config/REFERENCE.md+++ b/plugin/skills/agentmemory-config/REFERENCE.md@@ -3,7 +3,7 @@Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded.<!-- AUTOGEN:env START - generated by scripts/skills/generate.ts, do not edit by hand -->-Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 61 recognized variables:+Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 72 recognized variables:- `AGENTMEMORY_AGENT_SCOPE`- `AGENTMEMORY_ALLOW_AGENT_SDK`@@ -12,8 +12,19 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e- `AGENTMEMORY_AUDIT_RETENTION_MONTHS`- `AGENTMEMORY_AUTO_COMPRESS`- `AGENTMEMORY_CAPTURE_ALLOW`+- `AGENTMEMORY_CAPTURE_DEAD_MAX`+- `AGENTMEMORY_CAPTURE_DEDUP_HOURS`- `AGENTMEMORY_CAPTURE_DENY`+- `AGENTMEMORY_CAPTURE_DRAIN_CHILD`+- `AGENTMEMORY_CAPTURE_EVENTS_MAX`+- `AGENTMEMORY_CAPTURE_INBOX_MAX`+- `AGENTMEMORY_CAPTURE_MAX_ATTEMPTS`- `AGENTMEMORY_CAPTURE_OUTPUT_MAX`+- `AGENTMEMORY_CAPTURE_RETRY_INTERVAL_MS`+- `AGENTMEMORY_CAPTURE_SPOOL`+- `AGENTMEMORY_CAPTURE_SPOOL_DIR`+- `AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS`+- `AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES`- `AGENTMEMORY_COMMIT_SHA`- `AGENTMEMORY_CONSOLIDATION_COOLDOWN_MS`- `AGENTMEMORY_COPILOT_MCP_BLOCK`diff --git a/plugin/skills/agentmemory-rest-api/REFERENCE.md b/plugin/skills/agentmemory-rest-api/REFERENCE.mdindex 28164d3b..ae5b371f 100644--- a/plugin/skills/agentmemory-rest-api/REFERENCE.md+++ b/plugin/skills/agentmemory-rest-api/REFERENCE.md@@ -5,7 +5,7 @@ Generated from `src/triggers/api.ts`. Do not edit the block below by hand; run `<!-- AUTOGEN:rest START - generated by scripts/skills/generate.ts, do not edit by hand -->The REST API is the primary surface. All paths are under `http://localhost:3111` (override with `--port`). Send `Authorization: Bearer <secret>`: the value of `AGENTMEMORY_SECRET`, or the one generated into `~/.agentmemory/secret` on first start when it is unset. Writes must use `Content-Type: application/json`.-135 registered endpoints:+138 registered endpoints:| Method | Path || --- | --- |@@ -19,6 +19,9 @@ The REST API is the primary surface. All paths are under `http://localhost:3111`| GET | `/agentmemory/branch/detect` || GET | `/agentmemory/branch/sessions` || GET | `/agentmemory/branch/worktrees` |+| GET | `/agentmemory/capture` |+| POST | `/agentmemory/capture/drain` |+| POST | `/agentmemory/capture/retry` || POST | `/agentmemory/cascade-update` || GET | `/agentmemory/checkpoints` || POST | `/agentmemory/checkpoints` |diff --git a/src/capture/event-id.ts b/src/capture/event-id.tsnew file mode 100644index 00000000..270042b3--- /dev/null+++ b/src/capture/event-id.ts@@ -0,0 +1,64 @@+import { createHash } from "node:crypto";++export const EVENT_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{7,127}$/;++const HOST_ID_FIELDS = [+ "event_id",+ "eventId",+ "tool_use_id",+ "toolUseId",+ "tool_call_id",+ "toolCallId",+ "call_id",+ "callId",+ "prompt_id",+ "promptId",+ "message_id",+ "messageId",+];++const HOST_TIME_FIELDS = ["timestamp", "ts", "created_at", "createdAt", "event_time"];++export function isValidEventId(value: unknown): value is string {+ return typeof value === "string" && EVENT_ID_RE.test(value);+}++export function stableStringify(value: unknown): string {+ if (value === undefined) return "null";+ if (value === null || typeof value !== "object") {+ const encoded = JSON.stringify(value);+ return encoded === undefined ? "null" : encoded;+ }+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;+ const obj = value as Record<string, unknown>;+ const keys = Object.keys(obj)+ .filter((k) => obj[k] !== undefined)+ .sort();+ return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(",")}}`;+}++function hostScalar(host: Record<string, unknown>, fields: string[]): string | undefined {+ for (const field of fields) {+ const v = host[field];+ if (typeof v === "string" && v.trim()) return v.trim();+ if (typeof v === "number" && Number.isFinite(v)) return String(v);+ }+ return undefined;+}++function digest(parts: string[]): string {+ return createHash("sha256").update(parts.join("\u0000")).digest("hex").slice(0, 32);+}++export function deriveEventId(+ hookType: string,+ sessionId: string,+ host: Record<string, unknown> | null | undefined,+ content: unknown,+): string {+ const source = host && typeof host === "object" ? host : {};+ const hostId = hostScalar(source, HOST_ID_FIELDS);+ if (hostId) return `evh_${digest(["host", sessionId, hookType, hostId])}`;+ const hostTime = hostScalar(source, HOST_TIME_FIELDS) ?? "";+ return `evc_${digest(["content", sessionId, hookType, hostTime, stableStringify(content)])}`;+}diff --git a/src/capture/event-record.ts b/src/capture/event-record.tsnew file mode 100644index 00000000..97fde39c--- /dev/null+++ b/src/capture/event-record.ts@@ -0,0 +1,55 @@+import { KV } from "../state/schema.js";+import type { StateKV } from "../state/kv.js";++export const EVENT_SHARD_CHARS = 2;+export const EVENT_SHARDS = 16 ** EVENT_SHARD_CHARS;++export interface CompletedEvent {+ key: string;+ eventId: string;+ sessionId: string;+ project: string;+ observationId: string;+ acceptedAt: string;+ completedAt: string;+ attempts: number;+ state?: "deleted";+ deletedAt?: string;+}++export function captureEventShard(key: string): string {+ return key.slice(4, 4 + EVENT_SHARD_CHARS);+}++export function captureEventScope(key: string): string {+ return KV.captureEvents(captureEventShard(key));+}++export function isCaptureKey(value: unknown): value is string {+ return typeof value === "string" && /^cap_[0-9a-f]{40}$/.test(value);+}++export async function markCaptureEventDeleted(+ kv: StateKV,+ observation: { id: string; sessionId?: string; captureKey?: unknown } | null | undefined,+): Promise<void> {+ const key = observation?.captureKey;+ if (!observation || !isCaptureKey(key)) return;+ const scope = captureEventScope(key);+ const existing = await kv.get<CompletedEvent>(scope, key);+ if (existing?.state === "deleted") return;+ const now = new Date().toISOString();+ const marker: CompletedEvent = {+ key,+ eventId: existing?.eventId ?? "",+ sessionId: existing?.sessionId ?? observation.sessionId ?? "",+ project: existing?.project ?? "",+ observationId: existing?.observationId || observation.id,+ acceptedAt: existing?.acceptedAt ?? now,+ completedAt: existing?.completedAt ?? now,+ attempts: existing?.attempts ?? 0,+ state: "deleted",+ deletedAt: now,+ };+ await kv.set(scope, key, marker);+}diff --git a/src/capture/spool.ts b/src/capture/spool.tsnew file mode 100644index 00000000..b9d7cdc9--- /dev/null+++ b/src/capture/spool.ts@@ -0,0 +1,679 @@+import {+ chmodSync,+ closeSync,+ existsSync,+ fsyncSync,+ mkdirSync,+ openSync,+ readFileSync,+ readdirSync,+ renameSync,+ statSync,+ unlinkSync,+ utimesSync,+ writeFileSync,+ writeSync,+} from "node:fs";+import { join, resolve } from "node:path";+import { resolveDataDir } from "../cli-data-dir.js";+import { stripPrivateData } from "../functions/privacy.js";++export interface SpoolRecord {+ v: 1;+ eventId: string;+ spooledAt: string;+ reason: string;+ attempts: number;+ body: Record<string, unknown>;+ bootId?: string;+}++export interface SentMark {+ bootId: string;+ durableAfterMs: number;+}++export interface SentReconcile {+ released: number;+ requeued: number;+}++export interface SpoolPolicy {+ enabled: boolean;+ maxBytes: number;+ maxAgeMs: number;+ maxRecordBytes: number;+}++export interface SpoolStats {+ spooled: number;+ dropped: number;+ droppedBytes: number;+ expired: number;+ delivered: number;+ duplicates: number;+ rejected: number;+ lastSpooledAt?: string;+ lastSpoolReason?: string;+ lastDropAt?: string;+ lastDropReason?: string;+ lastDrainAt?: string;+ lastDrainDelivered?: number;+ lastDrainDuplicates?: number;+ lastDrainRejected?: number;+ lastDrainRemaining?: number;+ lastDrainError?: string;+}++export interface SpoolPaths {+ dir: string;+ name: string;+ file: string;+ lock: string;+ drainLock: string;+ stats: string;+}++export interface SpoolSummary {+ enabled: boolean;+ path: string;+ records: number;+ bytes: number;+ oldestAt: string | null;+ retained: number;+ retainedBytes: number;+ maxBytes: number;+ maxAgeHours: number;+ stats: SpoolStats;+}++export type SendOutcome = "delivered" | "duplicate" | "rejected" | "retry";++export interface DrainResult {+ claimed: number;+ delivered: number;+ duplicates: number;+ rejected: number;+ expired: number;+ corrupt: number;+ remaining: number;+ error?: string;+ skipped?: "locked" | "empty" | "disabled";+}++const DEFAULT_MAX_BYTES = 5 * 1024 * 1024;+const MIN_MAX_BYTES = 64 * 1024;+const DEFAULT_MAX_AGE_HOURS = 168;+const LOCK_STALE_MS = 10_000;+const DRAIN_LOCK_STALE_MS = 120_000;+const ORPHAN_CLAIM_MS = 120_000;+const LOCK_WAIT_MS = 500;+const IMAGE_PLACEHOLDER = "[image dropped from capture spool]";+const BOOT_ID_RE = /^[A-Za-z0-9]{8,64}$/;+const SENT_FILE_RE = /^([A-Za-z0-9]{8,64})-(\d+)\.jsonl$/;+const SENT_BUCKET_MS = 1000;+const MAX_DURABLE_AFTER_MS = 10 * 60_000;++function emptyStats(): SpoolStats {+ return { spooled: 0, dropped: 0, droppedBytes: 0, expired: 0, delivered: 0, duplicates: 0, rejected: 0 };+}++function positiveInt(raw: string | undefined, fallback: number): number {+ if (!raw || !/^\d+$/.test(raw.trim())) return fallback;+ const n = parseInt(raw.trim(), 10);+ return n > 0 ? n : fallback;+}++export function spoolPolicy(env: NodeJS.ProcessEnv = process.env): SpoolPolicy {+ const maxBytes = Math.max(MIN_MAX_BYTES, positiveInt(env["AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES"], DEFAULT_MAX_BYTES));+ const maxAgeHours = positiveInt(env["AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS"], DEFAULT_MAX_AGE_HOURS);+ return {+ enabled: env["AGENTMEMORY_CAPTURE_SPOOL"] !== "false",+ maxBytes,+ maxAgeMs: maxAgeHours * 3600_000,+ maxRecordBytes: Math.min(256 * 1024, Math.floor(maxBytes / 4)),+ };+}++export function spoolDir(env: NodeJS.ProcessEnv = process.env): string {+ const explicit = env["AGENTMEMORY_CAPTURE_SPOOL_DIR"];+ if (explicit && explicit.trim()) return resolve(explicit.trim());+ return join(resolveDataDir({ args: [], env }).dataDir, "capture-spool");+}++const LOOPBACK = new Set(["localhost", "127.0.0.1", "::1", "[::1]", "0.0.0.0"]);++export function spoolTargetName(url: string): string {+ try {+ const u = new URL(url);+ const host = LOOPBACK.has(u.hostname) ? "local" : u.hostname.toLowerCase().replace(/[^a-z0-9.-]/g, "_");+ const port = u.port || (u.protocol === "https:" ? "443" : "80");+ return `${host}-${port}`;+ } catch {+ return "local-3111";+ }+}++export function spoolPaths(url: string, dir: string = spoolDir()): SpoolPaths {+ const name = spoolTargetName(url);+ return {+ dir,+ name,+ file: join(dir, `${name}.jsonl`),+ lock: join(dir, `${name}.lock`),+ drainLock: join(dir, `${name}.drain.lock`),+ stats: join(dir, `${name}.stats.json`),+ };+}++function ensureDir(dir: string): void {+ mkdirSync(dir, { recursive: true, mode: 0o700 });+ try {+ chmodSync(dir, 0o700);+ } catch {}+}++function sleepSync(ms: number): void {+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);+}++function tryLock(path: string, staleMs: number): boolean {+ try {+ const fd = openSync(path, "wx", 0o600);+ writeSync(fd, `${process.pid}\n`);+ closeSync(fd);+ return true;+ } catch (err) {+ if ((err as NodeJS.ErrnoException).code !== "EEXIST") return false;+ try {+ if (Date.now() - statSync(path).mtimeMs > staleMs) unlinkSync(path);+ } catch {}+ return false;+ }+}++function releaseLock(path: string): void {+ try {+ unlinkSync(path);+ } catch {}+}++function withLock<T>(path: string, fn: (locked: boolean) => T, waitMs = LOCK_WAIT_MS): T {+ const deadline = Date.now() + waitMs;+ let locked = tryLock(path, LOCK_STALE_MS);+ while (!locked && Date.now() < deadline) {+ sleepSync(5);+ locked = tryLock(path, LOCK_STALE_MS);+ }+ try {+ return fn(locked);+ } finally {+ if (locked) releaseLock(path);+ }+}++export function readSpoolStats(paths: SpoolPaths): SpoolStats {+ try {+ const parsed = JSON.parse(readFileSync(paths.stats, "utf-8")) as Partial<SpoolStats>;+ return { ...emptyStats(), ...parsed };+ } catch {+ return emptyStats();+ }+}++function writeStats(paths: SpoolPaths, patch: (stats: SpoolStats) => void): void {+ try {+ const stats = readSpoolStats(paths);+ patch(stats);+ const tmp = `${paths.stats}.${process.pid}.tmp`;+ writeFileSync(tmp, JSON.stringify(stats), { mode: 0o600 });+ renameSync(tmp, paths.stats);+ } catch {}+}++function fileSize(path: string): number {+ try {+ return statSync(path).size;+ } catch {+ return 0;+ }+}++function sanitizeBody(body: Record<string, unknown>): Record<string, unknown> {+ try {+ return JSON.parse(stripPrivateData(JSON.stringify(body))) as Record<string, unknown>;+ } catch {+ return body;+ }+}++function withoutImage(body: Record<string, unknown>): Record<string, unknown> {+ const data = body["data"];+ if (!data || typeof data !== "object" || Array.isArray(data)) return body;+ const d = data as Record<string, unknown>;+ if (d["image_data"] === undefined) return body;+ return { ...body, data: { ...d, image_data: IMAGE_PLACEHOLDER } };+}++function parseLines(text: string): { records: SpoolRecord[]; corrupt: number } {+ const records: SpoolRecord[] = [];+ let corrupt = 0;+ for (const line of text.split("\n")) {+ if (!line.trim()) continue;+ try {+ const rec = JSON.parse(line) as SpoolRecord;+ if (rec && typeof rec.eventId === "string" && rec.body && typeof rec.body === "object") records.push(rec);+ else corrupt++;+ } catch {+ corrupt++;+ }+ }+ return { records, corrupt };+}++function isExpired(rec: SpoolRecord, policy: SpoolPolicy, now: number): boolean {+ const at = Date.parse(rec.spooledAt);+ return Number.isFinite(at) && now - at > policy.maxAgeMs;+}++function serialize(records: SpoolRecord[]): string {+ return records.map((r) => JSON.stringify(r)).join("\n") + (records.length ? "\n" : "");+}++function rewriteLocked(paths: SpoolPaths, records: SpoolRecord[]): void {+ const tmp = `${paths.file}.${process.pid}.tmp`;+ const fd = openSync(tmp, "w", 0o600);+ try {+ writeSync(fd, serialize(records));+ fsyncSync(fd);+ } finally {+ closeSync(fd);+ }+ renameSync(tmp, paths.file);+}++function pruneExpiredLocked(paths: SpoolPaths, policy: SpoolPolicy): number {+ if (!existsSync(paths.file)) return 0;+ const { records } = parseLines(readFileSync(paths.file, "utf-8"));+ const now = Date.now();+ const kept = records.filter((r) => !isExpired(r, policy, now));+ const expired = records.length - kept.length;+ if (expired > 0) rewriteLocked(paths, kept);+ return expired;+}++export interface AppendResult {+ spooled: boolean;+ dropped?: "disabled" | "too-large" | "full" | "error";+}++export function appendSpool(+ url: string,+ eventId: string,+ body: Record<string, unknown>,+ reason: string,+ options: { policy?: SpoolPolicy; dir?: string; attempts?: number } = {},+): AppendResult {+ const policy = options.policy ?? spoolPolicy();+ if (!policy.enabled) return { spooled: false, dropped: "disabled" };+ const paths = spoolPaths(url, options.dir);+ const note = (dropReason: NonNullable<AppendResult["dropped"]>, bytes: number) => {+ writeStats(paths, (s) => {+ s.dropped++;+ s.droppedBytes += bytes;+ s.lastDropAt = new Date().toISOString();+ s.lastDropReason = dropReason;+ });+ return { spooled: false, dropped: dropReason } as AppendResult;+ };+ try {+ ensureDir(paths.dir);+ const record: SpoolRecord = {+ v: 1,+ eventId,+ spooledAt: new Date().toISOString(),+ reason,+ attempts: options.attempts ?? 0,+ body: sanitizeBody(body),+ };+ let line = JSON.stringify(record) + "\n";+ if (Buffer.byteLength(line) > policy.maxRecordBytes) {+ record.body = withoutImage(record.body);+ line = JSON.stringify(record) + "\n";+ }+ const bytes = Buffer.byteLength(line);+ if (bytes > policy.maxRecordBytes) return note("too-large", bytes);+ return withLock(paths.lock, () => {+ let expired = 0;+ if (fileSize(paths.file) + bytes > policy.maxBytes) expired = pruneExpiredLocked(paths, policy);+ const over = fileSize(paths.file) + sentBytes(paths) + bytes - policy.maxBytes;+ if (over > 0) evictSent(paths, over);+ if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) {+ if (expired) writeStats(paths, (s) => void (s.expired += expired));+ return note("full", bytes);+ }+ const fd = openSync(paths.file, "a", 0o600);+ try {+ writeSync(fd, line);+ fsyncSync(fd);+ } finally {+ closeSync(fd);+ }+ writeStats(paths, (s) => {+ s.spooled++;+ s.expired += expired;+ s.lastSpooledAt = record.spooledAt;+ s.lastSpoolReason = reason;+ });+ return { spooled: true } as AppendResult;+ });+ } catch {+ return { spooled: false, dropped: "error" };+ }+}++interface SentSegment {+ file: string;+ bootId: string;+ dueAt: number;+ bytes: number;+}++function listSent(paths: SpoolPaths): SentSegment[] {+ const segments: SentSegment[] = [];+ let entries: string[];+ try {+ entries = readdirSync(paths.dir);+ } catch {+ return segments;+ }+ const prefix = `${paths.name}.sent-`;+ for (const entry of entries) {+ if (!entry.startsWith(prefix)) continue;+ const match = entry.slice(prefix.length).match(SENT_FILE_RE);+ if (!match) continue;+ const file = join(paths.dir, entry);+ segments.push({ file, bootId: match[1]!, dueAt: Number(match[2]), bytes: fileSize(file) });+ }+ return segments;+}++function sentBytes(paths: SpoolPaths): number {+ return listSent(paths).reduce((n, s) => n + s.bytes, 0);+}++function evictSent(paths: SpoolPaths, need: number): void {+ let freed = 0;+ for (const seg of listSent(paths).sort((a, b) => a.dueAt - b.dueAt)) {+ if (freed >= need) return;+ try {+ unlinkSync(seg.file);+ freed += seg.bytes;+ } catch {}+ }+}++export function parseSentMark(body: unknown): SentMark | null {+ if (!body || typeof body !== "object" || Array.isArray(body)) return null;+ const bootId = (body as Record<string, unknown>)["bootId"];+ const after = (body as Record<string, unknown>)["durableAfterMs"];+ if (typeof bootId !== "string" || !BOOT_ID_RE.test(bootId)) return null;+ if (typeof after !== "number" || !Number.isFinite(after) || after < 0) return null;+ return { bootId, durableAfterMs: Math.min(Math.ceil(after), MAX_DURABLE_AFTER_MS) };+}++export function retainSent(+ url: string,+ eventId: string,+ body: Record<string, unknown>,+ mark: SentMark,+ options: { policy?: SpoolPolicy; dir?: string; now?: number } = {},+): boolean {+ const policy = options.policy ?? spoolPolicy();+ if (!policy.enabled || !BOOT_ID_RE.test(mark.bootId)) return false;+ const paths = spoolPaths(url, options.dir);+ const now = options.now ?? Date.now();+ try {+ ensureDir(paths.dir);+ const record: SpoolRecord = {+ v: 1,+ eventId,+ spooledAt: new Date(now).toISOString(),+ reason: "sent",+ attempts: 0,+ body: sanitizeBody(body),+ bootId: mark.bootId,+ };+ let line = JSON.stringify(record) + "\n";+ if (Buffer.byteLength(line) > policy.maxRecordBytes) {+ record.body = withoutImage(record.body);+ line = JSON.stringify(record) + "\n";+ }+ const bytes = Buffer.byteLength(line);+ if (bytes > policy.maxRecordBytes) return false;+ const dueAt = Math.ceil((now + mark.durableAfterMs) / SENT_BUCKET_MS) * SENT_BUCKET_MS;+ const file = join(paths.dir, `${paths.name}.sent-${mark.bootId}-${dueAt}.jsonl`);+ return withLock(paths.lock, () => {+ if (fileSize(paths.file) + sentBytes(paths) + bytes > policy.maxBytes) return false;+ const fd = openSync(file, "a", 0o600);+ try {+ writeSync(fd, line);+ } finally {+ closeSync(fd);+ }+ return true;+ });+ } catch {+ return false;+ }+}++export function reconcileSent(+ url: string,+ bootId: string,+ options: { policy?: SpoolPolicy; dir?: string; now?: number } = {},+): SentReconcile {+ const result: SentReconcile = { released: 0, requeued: 0 };+ const policy = options.policy ?? spoolPolicy();+ if (!policy.enabled) return result;+ const paths = spoolPaths(url, options.dir);+ const now = options.now ?? Date.now();+ for (const seg of listSent(paths)) {+ if (seg.bootId === bootId) {+ if (seg.dueAt > now) continue;+ try {+ unlinkSync(seg.file);+ result.released++;+ } catch {}+ continue;+ }+ const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${now}-${result.requeued}.jsonl`);+ try {+ renameSync(seg.file, target);+ } catch {+ continue;+ }+ try {+ utimesSync(target, 0, 0);+ } catch {}+ result.requeued++;+ }+ return result;+}++function claimFiles(paths: SpoolPaths): string[] {+ const claimed: string[] = [];+ try {+ const prefix = `${paths.name}.draining-`;+ for (const entry of readdirSync(paths.dir)) {+ if (!entry.startsWith(prefix) || !entry.endsWith(".jsonl")) continue;+ const full = join(paths.dir, entry);+ try {+ if (Date.now() - statSync(full).mtimeMs > ORPHAN_CLAIM_MS) claimed.push(full);+ } catch {}+ }+ } catch {}+ if (fileSize(paths.file) > 0) {+ const target = join(paths.dir, `${paths.name}.draining-${process.pid}-${Date.now()}.jsonl`);+ withLock(paths.lock, () => {+ try {+ renameSync(paths.file, target);+ claimed.push(target);+ } catch {}+ });+ }+ return claimed;+}++function requeueLocked(paths: SpoolPaths, leftover: SpoolRecord[]): void {+ if (leftover.length === 0) return;+ const existing = existsSync(paths.file) ? parseLines(readFileSync(paths.file, "utf-8")).records : [];+ rewriteLocked(paths, [...leftover, ...existing]);+}++export function spoolHasRecords(url: string, dir?: string): boolean {+ return fileSize(spoolPaths(url, dir).file) > 0;+}++export function drainInProgress(url: string, dir?: string): boolean {+ const paths = spoolPaths(url, dir);+ try {+ return Date.now() - statSync(paths.drainLock).mtimeMs < DRAIN_LOCK_STALE_MS;+ } catch {+ return false;+ }+}++export async function drainSpool(+ url: string,+ send: (record: SpoolRecord) => Promise<SendOutcome>,+ options: { policy?: SpoolPolicy; dir?: string; maxRecords?: number; deadlineMs?: number } = {},+): Promise<DrainResult> {+ const policy = options.policy ?? spoolPolicy();+ const result: DrainResult = { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 };+ const paths = spoolPaths(url, options.dir);+ if (!existsSync(paths.dir)) return { ...result, skipped: "empty" };+ try {+ ensureDir(paths.dir);+ } catch {}+ if (!tryLock(paths.drainLock, DRAIN_LOCK_STALE_MS)) return { ...result, skipped: "locked" };+ try {+ const files = claimFiles(paths);+ if (files.length === 0) return { ...result, skipped: "empty" };+ const records: SpoolRecord[] = [];+ for (const file of files) {+ try {+ const parsed = parseLines(readFileSync(file, "utf-8"));+ records.push(...parsed.records);+ result.corrupt += parsed.corrupt;+ } catch {}+ }+ result.claimed = records.length;+ const seen = new Set<string>();+ const now = Date.now();+ const deadline = options.deadlineMs ? now + options.deadlineMs : Number.POSITIVE_INFINITY;+ const maxRecords = options.maxRecords ?? Number.POSITIVE_INFINITY;+ const leftover: SpoolRecord[] = [];+ let stopped = false;+ let processed = 0;+ for (const rec of records) {+ if (seen.has(rec.eventId)) {+ result.duplicates++;+ continue;+ }+ seen.add(rec.eventId);+ if (isExpired(rec, policy, now)) {+ result.expired++;+ continue;+ }+ if (stopped || processed >= maxRecords || Date.now() > deadline) {+ leftover.push(rec);+ continue;+ }+ processed++;+ let outcome: SendOutcome;+ try {+ outcome = await send(rec);+ } catch (err) {+ outcome = "retry";+ result.error = err instanceof Error ? err.message : String(err);+ }+ if (outcome === "delivered") result.delivered++;+ else if (outcome === "duplicate") result.duplicates++;+ else if (outcome === "rejected") result.rejected++;+ else {+ stopped = true;+ leftover.push({ ...rec, attempts: (rec.attempts ?? 0) + 1 });+ }+ }+ result.remaining = leftover.length;+ withLock(paths.lock, () => requeueLocked(paths, leftover), 2000);+ for (const file of files) {+ try {+ unlinkSync(file);+ } catch {}+ }+ writeStats(paths, (s) => {+ s.delivered += result.delivered;+ s.duplicates += result.duplicates;+ s.rejected += result.rejected;+ s.expired += result.expired;+ s.lastDrainAt = new Date().toISOString();+ s.lastDrainDelivered = result.delivered;+ s.lastDrainDuplicates = result.duplicates;+ s.lastDrainRejected = result.rejected;+ s.lastDrainRemaining = result.remaining;+ if (result.error) s.lastDrainError = result.error;+ else delete s.lastDrainError;+ });+ return result;+ } finally {+ releaseLock(paths.drainLock);+ }+}++export function spoolSummary(url: string, options: { policy?: SpoolPolicy; dir?: string } = {}): SpoolSummary {+ const policy = options.policy ?? spoolPolicy();+ const paths = spoolPaths(url, options.dir);+ let records = 0;+ let bytes = 0;+ let oldestAt: string | null = null;+ const files: string[] = [];+ if (existsSync(paths.file)) files.push(paths.file);+ try {+ for (const entry of readdirSync(paths.dir)) {+ if (entry.startsWith(`${paths.name}.draining-`) && entry.endsWith(".jsonl")) files.push(join(paths.dir, entry));+ }+ } catch {}+ for (const file of files) {+ try {+ const text = readFileSync(file, "utf-8");+ bytes += Buffer.byteLength(text);+ for (const rec of parseLines(text).records) {+ records++;+ if (!oldestAt || rec.spooledAt < oldestAt) oldestAt = rec.spooledAt;+ }+ } catch {}+ }+ let retained = 0;+ let retainedBytes = 0;+ for (const seg of listSent(paths)) {+ try {+ const text = readFileSync(seg.file, "utf-8");+ retainedBytes += Buffer.byteLength(text);+ retained += parseLines(text).records.length;+ } catch {}+ }+ return {+ enabled: policy.enabled,+ path: paths.file,+ records,+ bytes,+ oldestAt,+ retained,+ retainedBytes,+ maxBytes: policy.maxBytes,+ maxAgeHours: Math.round(policy.maxAgeMs / 3600_000),+ stats: readSpoolStats(paths),+ };+}diff --git a/src/cli.ts b/src/cli.tsindex e5d22335..6fbd15d3 100644--- a/src/cli.ts+++ b/src/cli.ts@@ -92,7 +92,8 @@ import { knownAgents } from "./cli/connect/index.js";const ALL_TOOLS_COUNT = getAllTools().length;const CORE_TOOLS_COUNT = getAllTools().filter((t) => ESSENTIAL_TOOLS.has(t.name)).length;import { resolveDataDir } from "./cli-data-dir.js";-import { bearerHeaders } from "./secret-store.js";+import { runCaptureCommand } from "./cli/capture.js";+import { bearerHeaders, resolveClientSecret } from "./secret-store.js";const __dirname = dirname(fileURLToPath(import.meta.url));const args = process.argv.slice(2);@@ -223,6 +224,9 @@ Commands:the engine was started natively but state file is missing).mcp Start standalone MCP shim — opt-in surface for MCP-only clients(Cursor, Gemini CLI, etc). REST always available at :3111.+ capture [--drain] Show capture health: the local offline spool and the server+ inbox (pending, retrying, dead letters). --drain sends+ spooled observations now. --json for machine output.import-jsonl [p] Import Claude Code JSONL transcripts (default: ~/.claude/projects)--max-files <N> | --max-files=<N>: override scan cap (default 200, max 1000;out-of-range is rejected; for trees >1000 files, batch by subdirectory)@@ -275,7 +279,7 @@ if (toolsIdx !== -1 && args[toolsIdx + 1]) {process.env["AGENTMEMORY_TOOLS"] = toolsMode;}-const URL_CLIENT_COMMANDS = new Set(["status", "doctor", "mcp"]);+const URL_CLIENT_COMMANDS = new Set(["status", "doctor", "mcp", "capture"]);let hasExplicitLocalPortOverride = false;let selectedInstance = 0;@@ -4093,6 +4097,15 @@ async function runConsole(): Promise<void> {process.exit(code);}+async function runCapture(): Promise<void> {+ const code = await runCaptureCommand({+ base: getBaseUrl(),+ args: args.slice(1),+ secret: resolveClientSecret(getBaseUrl()),+ });+ process.exit(code);+}+const commands: Record<string, () => Promise<void>> = {init: runInit,connect: runConnectCmd,@@ -4105,6 +4118,7 @@ const commands: Record<string, () => Promise<void>> = {remove: runRemove,mcp: runMcp,"import-jsonl": runImportJsonl,+ capture: runCapture,};const first = args[0] ?? "";diff --git a/src/cli/capture.ts b/src/cli/capture.tsnew file mode 100644index 00000000..2cfdb67e--- /dev/null+++ b/src/cli/capture.ts@@ -0,0 +1,105 @@+import { drainSpool, parseSentMark, retainSent, spoolSummary, type SendOutcome, type SpoolRecord } from "../capture/spool.js";++interface CaptureCommandOptions {+ base: string;+ args: string[];+ secret?: string;+ log?: (line: string) => void;+}++function headers(secret?: string): Record<string, string> {+ const h: Record<string, string> = { "Content-Type": "application/json" };+ if (secret) h["Authorization"] = `Bearer ${secret}`;+ return h;+}++function classify(status: number): SendOutcome {+ if (status === 200) return "duplicate";+ if (status >= 200 && status < 300) return "delivered";+ if (status === 408 || status === 429 || status >= 500) return "retry";+ return "rejected";+}++async function serverCapture(base: string, secret?: string): Promise<unknown> {+ try {+ const res = await fetch(`${base}/agentmemory/capture?limit=20`, {+ headers: headers(secret),+ signal: AbortSignal.timeout(5000),+ });+ if (!res.ok) return { error: `HTTP ${res.status}` };+ return await res.json();+ } catch (err) {+ return { error: err instanceof Error ? err.message : String(err) };+ }+}++export async function runCaptureCommand(options: CaptureCommandOptions): Promise<number> {+ const log = options.log ?? ((line: string) => console.log(line));+ const json = options.args.includes("--json");+ const drain = options.args.includes("--drain");+ const output: Record<string, unknown> = {};++ if (drain) {+ output["drain"] = await drainSpool(options.base, async (record: SpoolRecord) => {+ const res = await fetch(`${options.base}/agentmemory/observe`, {+ method: "POST",+ headers: headers(options.secret),+ body: JSON.stringify({ ...record.body, eventId: record.eventId }),+ signal: AbortSignal.timeout(10_000),+ });+ const text = await res.text().catch(() => "");+ const outcome = classify(res.status);+ if (outcome === "delivered" || outcome === "duplicate") {+ let mark = null;+ try {+ mark = parseSentMark(JSON.parse(text));+ } catch {}+ if (mark) retainSent(options.base, record.eventId, record.body, mark);+ }+ return outcome;+ });+ }+ output["spool"] = spoolSummary(options.base);+ output["server"] = await serverCapture(options.base, options.secret);++ if (json) {+ log(JSON.stringify(output, null, 2));+ return 0;+ }++ const spool = output["spool"] as ReturnType<typeof spoolSummary>;+ if (drain) {+ const d = output["drain"] as Awaited<ReturnType<typeof drainSpool>>;+ log(+ d.skipped+ ? `Drain skipped: ${d.skipped === "locked" ? "another drain is running" : "nothing to send"}`+ : `Drained ${d.claimed}: ${d.delivered} stored, ${d.duplicates} already stored, ${d.rejected} rejected, ${d.expired} expired, ${d.remaining} still waiting${d.error ? ` (${d.error})` : ""}`,+ );+ }+ log(`Local spool: ${spool.enabled ? `${spool.records} waiting, ${spool.bytes} bytes` : "off (AGENTMEMORY_CAPTURE_SPOOL=false)"}`);+ log(` file: ${spool.path}`);+ log(` limits: ${spool.maxBytes} bytes, ${spool.maxAgeHours} h`);+ log(` totals: ${spool.stats.spooled} spooled, ${spool.stats.delivered} delivered, ${spool.stats.duplicates} duplicates, ${spool.stats.dropped} dropped, ${spool.stats.expired} expired`);+ if (spool.stats.lastDrainAt) log(` last drain: ${spool.stats.lastDrainAt}`);+ if (spool.stats.lastDropAt) log(` last drop: ${spool.stats.lastDropAt} (${spool.stats.lastDropReason ?? "unknown"})`);++ const server = output["server"] as {+ error?: string;+ capture?: { inbox?: { pending: number; retrying: number; dead: number; lastError: string | null } | null; sinceStart?: Record<string, number> } | null;+ items?: Array<{ eventId: string; status: string; attempts: number; lastError?: string; preview?: string }>;+ };+ if (server.error) {+ log(`Server: unreachable at ${options.base} (${server.error})`);+ return 0;+ }+ const inbox = server.capture?.inbox;+ log(`Server inbox: ${inbox ? `${inbox.pending} pending, ${inbox.retrying} retrying, ${inbox.dead} dead letters` : "unknown"}`);+ if (inbox?.lastError) log(` last error: ${inbox.lastError}`);+ for (const item of server.items ?? []) {+ log(` ${item.status.padEnd(8)} ${item.eventId} attempts=${item.attempts} ${item.preview ?? ""}`);+ }+ if (inbox && inbox.dead > 0) {+ log(`Retry dead letters: curl -X POST ${options.base}/agentmemory/capture/retry -H "Content-Type: application/json" -d '{"all":true}'`);+ }+ return 0;+}diff --git a/src/cli/engine-config.ts b/src/cli/engine-config.tsindex 6f0a4136..37f1f7ac 100644--- a/src/cli/engine-config.ts+++ b/src/cli/engine-config.ts@@ -45,6 +45,14 @@ export function engineFlushWaitMs(return engineSaveIntervalMs(configTexts) + ENGINE_FLUSH_MARGIN_MS;}+export function captureDurableAfterMs(+ stateBackend: "file" | "redis",+ configTexts: readonly string[],+): number {+ if (stateBackend === "redis") return ENGINE_FLUSH_MARGIN_MS;+ return engineSaveIntervalMs(configTexts) + ENGINE_FLUSH_MARGIN_MS;+}+export function engineSaveIntervalMs(configTexts: readonly string[]): number {const intervals: number[] = [];for (const text of configTexts) {diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.tsindex bb778625..3ac936c8 100644--- a/src/functions/auto-forget.ts+++ b/src/functions/auto-forget.ts@@ -1,4 +1,5 @@import type { IIIClient } from "iii-sdk";+import { markCaptureEventDeleted } from "../capture/event-record.js";import type { Memory, CompressedObservation, Session } from "../types.js";import { KV } from "../state/schema.js";import { StateKV } from "../state/kv.js";@@ -167,6 +168,7 @@ export function registerAutoForgetFunction(sdk: IIIClient, kv: StateKV): void {if (!dryRun) {let deletedOk = false;try {+ await markCaptureEventDeleted(kv, obs);await kv.delete(KV.observations(sessions[i].id), obs.id);deletedOk = true;} catch {diff --git a/src/functions/capture.ts b/src/functions/capture.tsnew file mode 100644index 00000000..f2f25da9--- /dev/null+++ b/src/functions/capture.ts@@ -0,0 +1,607 @@+import { createHash, randomBytes } from "node:crypto";+import { readFileSync } from "node:fs";+import { homedir } from "node:os";+import { join } from "node:path";+import type { IIIClient } from "iii-sdk";+import type { CompressedObservation, HookPayload } from "../types.js";+import { KV, generateId } from "../state/schema.js";+import type { StateKV } from "../state/kv.js";+import { withKeyedLock } from "../state/keyed-mutex.js";+import { getEnvVar, getStateBackend } from "../config.js";+import { logger } from "../logger.js";+import { isValidEventId } from "../capture/event-id.js";+import {+ EVENT_SHARD_CHARS,+ EVENT_SHARDS,+ captureEventScope as eventScope,+ type CompletedEvent,+} from "../capture/event-record.js";+import { restoreIndexEntries } from "./observe.js";+import { scrubRecord } from "./privacy.js";+import { runtimeConfigPath } from "../cli/engine-launch.js";+import { captureDurableAfterMs, engineStateConfigPaths } from "../cli/engine-config.js";+import {+ drainSpool,+ reconcileSent,+ retainSent,+ spoolDir,+ spoolPolicy,+ spoolSummary,+ type DrainResult,+ type SendOutcome,+ type SentMark,+ type SpoolRecord,+ type SpoolSummary,+} from "../capture/spool.js";++export type InboxState = "pending" | "retrying" | "dead";++export interface InboxRecord {+ key: string;+ eventId: string;+ eventSource: "client" | "server";+ sessionId: string;+ project: string;+ hookType: string;+ observationId: string;+ status: InboxState;+ attempts: number;+ acceptedAt: string;+ updatedAt: string;+ nextAttemptAt?: string;+ deadAt?: string;+ lastError?: string;+ payload: HookPayload;+}++export type { CompletedEvent } from "../capture/event-record.js";+export { captureEventShard } from "../capture/event-record.js";++export type CaptureResult =+ | { status: "accepted"; state: "completed"; eventId: string; observationId: string; attempts: number }+ | { status: "accepted"; state: "retrying"; eventId: string; observationId: string; attempts: number; nextAttemptAt: string; error: string }+ | { status: "duplicate"; state: "completed" | InboxState; eventId: string; observationId?: string; deduplicated: true }+ | { status: "rejected"; eventId: string; error: string; retryable: boolean; observationId?: string };++export interface CapturePolicy {+ maxAttempts: number;+ retryIntervalMs: number;+ dedupRetentionHours: number;+ inboxMax: number;+ deadMax: number;+ eventsMax: number;+}++export interface CaptureStatus {+ policy: CapturePolicy;+ inbox: {+ pending: number;+ retrying: number;+ dead: number;+ oldestAcceptedAt: string | null;+ nextAttemptAt: string | null;+ lastError: string | null;+ } | null;+ sinceStart: {+ accepted: number;+ completed: number;+ duplicates: number;+ retried: number;+ recovered: number;+ deadLettered: number;+ rejected: number;+ pruned: number;+ };+ lastSweepAt: string | null;+ lastPruneAt: string | null;+ lastBootDrain: (DrainResult & { at: string }) | null;+ spool: SpoolSummary[];+}++const MAX_BACKOFF_MS = 15 * 60_000;+const ERROR_MAX_CHARS = 500;++function positiveInt(key: string, fallback: number): number {+ const raw = getEnvVar(key);+ if (!raw || !/^\d+$/.test(raw.trim())) return fallback;+ const n = parseInt(raw.trim(), 10);+ return n > 0 ? n : fallback;+}++export function capturePolicy(): CapturePolicy {+ return {+ maxAttempts: positiveInt("AGENTMEMORY_CAPTURE_MAX_ATTEMPTS", 5),+ retryIntervalMs: positiveInt("AGENTMEMORY_CAPTURE_RETRY_INTERVAL_MS", 10_000),+ dedupRetentionHours: positiveInt("AGENTMEMORY_CAPTURE_DEDUP_HOURS", 168),+ inboxMax: positiveInt("AGENTMEMORY_CAPTURE_INBOX_MAX", 10_000),+ deadMax: positiveInt("AGENTMEMORY_CAPTURE_DEAD_MAX", 1000),+ eventsMax: positiveInt("AGENTMEMORY_CAPTURE_EVENTS_MAX", 100_000),+ };+}++export function captureKey(project: string, sessionId: string, eventId: string): string {+ const hash = createHash("sha256").update(`${project}\u0000${sessionId}\u0000${eventId}`).digest("hex");+ return `cap_${hash.slice(0, 40)}`;+}++const MIN_ID_TIME = Date.UTC(2020, 0, 1);+const MAX_ID_TIME = Date.UTC(2100, 0, 1);++export function observationIdFor(key: string, timestamp: string | undefined, acceptedAt: number): string {+ const at = typeof timestamp === "string" ? Date.parse(timestamp) : NaN;+ const stamp = Number.isFinite(at) && at >= MIN_ID_TIME && at < MAX_ID_TIME ? at : acceptedAt;+ return `obs_${stamp.toString(36)}_${key.slice(4, 16)}`;+}++function backoffMs(attempts: number, policy: CapturePolicy): number {+ return Math.min(policy.retryIntervalMs * 2 ** Math.max(0, attempts - 1), MAX_BACKOFF_MS);+}++function errorText(err: unknown): string {+ return (err instanceof Error ? err.message : String(err)).slice(0, ERROR_MAX_CHARS);+}++function serverSpoolTargets(restPort: number): string[] {+ const url = `http://localhost:${restPort}`;+ const dirs = new Set<string>([spoolDir()]);+ try {+ const { AGENTMEMORY_DATA_DIR: _workerDataDir, AGENTMEMORY_CAPTURE_SPOOL_DIR: _spoolDir, ...hookEnv } = process.env;+ dirs.add(spoolDir(hookEnv));+ } catch {}+ return [...dirs].map((dir) => `${url}|${dir}`);+}++function splitTarget(target: string): { url: string; dir: string } {+ const at = target.indexOf("|");+ return { url: target.slice(0, at), dir: target.slice(at + 1) };+}++export function serverDurableAfterMs(): number {+ let backend: "file" | "redis" = "file";+ try {+ backend = getStateBackend();+ } catch {}+ const configTexts: string[] = [];+ const dataDir = getEnvVar("AGENTMEMORY_DATA_DIR");+ if (dataDir) {+ for (const path of engineStateConfigPaths(join(homedir(), ".agentmemory"), runtimeConfigPath(dataDir))) {+ try {+ configTexts.push(readFileSync(path, "utf-8"));+ } catch {}+ }+ }+ return captureDurableAfterMs(backend, configTexts);+}++let activeController: CaptureController | null = null;++export function getCaptureController(): CaptureController | null {+ return activeController;+}++export interface CaptureController {+ sweep(): Promise<{ processed: number; recovered: number; deadLettered: number }>;+ prune(): Promise<number>;+ drainLocalSpool(): Promise<DrainResult[]>;+ status(): Promise<CaptureStatus>;+ durability(): SentMark;+ start(): void;+ stop(): void;+}++export function registerCaptureFunctions(+ sdk: IIIClient,+ kv: StateKV,+ options: { restPort?: number; durableAfterMs?: number } = {},+): CaptureController {+ const policy = capturePolicy();+ const mark: SentMark = {+ bootId: randomBytes(12).toString("hex"),+ durableAfterMs: options.durableAfterMs ?? serverDurableAfterMs(),+ };+ const counters: CaptureStatus["sinceStart"] = {+ accepted: 0,+ completed: 0,+ duplicates: 0,+ retried: 0,+ recovered: 0,+ deadLettered: 0,+ rejected: 0,+ pruned: 0,+ };+ let lastSweepAt: string | null = null;+ let lastPruneAt: string | null = null;+ let lastBootDrain: CaptureStatus["lastBootDrain"] = null;+ let sweeping: Promise<{ processed: number; recovered: number; deadLettered: number }> | null = null;+ let inboxSize: number | null = null;+ const timers: Array<ReturnType<typeof setInterval>> = [];+ const spoolTargets = options.restPort ? serverSpoolTargets(options.restPort) : [];++ async function currentInboxSize(): Promise<number> {+ if (inboxSize === null) inboxSize = (await kv.list<InboxRecord>(KV.captureInbox)).length;+ return inboxSize;+ }++ async function complete(rec: InboxRecord, observationId: string): Promise<void> {+ const done: CompletedEvent = {+ key: rec.key,+ eventId: rec.eventId,+ sessionId: rec.sessionId,+ project: rec.project,+ observationId,+ acceptedAt: rec.acceptedAt,+ completedAt: new Date().toISOString(),+ attempts: rec.attempts,+ };+ if (rec.eventSource === "client") await kv.set(eventScope(rec.key), rec.key, done);+ await kv.delete(KV.captureInbox, rec.key);+ if (inboxSize !== null) inboxSize = Math.max(0, inboxSize - 1);+ counters.completed++;+ }++ async function storedObservation(done: CompletedEvent): Promise<CompressedObservation | null | undefined> {+ if (!done.observationId) return undefined;+ try {+ return (await kv.get<CompressedObservation>(KV.observations(done.sessionId), done.observationId)) ?? null;+ } catch {+ return undefined;+ }+ }++ async function attempt(rec: InboxRecord): Promise<CaptureResult> {+ rec.attempts++;+ type ObserveResult = { observationId?: string; deduplicated?: boolean; success?: boolean; error?: string };+ let result = null as ObserveResult | null;+ let failure: string | null = null;+ try {+ result = (await sdk.trigger({+ function_id: "mem::observe",+ payload: {+ ...rec.payload,+ ...(rec.eventSource === "client" ? { eventId: rec.eventId, captureKey: rec.key } : {}),+ observationId: rec.observationId,+ },+ })) as ObserveResult | null;+ } catch (err) {+ failure = errorText(err);+ }+ const now = new Date();+ if (!failure && result && typeof result.observationId === "string") {+ if (rec.attempts > 1) counters.recovered++;+ await complete(rec, result.observationId);+ return { status: "accepted", state: "completed", eventId: rec.eventId, observationId: result.observationId, attempts: rec.attempts };+ }+ if (!failure && result?.deduplicated) {+ await complete(rec, "");+ counters.duplicates++;+ return { status: "duplicate", state: "completed", eventId: rec.eventId, deduplicated: true };+ }+ const permanent = !failure;+ const message = failure ?? (result && typeof result.error === "string" ? result.error : "mem::observe returned no observation");+ if (permanent || rec.attempts >= policy.maxAttempts) {+ const dead: InboxRecord = { ...rec, status: "dead", updatedAt: now.toISOString(), deadAt: now.toISOString(), lastError: message };+ delete dead.nextAttemptAt;+ await kv.set(KV.captureInbox, rec.key, dead);+ counters.deadLettered++;+ logger.warn("capture moved to dead letters", { eventId: rec.eventId, attempts: rec.attempts, error: message });+ return { status: "rejected", eventId: rec.eventId, error: message, retryable: false, observationId: rec.observationId };+ }+ const nextAttemptAt = new Date(now.getTime() + backoffMs(rec.attempts, policy)).toISOString();+ const retrying: InboxRecord = { ...rec, status: "retrying", updatedAt: now.toISOString(), nextAttemptAt, lastError: message };+ await kv.set(KV.captureInbox, rec.key, retrying);+ counters.retried++;+ logger.warn("capture processing failed, will retry", { eventId: rec.eventId, attempts: rec.attempts, nextAttemptAt, error: message });+ return { status: "accepted", state: "retrying", eventId: rec.eventId, observationId: rec.observationId, attempts: rec.attempts, nextAttemptAt, error: message };+ }++ async function capture(input: { payload: HookPayload; eventId?: string }): Promise<CaptureResult> {+ const payload = input.payload;+ const clientEventId = isValidEventId(input.eventId) ? input.eventId : undefined;+ const eventId = clientEventId ?? generateId("evs");+ const key = captureKey(payload.project, payload.sessionId, eventId);+ return withKeyedLock(`capture:${key}`, async () => {+ let storedObservationId: string | undefined;+ if (clientEventId) {+ const done = await kv.get<CompletedEvent>(eventScope(key), key);+ if (done?.state === "deleted") {+ counters.duplicates++;+ return { status: "duplicate", state: "completed", eventId, observationId: done.observationId, deduplicated: true };+ }+ if (done) {+ const stored = await storedObservation(done);+ if (stored !== null) {+ if (stored) await restoreIndexEntries(stored).catch(() => {});+ counters.duplicates++;+ return { status: "duplicate", state: "completed", eventId, observationId: done.observationId, deduplicated: true };+ }+ storedObservationId = done.observationId;+ logger.warn("capture event was marked stored but its observation is missing, storing it again", {+ eventId,+ observationId: done.observationId,+ });+ }+ }+ const queued = await kv.get<InboxRecord>(KV.captureInbox, key);+ if (queued) {+ counters.duplicates++;+ return { status: "duplicate", state: queued.status, eventId, observationId: queued.observationId, deduplicated: true };+ }+ if ((await currentInboxSize()) >= policy.inboxMax) {+ counters.rejected++;+ return { status: "rejected", eventId, error: `capture inbox is full (${policy.inboxMax} unprocessed events)`, retryable: true };+ }+ const acceptedAt = Date.now();+ const { eventId: _ignoredEventId, observationId: _ignoredObservationId, captureKey: _ignoredCaptureKey, ...cleanPayload } = payload;+ const rec: InboxRecord = {+ key,+ eventId,+ eventSource: clientEventId ? "client" : "server",+ sessionId: payload.sessionId,+ project: payload.project,+ hookType: payload.hookType,+ observationId: storedObservationId ?? observationIdFor(key, payload.timestamp, acceptedAt),+ status: "pending",+ attempts: 0,+ acceptedAt: new Date(acceptedAt).toISOString(),+ updatedAt: new Date(acceptedAt).toISOString(),+ payload: { ...cleanPayload, data: scrubRecord(cleanPayload.data) },+ };+ await kv.set(KV.captureInbox, key, rec);+ if (inboxSize !== null) inboxSize++;+ counters.accepted++;+ return attempt(rec);+ });+ }++ async function retryOne(key: string, force: boolean): Promise<"recovered" | "dead" | "waiting" | "gone"> {+ return withKeyedLock(`capture:${key}`, async () => {+ const rec = await kv.get<InboxRecord>(KV.captureInbox, key);+ if (!rec) return "gone";+ if (rec.status === "dead" && !force) return "waiting";+ if (!force && rec.status === "retrying" && rec.nextAttemptAt && Date.parse(rec.nextAttemptAt) > Date.now()) return "waiting";+ if (rec.eventSource === "client" && (await kv.get<CompletedEvent>(eventScope(key), key))?.state === "deleted") {+ await kv.delete(KV.captureInbox, key);+ if (inboxSize !== null) inboxSize = Math.max(0, inboxSize - 1);+ counters.duplicates++;+ return "gone";+ }+ if (force) rec.attempts = 0;+ const outcome = await attempt({ ...rec });+ if (outcome.status === "accepted" && outcome.state === "completed") return "recovered";+ if (outcome.status === "rejected") return "dead";+ return "waiting";+ });+ }++ async function runSweep(): Promise<{ processed: number; recovered: number; deadLettered: number }> {+ const records = await kv.list<InboxRecord>(KV.captureInbox);+ inboxSize = records.length;+ let processed = 0;+ let recovered = 0;+ let deadLettered = 0;+ const now = Date.now();+ const due = records+ .filter((r) => r && r.status !== "dead")+ .filter((r) => r.status === "pending" || !r.nextAttemptAt || Date.parse(r.nextAttemptAt) <= now)+ .sort((a, b) => a.acceptedAt.localeCompare(b.acceptedAt));+ for (const rec of due) {+ const outcome = await retryOne(rec.key, false).catch(() => "waiting" as const);+ if (outcome === "gone") continue;+ processed++;+ if (outcome === "recovered") recovered++;+ if (outcome === "dead") deadLettered++;+ }+ lastSweepAt = new Date().toISOString();+ return { processed, recovered, deadLettered };+ }++ function sweep() {+ if (!sweeping) {+ sweeping = runSweep().finally(() => {+ sweeping = null;+ });+ }+ return sweeping;+ }++ async function prune(): Promise<number> {+ const cutoff = Date.now() - policy.dedupRetentionHours * 3600_000;+ const perShard = Math.max(1, Math.ceil(policy.eventsMax / EVENT_SHARDS));+ let removed = 0;+ for (let i = 0; i < EVENT_SHARDS; i++) {+ const scope = KV.captureEvents(i.toString(16).padStart(EVENT_SHARD_CHARS, "0"));+ const events = await kv.list<CompletedEvent>(scope).catch(() => [] as CompletedEvent[]);+ if (events.length === 0) continue;+ const sorted = events.filter(Boolean).sort((a, b) => b.completedAt.localeCompare(a.completedAt));+ for (let j = 0; j < sorted.length; j++) {+ const ev = sorted[j]!;+ if (j >= perShard || Date.parse(ev.deletedAt ?? ev.completedAt) < cutoff) {+ await kv.delete(scope, ev.key).catch(() => {});+ removed++;+ }+ }+ }+ const inbox = await kv.list<InboxRecord>(KV.captureInbox).catch(() => [] as InboxRecord[]);+ const dead = inbox.filter((r) => r && r.status === "dead").sort((a, b) => (b.deadAt ?? "").localeCompare(a.deadAt ?? ""));+ for (let j = 0; j < dead.length; j++) {+ const rec = dead[j]!;+ if (j >= policy.deadMax || Date.parse(rec.deadAt ?? rec.updatedAt) < cutoff) {+ await kv.delete(KV.captureInbox, rec.key).catch(() => {});+ removed++;+ }+ }+ inboxSize = null;+ counters.pruned += removed;+ lastPruneAt = new Date().toISOString();+ return removed;+ }++ function spoolSender(url: string, dir: string) {+ return async (record: SpoolRecord): Promise<SendOutcome> => {+ const body = record.body as Record<string, unknown>;+ const payload = spoolPayload(body);+ if (!payload) return "rejected";+ const outcome = await capture({ payload, eventId: record.eventId });+ if (outcome.status === "rejected") return outcome.retryable ? "retry" : "rejected";+ retainSent(url, record.eventId, body, mark, { dir });+ return outcome.status === "duplicate" ? "duplicate" : "delivered";+ };+ }++ async function settleRetained(): Promise<number> {+ let requeued = 0;+ for (const target of spoolTargets) {+ const { url, dir } = splitTarget(target);+ const settled = reconcileSent(url, mark.bootId, { dir });+ if (settled.requeued > 0) {+ await drainSpool(url, spoolSender(url, dir), { dir, policy: spoolPolicy() });+ requeued += settled.requeued;+ }+ }+ return requeued;+ }++ async function drainLocalSpool(): Promise<DrainResult[]> {+ const results: DrainResult[] = [];+ for (const target of spoolTargets) {+ const { url, dir } = splitTarget(target);+ reconcileSent(url, mark.bootId, { dir });+ results.push(await drainSpool(url, spoolSender(url, dir), { dir, policy: spoolPolicy() }));+ }+ const total = results.reduce<DrainResult>(+ (acc, r) => ({+ claimed: acc.claimed + r.claimed,+ delivered: acc.delivered + r.delivered,+ duplicates: acc.duplicates + r.duplicates,+ rejected: acc.rejected + r.rejected,+ expired: acc.expired + r.expired,+ corrupt: acc.corrupt + r.corrupt,+ remaining: acc.remaining + r.remaining,+ ...(r.error || acc.error ? { error: r.error ?? acc.error } : {}),+ }),+ { claimed: 0, delivered: 0, duplicates: 0, rejected: 0, expired: 0, corrupt: 0, remaining: 0 },+ );+ lastBootDrain = { ...total, at: new Date().toISOString() };+ return results;+ }++ async function status(): Promise<CaptureStatus> {+ let inbox: CaptureStatus["inbox"] = null;+ try {+ const records = (await kv.list<InboxRecord>(KV.captureInbox)).filter(Boolean);+ inboxSize = records.length;+ const live = records.filter((r) => r.status !== "dead");+ const next = live+ .map((r) => r.nextAttemptAt)+ .filter((v): v is string => typeof v === "string")+ .sort()[0];+ const lastFailed = records+ .filter((r) => r.lastError)+ .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))[0];+ inbox = {+ pending: records.filter((r) => r.status === "pending").length,+ retrying: records.filter((r) => r.status === "retrying").length,+ dead: records.filter((r) => r.status === "dead").length,+ oldestAcceptedAt: live.map((r) => r.acceptedAt).sort()[0] ?? null,+ nextAttemptAt: next ?? null,+ lastError: lastFailed?.lastError ?? null,+ };+ } catch {}+ const seen = new Set<string>();+ const spool: SpoolSummary[] = [];+ for (const target of spoolTargets) {+ const { url, dir } = splitTarget(target);+ const summary = spoolSummary(url, { dir });+ if (seen.has(summary.path)) continue;+ seen.add(summary.path);+ spool.push(summary);+ }+ return { policy, inbox, sinceStart: { ...counters }, lastSweepAt, lastPruneAt, lastBootDrain, spool };+ }++ sdk.registerFunction("mem::capture", async (input: { payload: HookPayload; eventId?: string }) => capture(input));++ sdk.registerFunction("mem::capture-retry", async (input: { eventId?: string; key?: string; all?: boolean } = {}) => {+ const records = (await kv.list<InboxRecord>(KV.captureInbox)).filter(Boolean);+ const targets = records.filter((r) => {+ if (input.key) return r.key === input.key;+ if (input.eventId) return r.eventId === input.eventId;+ return input.all === true;+ });+ const outcomes = { recovered: 0, dead: 0, waiting: 0 };+ for (const rec of targets) {+ const outcome = await retryOne(rec.key, true);+ if (outcome === "recovered") outcomes.recovered++;+ else if (outcome === "dead") outcomes.dead++;+ else if (outcome === "waiting") outcomes.waiting++;+ }+ return { matched: targets.length, ...outcomes };+ });++ sdk.registerFunction("mem::capture-list", async (input: { status?: InboxState; limit?: number } = {}) => {+ const limit = Math.min(Math.max(1, Number(input.limit) || 50), 500);+ const records = (await kv.list<InboxRecord>(KV.captureInbox))+ .filter((r) => r && (!input.status || r.status === input.status))+ .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));+ return {+ total: records.length,+ items: records.slice(0, limit).map(({ payload, ...rest }) => ({+ ...rest,+ preview: previewPayload(payload),+ })),+ };+ });++ sdk.registerFunction("mem::capture-drain", async () => {+ const results = await drainLocalSpool();+ return { results };+ });++ const controller: CaptureController = {+ sweep,+ prune,+ drainLocalSpool,+ status,+ durability: () => ({ ...mark }),+ start() {+ if (timers.length) return;+ const sweepTimer = setInterval(() => {+ void sweep().catch(() => {});+ void settleRetained().catch(() => {});+ }, policy.retryIntervalMs);+ sweepTimer.unref();+ const pruneTimer = setInterval(() => {+ void prune().catch(() => {});+ }, 60 * 60 * 1000);+ pruneTimer.unref();+ timers.push(sweepTimer, pruneTimer);+ },+ stop() {+ for (const t of timers.splice(0)) clearInterval(t);+ },+ };+ activeController = controller;+ return controller;+}++function spoolPayload(body: Record<string, unknown>): HookPayload | null {+ const str = (v: unknown) => (typeof v === "string" && v.trim() ? v : undefined);+ const hookType = str(body["hookType"]);+ const sessionId = str(body["sessionId"]);+ const project = str(body["project"]);+ const cwd = str(body["cwd"]);+ const timestamp = str(body["timestamp"]);+ if (!hookType || !sessionId || !project || !cwd || !timestamp) return null;+ return { hookType: hookType as HookPayload["hookType"], sessionId, project, cwd, timestamp, data: body["data"] };+}++function previewPayload(payload: HookPayload | undefined): string {+ if (!payload) return "";+ const d = payload.data && typeof payload.data === "object" ? (payload.data as Record<string, unknown>) : {};+ const label = typeof d["tool_name"] === "string" ? d["tool_name"] : payload.hookType;+ let text = "";+ try {+ text = JSON.stringify(d["tool_input"] ?? d["prompt"] ?? payload.data ?? "");+ } catch {}+ return `${label}: ${text}`.slice(0, 160);+}diff --git a/src/functions/compress-synthetic.ts b/src/functions/compress-synthetic.tsindex 8c64e208..0875a218 100644--- a/src/functions/compress-synthetic.ts+++ b/src/functions/compress-synthetic.ts@@ -107,5 +107,6 @@ export function buildSyntheticCompression(if (raw.imageData) result.imageData = raw.imageData;if (raw.agentId) result.agentId = raw.agentId;if (raw.origin) result.origin = raw.origin;+ if (raw.captureKey) result.captureKey = raw.captureKey;return result;}diff --git a/src/functions/compress.ts b/src/functions/compress.tsindex a6fabad4..4dfb0732 100644--- a/src/functions/compress.ts+++ b/src/functions/compress.ts@@ -175,6 +175,7 @@ export function registerCompressFunction(...(data.raw.imageData ? { imageRef: data.raw.imageData } : {}),...(data.raw.agentId ? { agentId: data.raw.agentId } : {}),...(data.raw.origin ? { origin: data.raw.origin } : {}),+ ...(data.raw.captureKey ? { captureKey: data.raw.captureKey } : {}),};const compressed = await withKeyedLock(`obs:${data.sessionId}`, async () => {diff --git a/src/functions/dedup.ts b/src/functions/dedup.tsindex 23f044a3..86005350 100644--- a/src/functions/dedup.ts+++ b/src/functions/dedup.ts@@ -3,6 +3,24 @@ import { createHash } from "node:crypto";const TTL_MS = 5 * 60 * 1000;const CLEANUP_INTERVAL_MS = 60_000;+let skippedSinceStart = 0;++export function recordDedupSkip(): void {+ skippedSinceStart++;+}++export function getDedupSkippedCount(): number {+ return skippedSinceStart;+}++export const DEDUP_WINDOW_MS = TTL_MS;++function outputDigest(toolOutput: unknown): string {+ if (toolOutput === undefined) return "";+ const text = typeof toolOutput === "string" ? toolOutput : JSON.stringify(toolOutput) ?? "";+ return createHash("sha256").update(text).digest("hex");+}+interface DedupEntry {hash: string;expiresAt: number;@@ -17,12 +35,13 @@ export class DedupMap {this.cleanupTimer.unref();}- computeHash(sessionId: string, toolName: string, toolInput: unknown): string {+ computeHash(sessionId: string, toolName: string, toolInput: unknown, toolOutput?: unknown): string {const input =typeof toolInput === "string"? toolInput.slice(0, 500): JSON.stringify(toolInput ?? "").slice(0, 500);- const raw = `${sessionId}:${toolName}:${input}`;+ const output = outputDigest(toolOutput);+ const raw = output ? `${sessionId}:${toolName}:${input}:${output}` : `${sessionId}:${toolName}:${input}`;return createHash("sha256").update(raw).digest("hex");}diff --git a/src/functions/evict.ts b/src/functions/evict.tsindex 1fe2b121..8c623dbe 100644--- a/src/functions/evict.ts+++ b/src/functions/evict.ts@@ -1,4 +1,5 @@import type { IIIClient } from "iii-sdk";+import { markCaptureEventDeleted } from "../capture/event-record.js";import type {Session,CompressedObservation,@@ -224,6 +225,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void {stats.lowImportanceObs++;} else {try {+ await markCaptureEventDeleted(kv, o);await kv.delete(KV.observations(session.id), o.id);stats.lowImportanceObs++;} catch (err) {@@ -268,6 +270,7 @@ export function registerEvictFunction(sdk: IIIClient, kv: StateKV): void {} else {for (const o of toEvict) {try {+ await markCaptureEventDeleted(kv, o);await kv.delete(KV.observations(o.sessionId), o.id);stats.capEvictions++;} catch (err) {diff --git a/src/functions/export-import.ts b/src/functions/export-import.tsindex 2a14ea27..eb49f012 100644--- a/src/functions/export-import.ts+++ b/src/functions/export-import.ts@@ -1,4 +1,5 @@import type { IIIClient } from "iii-sdk";+import { markCaptureEventDeleted } from "../capture/event-record.js";import type {Session,CompressedObservation,@@ -337,7 +338,7 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void// Collect observation deletes across all sessions, then run them in// one bounded pass: a runChunked nested inside a runChunked callback// multiplies in-flight deletes to chunk-size squared.- const obsDeletes: Array<{ sessionId: string; obsId: string }> = [];+ const obsDeletes: Array<{ sessionId: string; obsId: string; captureKey?: string }> = [];await runChunked(existing, async (session) => {await kv.delete(KV.sessions, session.id);await removeSessionFromProjectIndex(@@ -349,10 +350,11 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void.list<CompressedObservation>(KV.observations(session.id)).catch(() => []);for (const o of obs) {- obsDeletes.push({ sessionId: session.id, obsId: o.id });+ obsDeletes.push({ sessionId: session.id, obsId: o.id, captureKey: o.captureKey });}});await runChunked(obsDeletes, async (d) => {+ await markCaptureEventDeleted(kv, { id: d.obsId, sessionId: d.sessionId, captureKey: d.captureKey });await kv.delete(KV.observations(d.sessionId), d.obsId);await unindexObservationSession(kv, d.obsId).catch(() => {});});diff --git a/src/functions/observe.ts b/src/functions/observe.tsindex cd4ea315..ffdbe0bb 100644--- a/src/functions/observe.ts+++ b/src/functions/observe.ts@@ -8,11 +8,12 @@ import { trackViewerStreamItem, pruneViewerStreamIfDue } from "../state/viewer-simport { addSessionToProjectIndex } from "../state/session-index.js";import { indexObservationSession } from "../state/obs-index.js";import { stripPrivateData } from "./privacy.js";-import { DedupMap } from "./dedup.js";+import { DedupMap, recordDedupSkip } from "./dedup.js";import { withKeyedLock } from "../state/keyed-mutex.js";import { isAutoCompressEnabled } from "../config.js";import { buildSyntheticCompression } from "./compress-synthetic.js";-import { getSearchIndex, scheduleIndexSave, vectorIndexAddGuarded } from "./search.js";+import { isCaptureKey } from "../capture/event-record.js";+import { getSearchIndex, getVectorIndex, scheduleIndexSave, vectorIndexAddGuarded } from "./search.js";import { getAgentId } from "../config.js";import { logger } from "../logger.js";import { saveImageToDisk } from "../utils/image-store.js";@@ -42,6 +43,22 @@ export function extractImage(d: unknown): string | undefined {return undefined;}+export async function restoreIndexEntries(obs: CompressedObservation): Promise<void> {+ if (typeof obs.title !== "string" || !obs.sessionId) return;+ const search = getSearchIndex();+ if (!search.has(obs.id)) {+ search.add(obs);+ scheduleIndexSave();+ }+ const vectors = getVectorIndex();+ if (vectors && !vectors.has(obs.id)) {+ await vectorIndexAddGuarded(obs.id, obs.sessionId, obs.title + " " + (obs.narrative || ""), {+ kind: "synthetic",+ logId: obs.id,+ });+ }+}+export function registerObserveFunction(sdk: IIIClient,kv: StateKV,@@ -66,10 +83,12 @@ export function registerObserveFunction(};}- const obsId = generateId("obs");+ const durable =+ typeof payload.observationId === "string" && /^obs_[A-Za-z0-9_]{8,80}$/.test(payload.observationId);+ const obsId = durable ? payload.observationId! : generateId("obs");let dedupHash: string | undefined;- if (dedupMap) {+ if (dedupMap && typeof payload.eventId !== "string") {const dataIsObject =typeof payload.data === "object" && payload.data !== null;const d = dataIsObject@@ -84,12 +103,18 @@ export function registerObserveFunction(: dataIsObject? d: payload.data;+ const dedupOutput =+ d["tool_input"] !== undefined+ ? d["tool_response"] ?? d["tool_output"] ?? d["output"] ?? d["error"]+ : undefined;dedupHash = dedupMap.computeHash(payload.sessionId,toolName,dedupInput,+ dedupOutput,);if (dedupMap.isDuplicate(dedupHash)) {+ recordDedupSkip();return { deduplicated: true, sessionId: payload.sessionId };}}@@ -116,6 +141,8 @@ export function registerObserveFunction(channel: originChannel,capturedAt: payload.timestamp,},+ ...(typeof payload.eventId === "string" ? { eventId: payload.eventId } : {}),+ ...(isCaptureKey(payload.captureKey) ? { captureKey: payload.captureKey } : {}),};let extractedImage: string | undefined;@@ -151,6 +178,11 @@ export function registerObserveFunction(return withKeyedLock(`obs:${payload.sessionId}`, async () => {const existing = await kv.list<CompressedObservation>(KV.observations(payload.sessionId));+ const stored = durable ? existing.find((o) => o?.id === obsId) : undefined;+ if (stored) {+ await restoreIndexEntries(stored);+ return { observationId: obsId, deduplicated: true, existing: true };+ }if (maxObservationsPerSession && maxObservationsPerSession > 0) {if (existing.length >= maxObservationsPerSession) {return {diff --git a/src/functions/remember.ts b/src/functions/remember.tsindex 21fca5c6..0e9f5f5a 100644--- a/src/functions/remember.ts+++ b/src/functions/remember.ts@@ -1,4 +1,5 @@import { TriggerAction, type IIIClient } from "iii-sdk";+import { markCaptureEventDeleted } from "../capture/event-record.js";import type { Memory, Session } from "../types.js";import { KV, generateId, jaccardSimilarity } from "../state/schema.js";import { StateKV } from "../state/kv.js";@@ -270,7 +271,7 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {observationIds?: string[];memoryId?: string;}) => {- type ObservationRef = { id?: string; imageData?: string; imageRef?: string };+ type ObservationRef = { id?: string; imageData?: string; imageRef?: string; captureKey?: string };let deleted = 0;const deletedMemoryIds: string[] = [];const deletedObservationIds: string[] = [];@@ -320,6 +321,7 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {known ??(await kv.get<ObservationRef>(KV.observations(sessionId), obsId));if (!obs) return false;+ await markCaptureEventDeleted(kv, { ...obs, id: obsId, sessionId });await kv.delete(KV.observations(sessionId), obsId);deletedObservationIds.push(obsId);getSearchIndex().remove(obsId);diff --git a/src/functions/status.ts b/src/functions/status.tsindex 9cc858eb..3de59da5 100644--- a/src/functions/status.ts+++ b/src/functions/status.ts@@ -1,6 +1,7 @@import type { IndexLegStatus, IndexPersistenceStatus } from "../state/index-persistence.js";import type { VectorBackfillState } from "./search.js";import { describeGraphCompactBoot, type GraphCompactBootStatus } from "./graph-compact-boot.js";+import type { CaptureStatus } from "./capture.js";export type StatusLevel = "ok" | "info" | "warn" | "error";@@ -76,6 +77,13 @@ export interface StatusInputs {auditLegacy: { status: string; sizeBytes?: number } | null;indexPersistence?: IndexPersistenceStatus | null;stateStore?: { ok: boolean; latencyMs?: number } | null;+ capture?: CaptureStatus | null;+ observeDedup?: ObserveDedupStatus | null;+}++export interface ObserveDedupStatus {+ skippedSinceStart: number;+ windowSeconds: number;}export interface IndexBreakdown {@@ -107,6 +115,8 @@ export interface StatusReport {indexPersistence: IndexPersistenceStatus | null;graph: (GraphStatsInput & { ageSeconds: number | null; extractionEnabled: boolean }) | null;graphCompaction: GraphCompactBootStatus | null;+ capture: CaptureStatus | null;+ observeDedup: ObserveDedupStatus | null;functions: Array<FunctionMetricInput & { failureRate: number; offWithoutLlm: boolean }>;flags: Array<StatusFlag & { inactiveReason?: string }>;problems: StatusProblem[];@@ -436,6 +446,8 @@ export function evaluateStatus(input: StatusInputs): StatusReport {});}+ problems.push(...captureProblems(input.capture ?? null, input.now, input.ports.rest));+if (input.stateStore && !input.stateStore.ok) {problems.push({level: "error",@@ -479,6 +491,8 @@ export function evaluateStatus(input: StatusInputs): StatusReport {indexPersistence: persistence,graph,graphCompaction: compaction,+ capture: input.capture ?? null,+ observeDedup: input.observeDedup ?? null,functions,flags: input.flags.map((flag) =>flag.enabled && flag.needsLlm && noLlm@@ -489,6 +503,57 @@ export function evaluateStatus(input: StatusInputs): StatusReport {};}+const RECENT_DROP_MS = 24 * 60 * 60 * 1000;++export function captureRestBase(restPort: number | null): string {+ return `http://localhost:${restPort ?? 3111}/agentmemory`;+}++function captureProblems(capture: CaptureStatus | null, now: Date, restPort: number | null): StatusProblem[] {+ if (!capture) return [];+ const problems: StatusProblem[] = [];+ const base = captureRestBase(restPort);+ const inbox = capture.inbox;+ if (inbox && inbox.dead > 0) {+ problems.push({+ level: "warn",+ code: "capture-dead-letters",+ message: `${plural(inbox.dead, "captured observation")} could not be stored after ${plural(capture.policy.maxAttempts, "attempt")} and ${inbox.dead === 1 ? "is parked as a dead letter" : "are parked as dead letters"}${inbox.lastError ? `: ${inbox.lastError}` : "."}`,+ fix: `List them with curl -s '${base}/capture?status=dead'. After fixing the cause, retry them with curl -X POST ${base}/capture/retry -H "Content-Type: application/json" -d '{"all":true}'.`,+ });+ }+ if (inbox && inbox.retrying + inbox.pending > 0) {+ problems.push({+ level: "info",+ code: "capture-retrying",+ message: `${plural(inbox.retrying + inbox.pending, "accepted observation")} ${inbox.retrying + inbox.pending === 1 ? "is" : "are"} waiting to be stored${inbox.lastError ? ` after a failure: ${inbox.lastError}` : ""}.`,+ fix: `No action needed: the server retries every ${Math.round(capture.policy.retryIntervalMs / 1000)} s with backoff, up to ${capture.policy.maxAttempts} attempts, and again after a restart.`,+ });+ }+ const waiting = capture.spool.reduce((n, s) => n + s.records, 0);+ if (waiting > 0) {+ problems.push({+ level: "info",+ code: "capture-spool-waiting",+ message: `${plural(waiting, "observation")} captured while the server was unreachable ${waiting === 1 ? "is" : "are"} waiting in the local capture spool.`,+ fix: `They are sent on the next hook call and at every start. To send them now: npx @agentmemory/agentmemory capture --drain, or curl -X POST ${base}/capture/drain.`,+ });+ }+ for (const spool of capture.spool) {+ const lastDrop = spool.stats.lastDropAt ? Date.parse(spool.stats.lastDropAt) : NaN;+ if (Number.isFinite(lastDrop) && now.getTime() - lastDrop < RECENT_DROP_MS) {+ problems.push({+ level: "warn",+ code: "capture-spool-dropped",+ message: `The local capture spool dropped ${plural(spool.stats.dropped, "observation")} in total (latest reason: ${spool.stats.lastDropReason ?? "unknown"}), so those tool calls are not in memory.`,+ fix: `The spool holds at most ${Math.round(spool.maxBytes / 1024)} KiB for ${spool.maxAgeHours} hours. Raise AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES, keep agentmemory running while agents work, or check that AGENTMEMORY_CAPTURE_SPOOL is not false.`,+ });+ break;+ }+ }+ return problems;+}+function statusHeadline(status: StatusLevel, problems: StatusProblem[]): string {if (status === "ok") return "Every check passed. Capture, search and the stream are working.";if (status === "info") {@@ -599,6 +664,43 @@ function indexPersistenceRows(report: StatusReport): string {return rows;}+export function describeCaptureSpool(capture: CaptureStatus): string {+ const records = capture.spool.reduce((n, s) => n + s.records, 0);+ const bytes = capture.spool.reduce((n, s) => n + s.bytes, 0);+ if (!capture.spool.some((s) => s.enabled)) return "off (AGENTMEMORY_CAPTURE_SPOOL=false)";+ const retained = capture.spool.reduce((n, s) => n + (s.retained ?? 0), 0);+ const held = retained > 0 ? `, ${retained} sent and kept until saved` : "";+ return records === 0 ? `empty${held}` : `${plural(records, "observation")} waiting, ${Math.ceil(bytes / 1024)} KiB${held}`;+}++function captureRows(report: StatusReport): string {+ const capture = report.capture;+ if (!capture) return row("Capture", "not reported");+ const inbox = capture.inbox;+ const since = capture.sinceStart;+ let rows = row(+ "Inbox",+ escapeHtml(inbox ? `${inbox.pending} pending · ${inbox.retrying} retrying · ${inbox.dead} dead letters` : "not checked") ++ '<p class="note">Every accepted observation is written here first and removed once it is stored, so a failure or restart retries it instead of losing it.</p>',+ );+ rows += row(+ "Since start",+ escapeHtml(`${since.accepted} accepted · ${since.completed} stored · ${since.duplicates} duplicates skipped · ${since.recovered} recovered by retry · ${since.deadLettered} dead`),+ );+ rows += row(+ "Local spool",+ escapeHtml(describeCaptureSpool(capture)) ++ `<p class="note">Hooks write here when the server is unreachable, and keep each accepted observation here until the server has saved it to disk. ${escapeHtml(capture.spool.map((s) => s.path).join(", "))}</p>`,+ );+ const drained = capture.spool.map((s) => s.stats.lastDrainAt).filter(Boolean).sort().pop();+ if (drained) rows += row("Last spool drain", escapeHtml(`${formatDuration(secondsBetween(new Date(report.checkedAt), drained))} ago`));+ rows += row(+ "Duplicate protection",+ escapeHtml(`event ids kept ${capture.policy.dedupRetentionHours} h, retries up to ${plural(capture.policy.maxAttempts, "attempt")}`),+ );+ return rows;+}+function mb(bytes: number): string {return `${Math.round(bytes / (1024 * 1024))} MB`;}@@ -728,6 +830,10 @@ ${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))}${idx.vectorBackfillState ? row("Vector backfill", escapeHtml(describeBackfillState(idx.vectorBackfillState))) : ""}${indexPersistenceRows(report)}</table>+<h2>Capture</h2><table>+${captureRows(report)}+${report.observeDedup ? row("Repeats skipped", escapeHtml(`${report.observeDedup.skippedSinceStart} since start`) + `<p class="note">${escapeHtml(`A tool call with the same input and the same output as one stored in the last ${formatDuration(report.observeDedup.windowSeconds)} is skipped when the hook sends no event id.`)}</p>`) : ""}+</table><h2>Knowledge graph</h2><table>${graph? row("Extraction", graph.extractionEnabled ? "on" : "off") +diff --git a/src/hooks/_capture.ts b/src/hooks/_capture.tsnew file mode 100644index 00000000..c17fc669--- /dev/null+++ b/src/hooks/_capture.ts@@ -0,0 +1,126 @@+import { spawn } from "node:child_process";+import { resolveClientSecret } from "../secret-store.js";+import { deriveEventId } from "../capture/event-id.js";+import {+ appendSpool,+ drainInProgress,+ drainSpool,+ parseSentMark,+ reconcileSent,+ retainSent,+ spoolHasRecords,+ type SendOutcome,+ type SentMark,+ type SpoolRecord,+} from "../capture/spool.js";++export const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";+const SECRET = resolveClientSecret(REST_URL);+const DRAIN_CHILD_ENV = "AGENTMEMORY_CAPTURE_DRAIN_CHILD";+const DRAIN_MAX_RECORDS = 500;+const DRAIN_DEADLINE_MS = 20_000;+const DRAIN_REQUEST_TIMEOUT_MS = 5000;++export type CaptureOutcome = "delivered" | "duplicate" | "rejected" | "spooled" | "dropped";++export interface ObserveBody {+ hookType: string;+ sessionId: string;+ project: string;+ cwd: string;+ timestamp: string;+ data: unknown;+ eventId?: string;+}++export function authHeaders(): Record<string, string> {+ const h: Record<string, string> = { "Content-Type": "application/json" };+ if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;+ return h;+}++export function withEventId(body: ObserveBody, host: Record<string, unknown>, content?: unknown): ObserveBody {+ return {+ ...body,+ eventId: deriveEventId(body.hookType, body.sessionId, host, content ?? body.data),+ };+}++function classify(status: number): SendOutcome {+ if (status >= 200 && status < 300) return status === 200 ? "duplicate" : "delivered";+ if (status === 408 || status === 429 || status >= 500) return "retry";+ return "rejected";+}++async function post(body: Record<string, unknown>, timeoutMs: number): Promise<{ outcome: SendOutcome; mark: SentMark | null }> {+ const res = await fetch(`${REST_URL}/agentmemory/observe`, {+ method: "POST",+ headers: authHeaders(),+ body: JSON.stringify(body),+ signal: AbortSignal.timeout(timeoutMs),+ });+ const text = await res.text().catch(() => "");+ const outcome = classify(res.status);+ if (outcome !== "delivered" && outcome !== "duplicate") return { outcome, mark: null };+ try {+ return { outcome, mark: parseSentMark(JSON.parse(text)) };+ } catch {+ return { outcome, mark: null };+ }+}++function keepUntilDurable(eventId: string, body: Record<string, unknown>, mark: SentMark): number {+ const { requeued } = reconcileSent(REST_URL, mark.bootId);+ retainSent(REST_URL, eventId, body, mark);+ return requeued;+}++function startDrainChild(): void {+ if (process.env[DRAIN_CHILD_ENV] === "1") return;+ if (!process.argv[1] || drainInProgress(REST_URL)) return;+ try {+ const child = spawn(process.execPath, [process.argv[1]], {+ detached: true,+ stdio: "ignore",+ windowsHide: true,+ env: { ...process.env, [DRAIN_CHILD_ENV]: "1" },+ });+ child.on("error", () => {});+ child.unref();+ } catch {}+}++export async function captureObservation(body: ObserveBody, timeoutMs: number): Promise<CaptureOutcome> {+ const eventId = body.eventId ?? deriveEventId(body.hookType, body.sessionId, null, body.data);+ const payload = { ...body, eventId } as unknown as Record<string, unknown>;+ let reason: string;+ try {+ const { outcome, mark } = await post(payload, timeoutMs);+ if (outcome !== "retry") {+ const requeued = mark ? keepUntilDurable(eventId, payload, mark) : 0;+ if (requeued > 0 || spoolHasRecords(REST_URL)) startDrainChild();+ return outcome;+ }+ reason = "server-error";+ } catch (err) {+ const name = err instanceof Error ? err.name : "";+ reason = name === "TimeoutError" || name === "AbortError" ? "timeout" : "unreachable";+ }+ return appendSpool(REST_URL, eventId, payload, reason).spooled ? "spooled" : "dropped";+}++export function isDrainChild(): boolean {+ return process.env[DRAIN_CHILD_ENV] === "1";+}++export async function runDrainChild(): Promise<void> {+ await drainSpool(+ REST_URL,+ async (record: SpoolRecord) => {+ const { outcome, mark } = await post({ ...record.body, eventId: record.eventId }, DRAIN_REQUEST_TIMEOUT_MS);+ if (mark) retainSent(REST_URL, record.eventId, record.body, mark);+ return outcome;+ },+ { maxRecords: DRAIN_MAX_RECORDS, deadlineMs: DRAIN_DEADLINE_MS },+ ).catch(() => undefined);+}diff --git a/src/hooks/notification.ts b/src/hooks/notification.tsindex 07990205..694675c8 100644--- a/src/hooks/notification.ts+++ b/src/hooks/notification.ts@@ -1,6 +1,6 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import { resolveProject, hookCwd } from "./_project.js";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";function isSdkChildContext(payload: unknown): boolean {if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true;@@ -8,16 +8,11 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 2000;+const EXIT_CAP_MS = 2500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -42,24 +37,25 @@ async function main() {const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "notification",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: new Date().toISOString(),- data: {- notification_type: notificationType,- title: data.title,- message: data.message,+ void captureObservation(+ withEventId(+ {+ hookType: "notification",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: new Date().toISOString(),+ data: {+ notification_type: notificationType,+ title: data.title,+ message: data.message,+ },},- }),- signal: AbortSignal.timeout(2000),- }).catch(() => {});- setTimeout(() => process.exit(0), 2000).unref();+ data,+ ),+ OBSERVE_TIMEOUT_MS,+ );+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));diff --git a/src/hooks/post-tool-failure.ts b/src/hooks/post-tool-failure.tsindex 08eccf32..5b26bb48 100644--- a/src/hooks/post-tool-failure.ts+++ b/src/hooks/post-tool-failure.ts@@ -1,7 +1,7 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import { shouldCaptureTool } from "./_capture-filter.js";import { resolveProject, hookCwd } from "./_project.js";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";function isSdkChildContext(payload: unknown): boolean {if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true;@@ -9,16 +9,11 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 3000;+const EXIT_CAP_MS = 3500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -44,30 +39,31 @@ async function main() {const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "post_tool_failure",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: new Date().toISOString(),- data: {- tool_name: toolName,- tool_input:- typeof toolInput === "string"- ? toolInput.slice(0, 4000)- : JSON.stringify(toolInput ?? "").slice(0, 4000),- error:- typeof error === "string"- ? error.slice(0, 4000)- : JSON.stringify(error ?? "").slice(0, 4000),+ void captureObservation(+ withEventId(+ {+ hookType: "post_tool_failure",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: new Date().toISOString(),+ data: {+ tool_name: toolName,+ tool_input:+ typeof toolInput === "string"+ ? toolInput.slice(0, 4000)+ : JSON.stringify(toolInput ?? "").slice(0, 4000),+ error:+ typeof error === "string"+ ? error.slice(0, 4000)+ : JSON.stringify(error ?? "").slice(0, 4000),+ },},- }),- signal: AbortSignal.timeout(3000),- }).catch(() => {});- setTimeout(() => process.exit(0), 3000).unref();+ data,+ ),+ OBSERVE_TIMEOUT_MS,+ );+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));diff --git a/src/hooks/post-tool-use.ts b/src/hooks/post-tool-use.tsindex e62dae1c..58ea3db2 100644--- a/src/hooks/post-tool-use.ts+++ b/src/hooks/post-tool-use.ts@@ -1,11 +1,11 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import {captureOutputMax,shouldCaptureTool,truncateCaptureOutput,} from "./_capture-filter.js";import { resolveProject, hookCwd } from "./_project.js";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";function isSdkChildContext(payload: unknown): boolean {if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true;@@ -13,16 +13,11 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 3000;+const EXIT_CAP_MS = 3500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -48,10 +43,9 @@ async function main() {const cwd = hookCwd(data) || process.cwd();const outputMax = captureOutputMax();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({+ const toolOutputText = truncateCaptureOutput(cleanOutput, outputMax);+ const body = withEventId(+ {hookType: "post_tool_use",sessionId,project: resolveProject(cwd),@@ -60,13 +54,15 @@ async function main() {data: {tool_name: toolName,tool_input: toolInput,- tool_output: truncateCaptureOutput(cleanOutput, outputMax),+ tool_output: toolOutputText,...(imageData ? { image_data: imageData } : {}),},- }),- signal: AbortSignal.timeout(3000),- }).catch(() => {});- setTimeout(() => process.exit(0), 3000).unref();+ },+ data,+ { tool_name: toolName, tool_input: toolInput, tool_output: toolOutputText, image_data: imageData },+ );+ void captureObservation(body, OBSERVE_TIMEOUT_MS);+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}function toolOutput(data: Record<string, unknown>): unknown {diff --git a/src/hooks/prompt-submit.ts b/src/hooks/prompt-submit.tsindex 443e9ef9..0029d517 100644--- a/src/hooks/prompt-submit.ts+++ b/src/hooks/prompt-submit.ts@@ -1,6 +1,6 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import { resolveProject, hookCwd } from "./_project.js";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";function isSdkChildContext(payload: unknown): boolean {if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true;@@ -8,16 +8,11 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 3000;+const EXIT_CAP_MS = 3500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -37,20 +32,21 @@ async function main() {const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "prompt_submit",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: new Date().toISOString(),- data: { prompt: data.prompt ?? data.userPrompt },- }),- signal: AbortSignal.timeout(3000),- }).catch(() => {});- setTimeout(() => process.exit(0), 3000).unref();+ void captureObservation(+ withEventId(+ {+ hookType: "prompt_submit",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: new Date().toISOString(),+ data: { prompt: data.prompt ?? data.userPrompt },+ },+ data,+ ),+ OBSERVE_TIMEOUT_MS,+ );+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.tsindex 87b68e27..04bdf77e 100644--- a/src/hooks/session-end.ts+++ b/src/hooks/session-end.ts@@ -1,7 +1,7 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import { readFileSync } from "node:fs";import { resolveProject, hookCwd } from "./_project.js";+import { REST_URL, authHeaders, captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";function isSdkChildContext(payload: unknown): boolean {if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true;@@ -9,15 +9,6 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}-function extractTranscriptPrompts(data: Record<string, unknown>): string[] {const path = data.transcript_path;if (typeof path !== "string" || !path.endsWith(".jsonl")) return [];@@ -52,6 +43,7 @@ function extractTranscriptPrompts(data: Record<string, unknown>): string[] {}async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -75,20 +67,15 @@ async function main() {const project = resolveProject(cwd);const timestamp = new Date().toISOString();await Promise.allSettled(- transcriptPrompts.map((prompt) =>- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "prompt_submit",- sessionId,- project,- cwd,- timestamp,- data: { prompt },- }),- signal: AbortSignal.timeout(3000),- }),+ transcriptPrompts.map((prompt, index) =>+ captureObservation(+ withEventId(+ { hookType: "prompt_submit", sessionId, project, cwd, timestamp, data: { prompt } },+ {},+ { source: "transcript", transcript: data.transcript_path, index, prompt },+ ),+ 3000,+ ),),);}diff --git a/src/hooks/subagent-start.ts b/src/hooks/subagent-start.tsindex d6181aa2..3c37ca3f 100644--- a/src/hooks/subagent-start.ts+++ b/src/hooks/subagent-start.ts@@ -1,6 +1,6 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import { resolveProject, hookCwd } from "./_project.js";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";// Inlined from ./sdk-guard so each hook bundles to a single self-contained// .mjs (matches the pattern used by every other hook entry in tsdown.config).@@ -10,22 +10,15 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);-// Passive telemetry only — nothing reads the response, so the previous// `await` was pure latency. Tightened from 2000ms to a defensive cap so a// slow/unreachable server can't stack onto every concurrent subagent// startup (#221).-const TIMEOUT_MS = 800;--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 800;+const EXIT_CAP_MS = 1300;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -47,23 +40,24 @@ async function main() {const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "subagent_start",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: new Date().toISOString(),- data: {- agent_id: agentId,- agent_type: agentType,+ void captureObservation(+ withEventId(+ {+ hookType: "subagent_start",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: new Date().toISOString(),+ data: {+ agent_id: agentId,+ agent_type: agentType,+ },},- }),- signal: AbortSignal.timeout(TIMEOUT_MS),- }).catch(() => {});- setTimeout(() => process.exit(0), TIMEOUT_MS).unref();+ data,+ ),+ OBSERVE_TIMEOUT_MS,+ );+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));diff --git a/src/hooks/subagent-stop.ts b/src/hooks/subagent-stop.tsindex 7f3d946a..a201abe6 100644--- a/src/hooks/subagent-stop.ts+++ b/src/hooks/subagent-stop.ts@@ -1,6 +1,6 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import { resolveProject, hookCwd } from "./_project.js";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";function isSdkChildContext(payload: unknown): boolean {if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true;@@ -8,16 +8,11 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 2000;+const EXIT_CAP_MS = 2500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -43,24 +38,25 @@ async function main() {const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "subagent_stop",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: new Date().toISOString(),- data: {- agent_id: agentId,- agent_type: agentType,- last_message: lastMsg,+ void captureObservation(+ withEventId(+ {+ hookType: "subagent_stop",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: new Date().toISOString(),+ data: {+ agent_id: agentId,+ agent_type: agentType,+ last_message: lastMsg,+ },},- }),- signal: AbortSignal.timeout(2000),- }).catch(() => {});- setTimeout(() => process.exit(0), 2000).unref();+ data,+ ),+ OBSERVE_TIMEOUT_MS,+ );+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));diff --git a/src/hooks/task-completed.ts b/src/hooks/task-completed.tsindex 8f43f154..3d0bd396 100644--- a/src/hooks/task-completed.ts+++ b/src/hooks/task-completed.ts@@ -1,6 +1,6 @@#!/usr/bin/env node-import { resolveClientSecret } from "../secret-store.js";import { resolveProject, hookCwd } from "./_project.js";+import { captureObservation, isDrainChild, runDrainChild, withEventId } from "./_capture.js";function isSdkChildContext(payload: unknown): boolean {if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true;@@ -8,16 +8,11 @@ function isSdkChildContext(payload: unknown): boolean {return (payload as { entrypoint?: unknown }).entrypoint === "sdk-ts";}-const REST_URL = process.env["AGENTMEMORY_URL"] || "http://localhost:3111";-const SECRET = resolveClientSecret(REST_URL);--function authHeaders(): Record<string, string> {- const h: Record<string, string> = { "Content-Type": "application/json" };- if (SECRET) h["Authorization"] = `Bearer ${SECRET}`;- return h;-}+const OBSERVE_TIMEOUT_MS = 2000;+const EXIT_CAP_MS = 2500;async function main() {+ if (isDrainChild()) return runDrainChild();let input = "";for await (const chunk of process.stdin) {input += chunk;@@ -37,28 +32,29 @@ async function main() {const cwd = hookCwd(data) || process.cwd();- fetch(`${REST_URL}/agentmemory/observe`, {- method: "POST",- headers: authHeaders(),- body: JSON.stringify({- hookType: "task_completed",- sessionId,- project: resolveProject(cwd),- cwd,- timestamp: new Date().toISOString(),- data: {- task_id: data.task_id,- task_subject: data.task_subject,- task_description: typeof data.task_description === "string"- ? data.task_description.slice(0, 2000)- : "",- teammate_name: data.teammate_name,- team_name: data.team_name,+ void captureObservation(+ withEventId(+ {+ hookType: "task_completed",+ sessionId,+ project: resolveProject(cwd),+ cwd,+ timestamp: new Date().toISOString(),+ data: {+ task_id: data.task_id,+ task_subject: data.task_subject,+ task_description: typeof data.task_description === "string"+ ? data.task_description.slice(0, 2000)+ : "",+ teammate_name: data.teammate_name,+ team_name: data.team_name,+ },},- }),- signal: AbortSignal.timeout(2000),- }).catch(() => {});- setTimeout(() => process.exit(0), 2000).unref();+ data,+ ),+ OBSERVE_TIMEOUT_MS,+ );+ setTimeout(() => process.exit(0), EXIT_CAP_MS).unref();}main().catch(() => process.exit(0));diff --git a/src/index.ts b/src/index.tsindex 8b738d7b..c6e5166c 100644--- a/src/index.ts+++ b/src/index.ts@@ -34,6 +34,7 @@ import { IndexPersistence } from "./state/index-persistence.js";import { SHUTDOWN_FLUSH_TIMEOUT_MS, SHUTDOWN_HARD_EXIT_MS, settleWithin } from "./shutdown.js";import { registerPrivacyFunction, withWriteScrubbing } from "./functions/privacy.js";import { registerObserveFunction } from "./functions/observe.js";+import { registerCaptureFunctions } from "./functions/capture.js";import { seedViewerStreamTracker } from "./state/viewer-stream.js";import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js";import { registerVisionSearchFunctions } from "./functions/vision-search.js";@@ -275,6 +276,7 @@ async function main() {registerPrivacyFunction(sdk);registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession);+ const capture = registerCaptureFunctions(sdk, kv, { restPort: config.restPort });registerImageQuotaCleanup(sdk, kv);registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider);if (isSlotsEnabled()) {@@ -610,12 +612,32 @@ async function main() {`Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`,);bootLog(- `REST API: 135 endpoints at http://localhost:${config.restPort}/agentmemory/*`,+ `REST API: 138 endpoints at http://localhost:${config.restPort}/agentmemory/*`,);bootLog(`MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`,);+ void (async () => {+ try {+ const drained = await capture.drainLocalSpool();+ const delivered = drained.reduce((n, r) => n + r.delivered, 0);+ const duplicates = drained.reduce((n, r) => n + r.duplicates, 0);+ const remaining = drained.reduce((n, r) => n + r.remaining, 0);+ if (delivered + duplicates + remaining > 0) {+ bootLog(`Capture spool: ${delivered} recovered, ${duplicates} already stored, ${remaining} still waiting`);+ }+ const swept = await capture.sweep();+ if (swept.processed > 0) {+ bootLog(`Capture inbox: ${swept.recovered} of ${swept.processed} unfinished observations stored after restart`);+ }+ await capture.prune();+ } catch (err) {+ bootWarn(`Capture recovery at boot failed: ${err instanceof Error ? err.message : String(err)}`);+ }+ capture.start();+ })();+const viewerServer = startViewerServer(config.viewerPort,kv,@@ -731,6 +753,7 @@ async function main() {hardExit.unref();healthMonitor.stop();dedupMap.stop();+ capture.stop();indexPersistence.stop();const viewerClosed = new Promise<void>((resolve) =>viewerServer.close(() => resolve()),diff --git a/src/state/schema.ts b/src/state/schema.tsindex c2b3925d..08627556 100644--- a/src/state/schema.ts+++ b/src/state/schema.ts@@ -78,6 +78,8 @@ export const KV = {recentSearches: "mem:recent-searches",projectSessionsIndex: "mem:idx:project-sessions",obsSessionIndex: (shard: number) => `mem:idx:obs:${shard}`,+ captureInbox: "mem:capture:inbox",+ captureEvents: (shard: string) => `mem:capture:events:${shard}`,} as const;export const STREAM = {diff --git a/src/triggers/api.ts b/src/triggers/api.tsindex 5f81f59a..2c280f77 100644--- a/src/triggers/api.ts+++ b/src/triggers/api.ts@@ -1,4 +1,5 @@import { InvocationError, TriggerAction, type IIIClient } from "iii-sdk";+import { DEDUP_WINDOW_MS, getDedupSkippedCount } from "../functions/dedup.js";import type { HttpRequest } from "@iii-dev/helpers/http";import { randomBytes } from "node:crypto";import type { Session, CompressedObservation, HookPayload, CommitLink, SessionSummary, HealthSnapshot, AuditQueryResult, AuditMigrationState } from "../types.js";@@ -47,6 +48,8 @@ import { COMPACT_SCOPES, type GraphCompactScope } from "../functions/graph.js";import { getGraphCompactBootStatus } from "../functions/graph-compact-boot.js";import { LESSON_SOURCE_IDS_MAX, normalizeLessonSourceIds } from "../functions/lessons.js";import { logger } from "../logger.js";+import { isValidEventId } from "../capture/event-id.js";+import { getCaptureController, type CaptureResult } from "../functions/capture.js";import { withoutObservationSource } from "../functions/observation-source.js";import {isGraphExtractionEnabled,@@ -168,6 +171,21 @@ function reflectDisabledResponse(): Response {});}+export function captureStatusCode(result: CaptureResult): number {+ if (result.status === "duplicate") return 200;+ if (result.status === "accepted") return result.state === "completed" ? 201 : 202;+ return result.retryable ? 503 : 422;+}++export function captureResponseBody(+ result: CaptureResult,+ durability?: { bootId: string; durableAfterMs: number } | null,+): Record<string, unknown> {+ if (result.status === "rejected") return { ...result, success: false };+ if (!durability) return { ...result };+ return { ...result, bootId: durability.bootId, acceptedAt: new Date().toISOString(), durableAfterMs: durability.durableAfterMs };+}+function asNonEmptyString(value: unknown): string | null {if (typeof value !== "string") return null;const trimmed = value.trim();@@ -362,7 +380,7 @@ export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusReconst cached = scan.last && options.scanMaxAgeMs !== undefined && Date.now() - scan.last.at < options.scanMaxAgeMs? scan.last.value: null;- const [health, functionMetrics, graph, unindexed, auditMigrationState, stateStore] = await Promise.all([+ const [health, functionMetrics, graph, unindexed, auditMigrationState, stateStore, capture] = await Promise.all([options.health !== undefined ? Promise.resolve(options.health) : valueWithin(getLatestHealth(kv), STATUS_CHECK_TIMEOUT_MS),deps.metricsStore ? valueWithin(deps.metricsStore.getAll(), STATUS_CHECK_TIMEOUT_MS) : Promise.resolve([]),valueWithin(@@ -375,6 +393,7 @@ export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusReSTATUS_CHECK_TIMEOUT_MS,),probeStateStore(kv),+ valueWithin(getCaptureController()?.status() ?? Promise.resolve(null), STATUS_CHECK_TIMEOUT_MS),]);const observationsIndexed = [...idx.observationCountsBySession().values()].reduce((a, n) => a + n, 0);const documentKinds = idx.documentKindCounts();@@ -431,6 +450,8 @@ export function createStatusReporter(sdk: IIIClient, kv: StateKV, deps: StatusRe: null,indexPersistence: getIndexPersistenceStatus(),stateStore,+ capture: capture ?? null,+ observeDedup: { skippedSinceStart: getDedupSkippedCount(), windowSeconds: Math.round(DEDUP_WINDOW_MS / 1000) },});};}@@ -652,6 +673,16 @@ export function registerApiTriggers(},};}+ const eventId = body.eventId;+ if (eventId !== undefined && !isValidEventId(eventId)) {+ return {+ status_code: 400,+ body: {+ status: "rejected",+ error: "eventId must be 8 to 128 characters of letters, digits, '_', '.', ':' or '-'",+ },+ };+ }const payload: HookPayload = {hookType: hookType as HookPayload["hookType"],sessionId,@@ -660,10 +691,77 @@ export function registerApiTriggers(timestamp,data: body.data,};- const result = await sdk.trigger({ function_id: "mem::observe", payload });- return { status_code: 201, body: result };+ const result = await sdk.trigger<unknown, CaptureResult>({+ function_id: "mem::capture",+ payload: { payload, eventId },+ });+ return {+ status_code: captureStatusCode(result),+ body: captureResponseBody(result, getCaptureController()?.durability() ?? null),+ };+ },+ );++ sdk.registerFunction("api::capture",+ async (req: HttpRequest): Promise<Response> => {+ const authErr = checkAuth(req, secret);+ if (authErr) return authErr;+ const status = req.query_params?.["status"];+ const limit = Number(req.query_params?.["limit"]) || 50;+ const [summary, list] = await Promise.all([+ getCaptureController()?.status() ?? Promise.resolve(null),+ sdk.trigger({+ function_id: "mem::capture-list",+ payload: {+ ...(status === "pending" || status === "retrying" || status === "dead" ? { status } : {}),+ limit,+ },+ }),+ ]);+ return { status_code: 200, headers: { "Cache-Control": "no-store" }, body: { capture: summary, ...(list as object) } };+ },+ );+ sdk.registerTrigger({+ type: "http",+ function_id: "api::capture",+ config: { api_path: "/agentmemory/capture", http_method: "GET" },+ });++ sdk.registerFunction("api::capture-retry",+ async (req: HttpRequest<{ eventId?: string; all?: boolean }>): Promise<Response> => {+ const authErr = checkAuth(req, secret);+ if (authErr) return authErr;+ const body = (req.body ?? {}) as { eventId?: unknown; all?: unknown };+ const eventId = asNonEmptyString(body.eventId);+ if (!eventId && body.all !== true) {+ return { status_code: 400, body: { error: "pass an eventId, or all: true to retry every dead or waiting capture" } };+ }+ const result = await sdk.trigger({+ function_id: "mem::capture-retry",+ payload: eventId ? { eventId } : { all: true },+ });+ return { status_code: 200, body: result };+ },+ );+ sdk.registerTrigger({+ type: "http",+ function_id: "api::capture-retry",+ config: { api_path: "/agentmemory/capture/retry", http_method: "POST" },+ });++ sdk.registerFunction("api::capture-drain",+ async (req: HttpRequest): Promise<Response> => {+ const authErr = checkAuth(req, secret);+ if (authErr) return authErr;+ const result = await sdk.trigger({ function_id: "mem::capture-drain", payload: {} });+ return { status_code: 200, body: result };},);+ sdk.registerTrigger({+ type: "http",+ function_id: "api::capture-drain",+ config: { api_path: "/agentmemory/capture/drain", http_method: "POST" },+ });sdk.registerTrigger({type: "http",function_id: "api::observe",diff --git a/src/types.ts b/src/types.tsindex 2c46ec4e..cb28e20e 100644--- a/src/types.ts+++ b/src/types.ts@@ -66,6 +66,8 @@ export interface RawObservation {imageData?: string;agentId?: string;origin?: Origin;+ eventId?: string;+ captureKey?: string;}export interface ObservationSource {@@ -100,6 +102,7 @@ export interface CompressedObservation {modality?: "text" | "image" | "mixed";agentId?: string;origin?: Origin;+ captureKey?: string;}export type ObservationType =@@ -177,6 +180,9 @@ export interface HookPayload {cwd: string;timestamp: string;data: unknown;+ eventId?: string;+ observationId?: string;+ captureKey?: string;}export interface ProviderConfig {diff --git a/src/viewer/index.html b/src/viewer/index.htmlindex 1e43ea3d..c09868e8 100644--- a/src/viewer/index.html+++ b/src/viewer/index.html@@ -1645,6 +1645,8 @@vector: { term: 'Vector index', text: 'Embeddings used for meaning-based search. It needs an embedding provider; without one, search is keyword only.' },observationsIndexed: { term: 'Observations indexed', text: 'Observations that are searchable. The keyword index also contains memories and lessons, which is why its total is larger.' },missingFromIndex: { term: 'Missing from index', text: 'Stored observations that are not searchable yet. They are added in the background; a large number that does not shrink means indexing is stuck.' },+ captureInbox: { term: 'Capture inbox', text: 'Every observation a hook sends is written here first and removed once it is stored. If storing fails or the server restarts, it is retried instead of lost. Dead letters are the ones that still failed after every retry.' },+ captureSpool: { term: 'Local capture spool', text: 'A small file on this machine where hooks keep observations when the server is down or does not answer. It is sent automatically on the next hook call and at every start, and each observation carries an event id so nothing is stored twice.' },pendingBackfill: { term: 'Pending vector backfill', text: 'Records waiting for an embedding so they can be found by meaning, not only by keywords.' },graph: { term: 'Knowledge graph', text: 'Entities (files, functions, concepts, errors, decisions) and the relations between them. When a session ends, the files and concepts in its observations become nodes, and ones that appear in the same observation get a "related to" relation; no LLM is needed for that. With GRAPH_EXTRACTION_ENABLED=true and an LLM provider, an LLM also adds typed entities and relations such as "uses" or "fixes". A structural code graph can be imported with POST /graph/import-graphify.' },graphDegree: { term: 'Relations (degree)', text: 'How many relations touch this node. Nodes with more relations are drawn larger and get their labels first.' },@@ -7247,6 +7249,35 @@return html + '</div>';}+ function captureCardHtml(r) {+ var cap = r && r.capture;+ var html = '<div class="card"><div class="card-title">Capture ' + help('captureInbox') + '</div>';+ if (!cap) return html + hrow('Capture', 'waiting for the first report', null, 'Shows whether observations from hooks are stored, retried or waiting offline.') + '</div>';+ var inbox = cap.inbox;+ var since = cap.sinceStart || {};+ var waiting = (cap.spool || []).reduce(function(n, s) { return n + (s.records || 0); }, 0);+ var bytes = (cap.spool || []).reduce(function(n, s) { return n + (s.bytes || 0); }, 0);+ var spoolOn = (cap.spool || []).some(function(s) { return s.enabled; });+ var drainedAt = (cap.spool || []).map(function(s) { return s.stats && s.stats.lastDrainAt; }).filter(Boolean).sort().pop();+ var dropped = (cap.spool || []).reduce(function(n, s) { return n + ((s.stats && s.stats.dropped) || 0); }, 0);+ html += hrow('Inbox', inbox ? esc(fmtNum(inbox.pending) + ' pending · ' + fmtNum(inbox.retrying) + ' retrying · ' + fmtNum(inbox.dead) + ' dead') : 'not checked', 'captureInbox',+ inbox && inbox.dead ? '<span style="overflow-wrap:anywhere;">' + esc(inbox.lastError || 'Could not be stored.') + '</span> Fix the cause, then retry them:' + cmdBlock(curlPost('capture/retry', { all: true })) :+ inbox && inbox.retrying ? '<span style="overflow-wrap:anywhere;">' + esc('Retried every ' + durationText(cap.policy.retryIntervalMs) + ' with backoff, up to ' + cap.policy.maxAttempts + ' attempts' + (inbox.lastError ? '. Last error: ' + inbox.lastError : '.')) + '</span>' :+ 'Empty means every accepted observation is stored.');+ html += hrow('Since start', esc(fmtNum(since.accepted || 0) + ' accepted · ' + fmtNum(since.completed || 0) + ' stored · ' + fmtNum(since.duplicates || 0) + ' duplicates skipped · ' + fmtNum(since.recovered || 0) + ' recovered by retry · ' + fmtNum(since.deadLettered || 0) + ' dead'), null,+ 'Duplicates are the same event delivered twice, for example a replay after a timeout. They are kept once.');+ if (r.observeDedup) {+ html += hrow('Repeats skipped', esc(fmtNum(r.observeDedup.skippedSinceStart || 0) + ' since start'), null,+ esc('A tool call with the same input and the same output as one stored in the last ' + durationText((r.observeDedup.windowSeconds || 300) * 1000) + ' is skipped when the hook sends no event id.'));+ }+ html += hrow('Offline spool', !spoolOn ? 'off' : waiting ? esc(fmtNum(waiting) + ' waiting · ' + Math.ceil(bytes / 1024) + ' KiB') : 'empty', 'captureSpool',+ !spoolOn ? 'Turned off with AGENTMEMORY_CAPTURE_SPOOL=false, so hooks drop observations while the server is down.' :+ (waiting ? 'Sent on the next hook call or at the next start. To send now:' + cmdBlock(curlPost('capture/drain', {})) : esc('Hooks write here only while the server is unreachable.' + (drainedAt ? ' Last drained ' + timeAgo(drainedAt) + '.' : ''))) ++ (dropped ? '<div class="hnote" style="color:var(--accent);">' + esc(fmtNum(dropped) + ' dropped because the spool was full or too old. Raise AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES.') + '</div>' : ''));+ html += hrow('Inspect', esc('from a terminal'), null, 'Spool and inbox, with --drain to send the spool now:' + cmdBlock((customPort() ? 'AGENTMEMORY_URL=' + agentUrl() + ' ' : '') + 'npx @agentmemory/agentmemory capture') + 'Waiting and dead events as JSON:' + cmdBlock(curlGet('capture?status=dead')));+ return html + '</div>';+ }+function recordsCardHtml(c) {return '<div class="card"><div class="card-title">Stored records</div>' +hrow('Sessions', esc(fmtNum(c.sessions) + ' (' + fmtNum(c.activeSessions) + ' running)'), 'session') +@@ -7385,7 +7416,7 @@var c = store.counts || {};var html = '<div class="inline-note"><strong>Health</strong> ' + help('health') + ' is the live version of the <a class="empty-link" href="' + esc(REST) + '/agentmemory/status" target="_blank" rel="noopener">/agentmemory/status</a> page: the same checks and verdict, pushed over the stream every 30 seconds. Every problem says what to do. Scroll down for how consolidation works.</div>';html += healthVerdictHtml(r);- html += '<div class="hgrid">' + engineCardHtml(r, snap) + indexCardHtml(r, idx, c) + llmCardHtml(r) + flagsCardHtml(r) + processCardHtml(snap) + recordsCardHtml(c) + '</div>';+ html += '<div class="hgrid">' + engineCardHtml(r, snap) + indexCardHtml(r, idx, c) + captureCardHtml(r) + llmCardHtml(r) + flagsCardHtml(r) + processCardHtml(snap) + recordsCardHtml(c) + '</div>';html += consolidationPanelHtml(store.consolidationStatus, c);html += functionsCardHtml(r);setViewHtml(el, html);diff --git a/test/capture-deleted.test.ts b/test/capture-deleted.test.tsnew file mode 100644index 00000000..f6f7052e--- /dev/null+++ b/test/capture-deleted.test.ts@@ -0,0 +1,156 @@+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";+import { mockKV, mockSdk } from "./helpers/mocks.js";+import { KV } from "../src/state/schema.js";++vi.mock("../src/logger.js", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++type Kv = ReturnType<typeof mockKV>;++function payload(marker: string, sessionId = "ses_deleted") {+ return {+ hookType: "post_tool_use",+ sessionId,+ project: "proj",+ cwd: "/work/proj",+ timestamp: new Date().toISOString(),+ data: { tool_name: "Bash", tool_input: { command: `echo ${marker}` }, tool_output: marker },+ };+}++async function boot(kv: Kv) {+ const { registerObserveFunction } = await import("../src/functions/observe.js");+ const { registerCaptureFunctions } = await import("../src/functions/capture.js");+ const { registerRememberFunction } = await import("../src/functions/remember.js");+ const { DedupMap } = await import("../src/functions/dedup.js");+ const { getSearchIndex } = await import("../src/functions/search.js");+ const sdk = mockSdk({ looseTrigger: true });+ registerObserveFunction(sdk as never, kv as never, new DedupMap());+ registerRememberFunction(sdk as never, kv as never);+ const controller = registerCaptureFunctions(sdk as never, kv as never);+ const capture = (p: unknown, eventId?: string) =>+ sdk.trigger("mem::capture", { payload: p, eventId }) as Promise<Record<string, unknown>>;+ const forget = (sessionId: string, observationIds: string[]) =>+ sdk.trigger("mem::forget", { sessionId, observationIds }) as Promise<Record<string, unknown>>;+ return { sdk, controller, capture, forget, searchIndex: getSearchIndex };+}++function observations(kv: Kv, sessionId = "ses_deleted") {+ return [...(kv.store.get(KV.observations(sessionId))?.values() ?? [])] as Array<{ id: string; captureKey?: string }>;+}++function eventRecords(kv: Kv) {+ return [...kv.store.entries()]+ .filter(([scope]) => scope.startsWith("mem:capture:events:"))+ .flatMap(([, entries]) => [...entries.values()]) as Array<Record<string, unknown>>;+}++function clearEventRecords(kv: Kv) {+ for (const [scope, entries] of kv.store) {+ if (scope.startsWith("mem:capture:events:")) entries.clear();+ }+}++describe("deleted captured observations", () => {+ beforeEach(() => {+ vi.resetModules();+ });+ afterEach(() => {+ vi.unstubAllEnvs();+ vi.useRealTimers();+ });++ it("records the capture key on the stored observation and ignores one sent by a client", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ const forged = `cap_${"f".repeat(40)}`;+ await capture({ ...payload("keyed"), captureKey: forged }, "evc_000000000101");+ const [obs] = observations(kv);+ expect(obs!.captureKey).toMatch(/^cap_[0-9a-f]{40}$/);+ expect(obs!.captureKey).not.toBe(forged);+ expect(eventRecords(kv).map((r) => r.key)).toEqual([obs!.captureKey]);+ });++ it("keeps a forgotten observation deleted when its event is replayed", async () => {+ const kv = mockKV();+ const { capture, forget, sdk, searchIndex } = await boot(kv);+ const body = payload("forgotten");+ const first = await capture(body, "evc_000000000102");+ await forget("ses_deleted", [first.observationId as string]);+ expect(observations(kv)).toHaveLength(0);+ expect(eventRecords(kv)[0]).toMatchObject({ state: "deleted", observationId: first.observationId });++ const observeCalls = vi.fn(sdk.fns.get("mem::observe")!);+ sdk.fns.set("mem::observe", observeCalls);+ const replay = await capture(body, "evc_000000000102");+ expect(replay).toMatchObject({ status: "duplicate", observationId: first.observationId, deduplicated: true });+ expect(observeCalls).not.toHaveBeenCalled();+ expect(observations(kv)).toHaveLength(0);+ expect(searchIndex().has(first.observationId as string)).toBe(false);+ });++ it("keeps it deleted when the event record was lost in a crash before the delete", async () => {+ const kv = mockKV();+ const { capture, forget } = await boot(kv);+ const body = payload("crash-split");+ const first = await capture(body, "evc_000000000103");+ clearEventRecords(kv);+ await forget("ses_deleted", [first.observationId as string]);+ expect(eventRecords(kv)).toEqual([+ expect.objectContaining({ state: "deleted", observationId: first.observationId, sessionId: "ses_deleted" }),+ ]);+ const replay = await capture(body, "evc_000000000103");+ expect(replay).toMatchObject({ status: "duplicate", observationId: first.observationId });+ expect(observations(kv)).toHaveLength(0);+ });++ it("drops a stranded inbox entry for a deleted observation instead of storing it again", async () => {+ const kv = mockKV();+ const { capture, forget, controller } = await boot(kv);+ const body = payload("stranded-deleted");+ const first = await capture(body, "evc_000000000104");+ const [record] = eventRecords(kv);+ clearEventRecords(kv);+ await kv.set(KV.captureInbox, record!.key as string, {+ ...record,+ eventSource: "client",+ hookType: "post_tool_use",+ status: "pending",+ attempts: 1,+ updatedAt: new Date().toISOString(),+ payload: body,+ });+ await forget("ses_deleted", [first.observationId as string]);+ await controller.sweep();+ expect(observations(kv)).toHaveLength(0);+ expect([...(kv.store.get(KV.captureInbox)?.values() ?? [])]).toHaveLength(0);+ expect((await capture(body, "evc_000000000104")).status).toBe("duplicate");+ expect(observations(kv)).toHaveLength(0);+ });++ it("still stores an observation lost in a crash when nobody deleted it", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ const body = payload("lost-not-deleted");+ const first = await capture(body, "evc_000000000105");+ kv.store.get(KV.observations("ses_deleted"))!.clear();+ const resent = await capture(body, "evc_000000000105");+ expect(resent).toMatchObject({ status: "accepted", state: "completed", observationId: first.observationId });+ expect(observations(kv).map((o) => o.id)).toEqual([first.observationId]);+ });++ it("keeps the deleted marker for the retention window counted from the delete", async () => {+ const kv = mockKV();+ vi.stubEnv("AGENTMEMORY_CAPTURE_DEDUP_HOURS", "1");+ const { capture, forget, controller } = await boot(kv);+ const body = payload("retained-marker");+ const first = await capture(body, "evc_000000000106");+ vi.useFakeTimers({ now: Date.now() + 50 * 60_000, toFake: ["Date"] });+ await forget("ses_deleted", [first.observationId as string]);+ vi.setSystemTime(Date.now() + 20 * 60_000);+ expect(await controller.prune()).toBe(0);+ expect((await capture(body, "evc_000000000106")).status).toBe("duplicate");+ expect(observations(kv)).toHaveLength(0);+ });+});diff --git a/test/capture-durable.test.ts b/test/capture-durable.test.tsnew file mode 100644index 00000000..56c48c96--- /dev/null+++ b/test/capture-durable.test.ts@@ -0,0 +1,485 @@+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";+import { existsSync, mkdtempSync, readFileSync, readdirSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { mockKV, mockSdk } from "./helpers/mocks.js";+import { KV } from "../src/state/schema.js";++vi.mock("../src/logger.js", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++type Kv = ReturnType<typeof mockKV>;++function payload(marker: string, sessionId = "ses_capture") {+ return {+ hookType: "post_tool_use",+ sessionId,+ project: "proj",+ cwd: "/work/proj",+ timestamp: new Date().toISOString(),+ data: { tool_name: "Bash", tool_input: { command: `echo ${marker}` }, tool_output: marker },+ };+}++async function boot(kv: Kv, options: { restPort?: number; durableAfterMs?: number } = {}) {+ const { registerObserveFunction } = await import("../src/functions/observe.js");+ const { registerCaptureFunctions } = await import("../src/functions/capture.js");+ const { DedupMap } = await import("../src/functions/dedup.js");+ const sdk = mockSdk({ looseTrigger: true });+ const dedup = new DedupMap();+ registerObserveFunction(sdk as never, kv as never, dedup);+ const controller = registerCaptureFunctions(sdk as never, kv as never, options);+ const capture = (p: unknown, eventId?: string) =>+ sdk.trigger("mem::capture", { payload: p, eventId }) as Promise<Record<string, unknown>>;+ return { sdk, controller, capture, dedup };+}++function observations(kv: Kv, sessionId = "ses_capture") {+ return [...(kv.store.get(KV.observations(sessionId))?.values() ?? [])] as Array<{ id: string; eventId?: string }>;+}++function inbox(kv: Kv) {+ return [...(kv.store.get(KV.captureInbox)?.values() ?? [])] as Array<Record<string, unknown>>;+}++describe("durable capture", () => {+ beforeEach(() => {+ vi.resetModules();+ vi.stubEnv("AGENTMEMORY_CAPTURE_RETRY_INTERVAL_MS", "1000");+ vi.stubEnv("AGENTMEMORY_CAPTURE_MAX_ATTEMPTS", "3");+ });+ afterEach(() => {+ vi.unstubAllEnvs();+ vi.useRealTimers();+ });++ it("stores an event once and answers a second delivery with the same observation", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ const first = await capture(payload("m1"), "evc_000000000001");+ const second = await capture(payload("m1"), "evc_000000000001");+ expect(first).toMatchObject({ status: "accepted", state: "completed" });+ expect(second).toMatchObject({ status: "duplicate", deduplicated: true, observationId: first.observationId });+ expect(observations(kv)).toHaveLength(1);+ expect(inbox(kv)).toHaveLength(0);+ });++ it("collapses concurrent deliveries of the same event", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ const results = await Promise.all(Array.from({ length: 8 }, () => capture(payload("m2"), "evc_000000000002")));+ expect(results.filter((r) => r.status === "accepted")).toHaveLength(1);+ expect(results.filter((r) => r.status === "duplicate")).toHaveLength(7);+ expect(observations(kv)).toHaveLength(1);+ });++ it("keeps distinct events with identical content", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ await capture(payload("same"), "evh_000000000003");+ await capture(payload("same"), "evh_000000000004");+ expect(observations(kv)).toHaveLength(2);+ });++ it("still deduplicates a replay after a restart", async () => {+ const kv = mockKV();+ const before = await boot(kv);+ const first = await before.capture(payload("restart"), "evc_000000000005");+ before.dedup.stop();+ vi.resetModules();+ const after = await boot(kv);+ const replay = await after.capture(payload("restart"), "evc_000000000005");+ expect(replay).toMatchObject({ status: "duplicate", observationId: first.observationId });+ expect(observations(kv)).toHaveLength(1);+ });++ it("reuses the observation id when the event record was lost in a crash", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ const body = payload("lost-event-record");+ const first = await capture(body, "evc_000000000021");+ for (const [scope, entries] of kv.store) {+ if (scope.startsWith("mem:capture:events:")) entries.clear();+ }+ const resent = await capture(body, "evc_000000000021");+ expect(resent).toMatchObject({ status: "accepted", state: "completed", observationId: first.observationId });+ expect(observations(kv)).toHaveLength(1);+ expect(observations(kv)[0]!.id).toBe(first.observationId);+ });++ it("stores the observation again when the event record survived a crash but the observation did not", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ const body = payload("lost-observation");+ const first = await capture(body, "evc_000000000022");+ kv.store.get(KV.observations("ses_capture"))!.clear();+ const resent = await capture(body, "evc_000000000022");+ expect(resent).toMatchObject({ status: "accepted", state: "completed", observationId: first.observationId });+ expect(observations(kv).map((o) => o.id)).toEqual([first.observationId]);+ const again = await capture(body, "evc_000000000022");+ expect(again).toMatchObject({ status: "duplicate", observationId: first.observationId });+ expect(observations(kv)).toHaveLength(1);+ });++ it("restores a lost vector when a re-sent event finds its observation already stored", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ const search = await import("../src/functions/search.js");+ const { VectorIndex } = await import("../src/state/vector-index.js");+ const vectors = new VectorIndex();+ const embed = vi.fn(async () => new Float32Array([1, 0, 0]));+ search.setVectorIndex(vectors);+ search.setEmbeddingProvider({ name: "test", dimensions: 3, embed, embedBatch: async (t: string[]) => t.map(() => new Float32Array([1, 0, 0])) } as never);+ try {+ const body = payload("lost-vector");+ const first = await capture(body, "evc_000000000023");+ const id = first.observationId as string;+ expect(vectors.has(id)).toBe(true);+ vectors.remove(id);+ search.getSearchIndex().remove(id);+ for (const [scope, entries] of kv.store) {+ if (scope.startsWith("mem:capture:events:")) entries.clear();+ }+ const resent = await capture(body, "evc_000000000023");+ expect(resent).toMatchObject({ status: "accepted", observationId: id });+ expect(vectors.has(id)).toBe(true);+ expect(vectors.size).toBe(1);+ expect(search.getSearchIndex().has(id)).toBe(true);+ expect(embed).toHaveBeenCalledTimes(2);+ await capture(body, "evc_000000000023");+ expect(embed).toHaveBeenCalledTimes(2);+ vectors.remove(id);+ const duplicate = await capture(body, "evc_000000000023");+ expect(duplicate).toMatchObject({ status: "duplicate", observationId: id });+ expect(vectors.has(id)).toBe(true);+ expect(embed).toHaveBeenCalledTimes(3);+ expect(observations(kv)).toHaveLength(1);+ } finally {+ search.setVectorIndex(null);+ search.setEmbeddingProvider(null);+ }+ });++ it("scrubs credentials before an event waits in the inbox", async () => {+ const kv = mockKV();+ const { sdk, capture } = await boot(kv);+ sdk.fns.set("mem::observe", async () => {+ throw new Error("state write timed out");+ });+ const body = payload("scrub");+ body.data.tool_output = "cloned https://deploy:[email protected]/repo.git";+ await capture(body, "evc_000000000024");+ const stored = JSON.stringify(inbox(kv));+ expect(stored).not.toContain("hunter2secret");+ expect(stored).toContain("[REDACTED_SECRET]");+ });++ it("derives the observation id from the event key and its timestamp", async () => {+ const { observationIdFor } = await import("../src/functions/capture.js");+ const key = "cap_0123456789abcdef0123456789abcdef01234567";+ const at = "2026-10-02T06:58:06.512Z";+ expect(observationIdFor(key, at, 1)).toBe(observationIdFor(key, at, 2));+ expect(observationIdFor(key, at, 1)).toBe(`obs_${Date.parse(at).toString(36)}_0123456789ab`);+ expect(observationIdFor(key, "not a time", Date.UTC(2026, 0, 1))).toBe(`obs_${Date.UTC(2026, 0, 1).toString(36)}_0123456789ab`);+ });++ it("scopes event ids by project and session", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ await capture(payload("a", "ses_a"), "evc_000000000006");+ const other = await capture(payload("a", "ses_b"), "evc_000000000006");+ expect(other.status).toBe("accepted");+ });++ it("keeps the 5-minute content dedup for clients that send no event id", async () => {+ const kv = mockKV();+ const { capture } = await boot(kv);+ expect((await capture(payload("legacy"))).status).toBe("accepted");+ expect((await capture(payload("legacy"))).status).toBe("duplicate");+ expect(observations(kv)).toHaveLength(1);+ });++ it("retries a failed write, survives a restart and stores it once", async () => {+ const kv = mockKV();+ const first = await boot(kv);+ const observe = first.sdk.fns.get("mem::observe")!;+ first.sdk.fns.set("mem::observe", async () => {+ throw new Error("state write timed out");+ });+ const accepted = await first.capture(payload("flaky"), "evc_000000000007");+ expect(accepted).toMatchObject({ status: "accepted", state: "retrying", attempts: 1, error: "state write timed out" });+ expect(inbox(kv)).toHaveLength(1);+ expect(observations(kv)).toHaveLength(0);+ first.sdk.fns.set("mem::observe", observe);++ vi.resetModules();+ const second = await boot(kv);+ vi.useFakeTimers({ now: Date.now() + 5_000, toFake: ["Date"] });+ const swept = await second.controller.sweep();+ expect(swept).toMatchObject({ processed: 1, recovered: 1 });+ expect(observations(kv)).toHaveLength(1);+ expect(observations(kv)[0]!.id).toBe(accepted.observationId);+ expect(inbox(kv)).toHaveLength(0);+ const replay = await second.capture(payload("flaky"), "evc_000000000007");+ expect(replay).toMatchObject({ status: "duplicate", observationId: accepted.observationId });+ });++ it("waits for the backoff before retrying", async () => {+ const kv = mockKV();+ const { sdk, capture, controller } = await boot(kv);+ const observe = sdk.fns.get("mem::observe")!;+ sdk.fns.set("mem::observe", async () => {+ throw new Error("boom");+ });+ await capture(payload("backoff"), "evc_000000000008");+ sdk.fns.set("mem::observe", observe);+ expect(await controller.sweep()).toMatchObject({ processed: 0 });+ expect(observations(kv)).toHaveLength(0);+ });++ it("moves an event to dead letters after the last attempt and can retry it on request", async () => {+ const kv = mockKV();+ const { sdk, capture, controller } = await boot(kv);+ const observe = sdk.fns.get("mem::observe")!;+ sdk.fns.set("mem::observe", async () => {+ throw new Error("disk full");+ });+ await capture(payload("dead"), "evc_000000000009");+ for (let i = 1; i <= 3; i++) {+ vi.useFakeTimers({ now: Date.now() + i * 60_000, toFake: ["Date"] });+ await controller.sweep();+ }+ const [rec] = inbox(kv);+ expect(rec).toMatchObject({ status: "dead", attempts: 3, lastError: "disk full" });+ const status = await controller.status();+ expect(status.inbox).toMatchObject({ dead: 1, pending: 0, retrying: 0, lastError: "disk full" });+ expect(status.sinceStart.deadLettered).toBe(1);++ sdk.fns.set("mem::observe", observe);+ expect(await controller.sweep()).toMatchObject({ processed: 0 });+ const retried = await sdk.trigger("mem::capture-retry", { all: true });+ expect(retried).toMatchObject({ matched: 1, recovered: 1 });+ expect(observations(kv)).toHaveLength(1);+ expect(inbox(kv)).toHaveLength(0);+ });++ it("dead-letters a permanent rejection without retrying it", async () => {+ const kv = mockKV();+ const { sdk, capture } = await boot(kv);+ sdk.fns.set("mem::observe", async () => ({ success: false, error: "Session observation limit reached (1)" }));+ const result = await capture(payload("limit"), "evc_000000000010");+ expect(result).toMatchObject({ status: "rejected", retryable: false });+ expect(inbox(kv)[0]).toMatchObject({ status: "dead", attempts: 1 });+ });++ it("finishes an event that crashed before it was processed", async () => {+ const kv = mockKV();+ const { controller } = await boot(kv);+ await kv.set(KV.captureInbox, "cap_00aa", {+ key: "cap_00aa",+ eventId: "evc_000000000011",+ eventSource: "client",+ sessionId: "ses_capture",+ project: "proj",+ hookType: "post_tool_use",+ observationId: "obs_mfabcdef_00aa0000aaaa",+ status: "pending",+ attempts: 0,+ acceptedAt: new Date().toISOString(),+ updatedAt: new Date().toISOString(),+ payload: payload("stranded"),+ });+ expect(await controller.sweep()).toMatchObject({ processed: 1, recovered: 1 });+ expect(observations(kv).map((o) => o.id)).toEqual(["obs_mfabcdef_00aa0000aaaa"]);+ expect(inbox(kv)).toHaveLength(0);+ });++ it("does not write a second observation when the crash came after the write", async () => {+ const kv = mockKV();+ const { capture, controller, sdk } = await boot(kv);+ const done = await capture(payload("after-write"), "evc_000000000012");+ const [eventScope] = [...kv.store.keys()].filter((k) => k.startsWith("mem:capture:events:"));+ const completed = [...kv.store.get(eventScope!)!.values()][0] as Record<string, unknown>;+ kv.store.get(eventScope!)!.clear();+ await kv.set(KV.captureInbox, completed.key as string, {+ ...completed,+ eventSource: "client",+ hookType: "post_tool_use",+ status: "pending",+ attempts: 1,+ updatedAt: new Date().toISOString(),+ payload: payload("after-write"),+ });+ const observeCalls = vi.fn(sdk.fns.get("mem::observe")!);+ sdk.fns.set("mem::observe", observeCalls);+ await controller.sweep();+ expect(observeCalls).toHaveBeenCalledTimes(1);+ expect(await observeCalls.mock.results[0]!.value).toMatchObject({ observationId: done.observationId, existing: true });+ expect(observations(kv)).toHaveLength(1);+ expect(inbox(kv)).toHaveLength(0);+ });++ it("prunes event ids past the retention window and old dead letters", async () => {+ const kv = mockKV();+ vi.stubEnv("AGENTMEMORY_CAPTURE_DEDUP_HOURS", "1");+ const { capture, controller } = await boot(kv);+ await capture(payload("old"), "evc_000000000013");+ vi.useFakeTimers({ now: Date.now() + 2 * 3600_000, toFake: ["Date"] });+ await capture(payload("new"), "evc_000000000014");+ expect(await controller.prune()).toBe(1);+ const replayOld = await capture(payload("old"), "evc_000000000013");+ expect(replayOld.status).toBe("accepted");+ });++ it("rejects new events while the inbox is full so hooks keep them", async () => {+ vi.stubEnv("AGENTMEMORY_CAPTURE_INBOX_MAX", "1");+ const kv = mockKV();+ const { sdk, capture } = await boot(kv);+ sdk.fns.set("mem::observe", async () => {+ throw new Error("down");+ });+ await capture(payload("one"), "evc_000000000015");+ const full = await capture(payload("two"), "evc_000000000016");+ expect(full).toMatchObject({ status: "rejected", retryable: true });+ });++ it("drains the local spool at boot without storing an event twice", async () => {+ const dir = join(mkdtempSync(join(tmpdir(), "am-cap-")), "capture-spool");+ vi.stubEnv("AGENTMEMORY_CAPTURE_SPOOL_DIR", dir);+ const { appendSpool } = await import("../src/capture/spool.js");+ const kv = mockKV();+ const { capture, controller } = await boot(kv, { restPort: 4811 });+ await capture(payload("arrived"), "evc_000000000017");+ appendSpool("http://127.0.0.1:4811", "evc_000000000017", payload("arrived"), "timeout", { dir });+ appendSpool("http://127.0.0.1:4811", "evc_000000000018", payload("offline"), "unreachable", { dir });+ const results = await controller.drainLocalSpool();+ expect(results[0]).toMatchObject({ claimed: 2, delivered: 1, duplicates: 1, remaining: 0 });+ expect(observations(kv)).toHaveLength(2);+ const status = await controller.status();+ expect(status.spool[0]).toMatchObject({ records: 0 });+ expect(status.lastBootDrain).toMatchObject({ delivered: 1, duplicates: 1 });+ });+});++describe("observe response codes", () => {+ it("maps capture results to HTTP status codes", async () => {+ const { captureStatusCode, captureResponseBody } = await import("../src/triggers/api.js");+ expect(captureStatusCode({ status: "accepted", state: "completed", eventId: "e", observationId: "o", attempts: 1 })).toBe(201);+ expect(captureStatusCode({ status: "accepted", state: "retrying", eventId: "e", observationId: "o", attempts: 1, nextAttemptAt: "", error: "x" })).toBe(202);+ expect(captureStatusCode({ status: "duplicate", state: "completed", eventId: "e", observationId: "o", deduplicated: true })).toBe(200);+ expect(captureStatusCode({ status: "rejected", eventId: "e", error: "x", retryable: true })).toBe(503);+ expect(captureStatusCode({ status: "rejected", eventId: "e", error: "x", retryable: false })).toBe(422);+ expect(captureResponseBody({ status: "rejected", eventId: "e", error: "x", retryable: false })).toMatchObject({ success: false });+ });+});++describe("capture status problems", () => {+ it("reports dead letters, pending retries, spooled events and recent drops", async () => {+ const { evaluateStatus } = await import("../src/functions/status.js");+ const report = evaluateStatus({+ now: new Date(),+ version: "0",+ engineVersion: "0",+ uptimeSeconds: 1,+ stateBackend: "file",+ ports: { rest: 4811, streams: null, viewer: null },+ health: null,+ circuitBreaker: null,+ functionMetrics: [],+ provider: "noop",+ embeddingProvider: "none",+ flags: [],+ index: {+ bm25Documents: 0,+ vectorDocuments: null,+ observationsIndexed: 0,+ missingObservations: 0,+ sessions: 0,+ bm25Incomplete: false,+ pendingVectorBackfill: 0,+ },+ graph: null,+ graphExtractionEnabled: false,+ auditLegacy: null,+ capture: {+ policy: { maxAttempts: 5, retryIntervalMs: 10_000, dedupRetentionHours: 168, inboxMax: 10_000, deadMax: 1000, eventsMax: 100_000 },+ inbox: { pending: 0, retrying: 2, dead: 1, oldestAcceptedAt: null, nextAttemptAt: null, lastError: "disk full" },+ sinceStart: { accepted: 3, completed: 0, duplicates: 0, retried: 2, recovered: 0, deadLettered: 1, rejected: 0, pruned: 0 },+ lastSweepAt: null,+ lastPruneAt: null,+ lastBootDrain: null,+ spool: [+ {+ enabled: true,+ path: "/x/local-4811.jsonl",+ records: 4,+ bytes: 2000,+ oldestAt: null,+ maxBytes: 5 * 1024 * 1024,+ maxAgeHours: 168,+ stats: { spooled: 4, dropped: 2, droppedBytes: 900, expired: 0, delivered: 0, duplicates: 0, rejected: 0, lastDropAt: new Date().toISOString(), lastDropReason: "full" },+ },+ ],+ },+ });+ const codes = report.problems.map((p) => p.code);+ expect(codes).toEqual(expect.arrayContaining(["capture-dead-letters", "capture-retrying", "capture-spool-waiting", "capture-spool-dropped"]));+ expect(report.capture?.inbox?.dead).toBe(1);+ expect(report.problems.find((p) => p.code === "capture-dead-letters")!.fix).toContain("http://localhost:4811/agentmemory/capture/retry");+ });+});++describe("acceptance durability across a restart", () => {+ beforeEach(() => {+ vi.resetModules();+ });+ afterEach(() => {+ vi.unstubAllEnvs();+ });++ function sentRecords(dir: string): Array<{ eventId: string; bootId?: string }> {+ if (!existsSync(dir)) return [];+ return readdirSync(dir)+ .filter((f) => f.includes(".sent-"))+ .flatMap((f) => readFileSync(join(dir, f), "utf-8").split("\n").filter(Boolean).map((l) => JSON.parse(l)));+ }++ it("tells clients its boot id and save window on every accepted response", async () => {+ const kv = mockKV();+ const { controller } = await boot(kv, { restPort: 4812, durableAfterMs: 3500 });+ const mark = controller.durability();+ expect(mark.bootId).toMatch(/^[0-9a-f]{24}$/);+ expect(mark.durableAfterMs).toBe(3500);+ const { captureResponseBody } = await import("../src/triggers/api.js");+ const body = captureResponseBody({ status: "accepted", state: "completed", eventId: "e", observationId: "o", attempts: 1 }, mark);+ expect(body).toMatchObject({ bootId: mark.bootId, durableAfterMs: 3500 });+ expect(typeof body["acceptedAt"]).toBe("string");+ expect(captureResponseBody({ status: "rejected", eventId: "e", error: "x", retryable: true }, mark)["bootId"]).toBeUndefined();+ const second = await boot(mockKV(), { restPort: 4812, durableAfterMs: 3500 });+ expect(second.controller.durability().bootId).not.toBe(mark.bootId);+ });++ it("uses the engine save interval plus a margin for the file store and a short fixed window for redis", async () => {+ const { captureDurableAfterMs } = await import("../src/cli/engine-config.js");+ expect(captureDurableAfterMs("file", ["save_interval_ms: 2000"])).toBe(3500);+ expect(captureDurableAfterMs("file", [])).toBe(6500);+ expect(captureDurableAfterMs("redis", ["save_interval_ms: 2000"])).toBe(1500);+ });++ it("re-sends observations a client kept from the previous boot and keeps them under the new boot", async () => {+ const dir = join(mkdtempSync(join(tmpdir(), "am-cap-")), "capture-spool");+ vi.stubEnv("AGENTMEMORY_CAPTURE_SPOOL_DIR", dir);+ const { retainSent } = await import("../src/capture/spool.js");+ retainSent("http://127.0.0.1:4813", "evc_lostinflush01", payload("lost-in-flush"), { bootId: "cccccccccccccccccccccccc", durableAfterMs: 3500 }, { dir });+ const kv = mockKV();+ const { controller } = await boot(kv, { restPort: 4813, durableAfterMs: 3500 });+ const results = await controller.drainLocalSpool();+ expect(results[0]).toMatchObject({ claimed: 1, delivered: 1, remaining: 0 });+ expect(observations(kv)).toHaveLength(1);+ const kept = sentRecords(dir);+ expect(kept.map((r) => r.eventId)).toEqual(["evc_lostinflush01"]);+ expect(kept[0]!.bootId).toBe(controller.durability().bootId);+ expect((await controller.status()).spool[0]).toMatchObject({ records: 0, retained: 1 });+ });+});diff --git a/test/capture-event-id.test.ts b/test/capture-event-id.test.tsnew file mode 100644index 00000000..e537f458--- /dev/null+++ b/test/capture-event-id.test.ts@@ -0,0 +1,52 @@+import { describe, it, expect } from "vitest";+import { deriveEventId, isValidEventId, stableStringify } from "../src/capture/event-id.js";++describe("capture event id derivation", () => {+ const content = { tool_name: "Bash", tool_input: { command: "npm test" }, tool_output: "12 passed" };++ it("uses the host id when the payload carries one, whatever the content", () => {+ const a = deriveEventId("post_tool_use", "s1", { tool_use_id: "toolu_01" }, content);+ const b = deriveEventId("post_tool_use", "s1", { tool_use_id: "toolu_01" }, { ...content, tool_output: "changed" });+ expect(a).toBe(b);+ expect(a.startsWith("evh_")).toBe(true);+ expect(isValidEventId(a)).toBe(true);+ });++ it("keeps two host events with identical content apart", () => {+ const a = deriveEventId("post_tool_use", "s1", { tool_use_id: "toolu_01" }, content);+ const b = deriveEventId("post_tool_use", "s1", { tool_use_id: "toolu_02" }, content);+ expect(a).not.toBe(b);+ });++ it("hashes session, hook type and content when there is no host id", () => {+ const a = deriveEventId("post_tool_use", "s1", {}, content);+ expect(a.startsWith("evc_")).toBe(true);+ expect(deriveEventId("post_tool_use", "s1", {}, { ...content })).toBe(a);+ expect(deriveEventId("post_tool_use", "s2", {}, content)).not.toBe(a);+ expect(deriveEventId("post_tool_failure", "s1", {}, content)).not.toBe(a);+ expect(deriveEventId("post_tool_use", "s1", {}, { ...content, tool_output: "13 passed" })).not.toBe(a);+ });++ it("separates identical content by the host timestamp when present", () => {+ const a = deriveEventId("prompt_submit", "s1", { timestamp: "2026-10-01T10:00:00Z" }, { prompt: "continue" });+ const b = deriveEventId("prompt_submit", "s1", { timestamp: "2026-10-01T10:05:00Z" }, { prompt: "continue" });+ expect(a).not.toBe(b);+ });++ it("does not depend on object key order", () => {+ expect(stableStringify({ b: 1, a: { d: [1, { y: 2, x: 1 }], c: null } })).toBe(+ stableStringify({ a: { c: null, d: [1, { x: 1, y: 2 }] }, b: 1 }),+ );+ expect(deriveEventId("post_tool_use", "s1", {}, { tool_input: { a: 1, b: 2 } })).toBe(+ deriveEventId("post_tool_use", "s1", {}, { tool_input: { b: 2, a: 1 } }),+ );+ });++ it("validates event ids", () => {+ expect(isValidEventId("evh_0123456789abcdef")).toBe(true);+ expect(isValidEventId("short")).toBe(false);+ expect(isValidEventId("has space in it")).toBe(false);+ expect(isValidEventId(42)).toBe(false);+ expect(isValidEventId("x".repeat(129))).toBe(false);+ });+});diff --git a/test/capture-hook.test.ts b/test/capture-hook.test.tsnew file mode 100644index 00000000..2e79517c--- /dev/null+++ b/test/capture-hook.test.ts@@ -0,0 +1,274 @@+import { describe, it, expect, afterEach } from "vitest";+import { spawn } from "node:child_process";+import { createServer, type Server } from "node:http";+import { existsSync, mkdtempSync, readFileSync, readdirSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";++const HOOKS_DIR = join(import.meta.dirname, "..", "plugin", "scripts");++function runHook(+ scriptName: string,+ stdin: string,+ env: Record<string, string>,+): Promise<{ exitCode: number | null; ms: number }> {+ const started = Date.now();+ return new Promise((resolve, reject) => {+ const child = spawn(process.execPath, [join(HOOKS_DIR, scriptName)], {+ env: { PATH: process.env["PATH"] ?? "", HOME: env["HOME"] ?? tmpdir(), ...env },+ stdio: ["pipe", "ignore", "ignore"],+ });+ child.on("error", reject);+ child.on("close", (exitCode) => resolve({ exitCode, ms: Date.now() - started }));+ child.stdin.write(stdin);+ child.stdin.end();+ });+}++function toolPayload(marker: string, extra: Record<string, unknown> = {}): string {+ return JSON.stringify({+ session_id: "ses_hook_capture",+ cwd: "/work/hook-proj",+ hook_event_name: "PostToolUse",+ tool_name: "Bash",+ tool_input: { command: `echo ${marker}` },+ tool_response: { stdout: marker },+ ...extra,+ });+}++function spoolLines(dir: string, port: number): Array<{ eventId: string; body: Record<string, unknown> }> {+ const file = join(dir, `local-${port}.jsonl`);+ if (!existsSync(file)) return [];+ return readFileSync(file, "utf-8")+ .split("\n")+ .filter(Boolean)+ .map((l) => JSON.parse(l));+}++async function waitFor(check: () => boolean, ms: number): Promise<boolean> {+ const deadline = Date.now() + ms;+ while (Date.now() < deadline) {+ if (check()) return true;+ await new Promise((r) => setTimeout(r, 50));+ }+ return check();+}++describe("hooks capture when the server is unavailable", () => {+ let server: Server | undefined;++ afterEach(async () => {+ if (server) await new Promise<void>((resolve) => server!.close(() => resolve()));+ server = undefined;+ });++ async function startServer(status: number, received: Array<Record<string, unknown>>): Promise<number> {+ server = createServer((req, res) => {+ let body = "";+ req.on("data", (c) => (body += c));+ req.on("end", () => {+ try {+ received.push(JSON.parse(body));+ } catch {}+ res.writeHead(status, { "content-type": "application/json" });+ res.end(JSON.stringify({ status: status === 201 ? "accepted" : "x" }));+ });+ });+ await new Promise<void>((resolve) => server!.listen(0, "127.0.0.1", () => resolve()));+ const addr = server.address();+ return typeof addr === "object" && addr ? addr.port : 0;+ }++ async function closedPort(): Promise<number> {+ const probe = createServer();+ await new Promise<void>((resolve) => probe.listen(0, "127.0.0.1", () => resolve()));+ const addr = probe.address();+ const port = typeof addr === "object" && addr ? addr.port : 0;+ await new Promise<void>((resolve) => probe.close(() => resolve()));+ return port;+ }++ it("exits 0 quickly and spools the observation with its event id", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-spool-"));+ const port = await closedPort();+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ const result = await runHook("post-tool-use.mjs", toolPayload("offlinecerulean947", { tool_use_id: "toolu_1" }), env);+ expect(result.exitCode).toBe(0);+ expect(result.ms).toBeLessThan(3000);+ const [rec] = spoolLines(dir, port);+ expect(rec!.eventId).toMatch(/^evh_[0-9a-f]{32}$/);+ expect(JSON.stringify(rec!.body)).toContain("offlinecerulean947");+ const again = await runHook("post-tool-use.mjs", toolPayload("offlinecerulean947", { tool_use_id: "toolu_1" }), env);+ expect(again.exitCode).toBe(0);+ const recs = spoolLines(dir, port);+ expect(recs).toHaveLength(2);+ expect(recs[1]!.eventId).toBe(recs[0]!.eventId);+ });++ it("gives two screenshots with the same tool input different event ids", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-spool-"));+ const port = await closedPort();+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ await runHook("post-tool-use.mjs", toolPayload("shot", { tool_response: "iVBORw0KGgoAAAA1" }), env);+ await runHook("post-tool-use.mjs", toolPayload("shot", { tool_response: "iVBORw0KGgoAAAA2" }), env);+ const recs = spoolLines(dir, port);+ expect(recs).toHaveLength(2);+ expect(recs[0]!.eventId).not.toBe(recs[1]!.eventId);+ });++ it("spools on a 5xx answer but not on a 4xx rejection", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-spool-"));+ const received: Array<Record<string, unknown>> = [];+ const port = await startServer(500, received);+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ expect((await runHook("post-tool-use.mjs", toolPayload("five-hundred"), env)).exitCode).toBe(0);+ expect(spoolLines(dir, port)).toHaveLength(1);+ expect(received[0]!.eventId).toBe(spoolLines(dir, port)[0]!.eventId);+ await new Promise<void>((resolve) => server!.close(() => resolve()));+ server = undefined;++ const dir2 = mkdtempSync(join(tmpdir(), "am-hook-spool-"));+ const port2 = await startServer(400, []);+ const env2 = { AGENTMEMORY_URL: `http://127.0.0.1:${port2}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir2 };+ expect((await runHook("post-tool-use.mjs", toolPayload("four-hundred"), env2)).exitCode).toBe(0);+ expect(spoolLines(dir2, port2)).toHaveLength(0);+ });++ it("does not spool when the spool is turned off", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-spool-"));+ const port = await closedPort();+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir, AGENTMEMORY_CAPTURE_SPOOL: "false" };+ expect((await runHook("post-tool-use.mjs", toolPayload("disabled"), env)).exitCode).toBe(0);+ expect(spoolLines(dir, port)).toHaveLength(0);+ });++ it("keeps every record when many hooks spool at once", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-spool-"));+ const port = await closedPort();+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ const results = await Promise.all(+ Array.from({ length: 12 }, (_, i) => runHook("post-tool-use.mjs", toolPayload(`parallel-${i}`), env)),+ );+ expect(results.every((r) => r.exitCode === 0)).toBe(true);+ const recs = spoolLines(dir, port);+ expect(recs).toHaveLength(12);+ expect(new Set(recs.map((r) => r.eventId)).size).toBe(12);+ });++ it("sends the spool in the background once the server answers again", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-spool-"));+ const received: Array<Record<string, unknown>> = [];+ const port = await startServer(201, received);+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ await new Promise<void>((resolve) => server!.close(() => resolve()));+ server = undefined;+ expect((await runHook("prompt-submit.mjs", JSON.stringify({ session_id: "ses_hook_capture", cwd: "/work/hook-proj", prompt: "spooled prompt" }), env)).exitCode).toBe(0);+ const spooled = spoolLines(dir, port);+ expect(spooled).toHaveLength(1);++ server = createServer((req, res) => {+ let body = "";+ req.on("data", (c) => (body += c));+ req.on("end", () => {+ received.push(JSON.parse(body));+ res.writeHead(201, { "content-type": "application/json" });+ res.end("{}");+ });+ });+ await new Promise<void>((resolve) => server!.listen(port, "127.0.0.1", () => resolve()));+ const live = await runHook("post-tool-use.mjs", toolPayload("back-online"), env);+ expect(live.exitCode).toBe(0);+ expect(live.ms).toBeLessThan(3000);+ expect(await waitFor(() => received.some((r) => r.eventId === spooled[0]!.eventId), 8000)).toBe(true);+ expect(await waitFor(() => spoolLines(dir, port).length === 0, 3000)).toBe(true);+ expect(received.filter((r) => r.eventId === spooled[0]!.eventId)).toHaveLength(1);+ });+});++describe("hooks keep accepted observations until the server saved them", () => {+ let server: Server | undefined;++ afterEach(async () => {+ if (server) await new Promise<void>((resolve) => server!.close(() => resolve()));+ server = undefined;+ });++ async function markServer(received: Array<Record<string, unknown>>, mark: () => Record<string, unknown>): Promise<number> {+ server = createServer((req, res) => {+ let body = "";+ req.on("data", (c) => (body += c));+ req.on("end", () => {+ try {+ received.push(JSON.parse(body));+ } catch {}+ res.writeHead(201, { "content-type": "application/json" });+ res.end(JSON.stringify({ status: "accepted", state: "completed", ...mark() }));+ });+ });+ await new Promise<void>((resolve) => server!.listen(0, "127.0.0.1", () => resolve()));+ const addr = server.address();+ return typeof addr === "object" && addr ? addr.port : 0;+ }++ function sentRecords(dir: string): Array<{ eventId: string; bootId?: string }> {+ if (!existsSync(dir)) return [];+ return readdirSync(dir)+ .filter((f) => f.includes(".sent-"))+ .flatMap((f) => readFileSync(join(dir, f), "utf-8").split("\n").filter(Boolean).map((l) => JSON.parse(l)));+ }++ it("releases a kept observation on a later call once the window passed, without sending it again", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-keep-"));+ const received: Array<Record<string, unknown>> = [];+ const port = await markServer(received, () => ({ bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 0 }));+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ expect((await runHook("post-tool-use.mjs", toolPayload("kept-one", { tool_use_id: "toolu_k1" }), env)).exitCode).toBe(0);+ const first = sentRecords(dir);+ expect(first).toHaveLength(1);+ expect(first[0]!.eventId).toBe(received[0]!.eventId);+ expect(spoolLines(dir, port)).toHaveLength(0);+ await new Promise((r) => setTimeout(r, 1100));+ expect((await runHook("post-tool-use.mjs", toolPayload("kept-two", { tool_use_id: "toolu_k2" }), env)).exitCode).toBe(0);+ const second = sentRecords(dir);+ expect(second.map((r) => r.eventId)).toEqual([received[1]!.eventId]);+ await new Promise((r) => setTimeout(r, 500));+ expect(received).toHaveLength(2);+ });++ it("re-sends a kept observation with the same event id after the server restarted", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-keep-"));+ const received: Array<Record<string, unknown>> = [];+ let bootId = "aaaaaaaaaaaaaaaaaaaaaaaa";+ const port = await markServer(received, () => ({ bootId, durableAfterMs: 60_000 }));+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ await runHook("post-tool-use.mjs", toolPayload("before-restart", { tool_use_id: "toolu_r1" }), env);+ const lost = received[0]!.eventId;+ bootId = "bbbbbbbbbbbbbbbbbbbbbbbb";+ expect((await runHook("post-tool-use.mjs", toolPayload("after-restart", { tool_use_id: "toolu_r2" }), env)).exitCode).toBe(0);+ expect(await waitFor(() => received.filter((r) => r.eventId === lost).length === 2, 8000)).toBe(true);+ expect(await waitFor(() => sentRecords(dir).every((r) => r.bootId === bootId) && sentRecords(dir).length === 2, 3000)).toBe(true);+ expect(spoolLines(dir, port)).toHaveLength(0);+ });++ it("keeps nothing when the server sends no boot id", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-keep-"));+ const received: Array<Record<string, unknown>> = [];+ const port = await markServer(received, () => ({}));+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir };+ expect((await runHook("post-tool-use.mjs", toolPayload("old-server"), env)).exitCode).toBe(0);+ expect(received).toHaveLength(1);+ expect(sentRecords(dir)).toHaveLength(0);+ expect(spoolLines(dir, port)).toHaveLength(0);+ });++ it("keeps nothing when the spool is turned off", async () => {+ const dir = mkdtempSync(join(tmpdir(), "am-hook-keep-"));+ const received: Array<Record<string, unknown>> = [];+ const port = await markServer(received, () => ({ bootId: "aaaaaaaaaaaaaaaaaaaaaaaa", durableAfterMs: 60_000 }));+ const env = { AGENTMEMORY_URL: `http://127.0.0.1:${port}`, AGENTMEMORY_CAPTURE_SPOOL_DIR: dir, AGENTMEMORY_CAPTURE_SPOOL: "false" };+ expect((await runHook("post-tool-use.mjs", toolPayload("spool-off"), env)).exitCode).toBe(0);+ expect(received).toHaveLength(1);+ expect(existsSync(dir) ? readdirSync(dir) : []).toHaveLength(0);+ });+});diff --git a/test/capture-spool.test.ts b/test/capture-spool.test.tsnew file mode 100644index 00000000..c42178a1--- /dev/null+++ b/test/capture-spool.test.ts@@ -0,0 +1,268 @@+import { describe, it, expect, beforeEach } from "vitest";+import { mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync, existsSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import {+ appendSpool,+ drainSpool,+ parseSentMark,+ reconcileSent,+ retainSent,+ spoolPaths,+ spoolPolicy,+ spoolSummary,+ spoolTargetName,+ type SpoolPolicy,+ type SpoolRecord,+} from "../src/capture/spool.js";++const URL = "http://localhost:4811";++function body(marker: string, extra: Record<string, unknown> = {}) {+ return {+ hookType: "post_tool_use",+ sessionId: "ses_spool",+ project: "proj",+ cwd: "/work/proj",+ timestamp: new Date().toISOString(),+ data: { tool_name: "Bash", tool_input: { command: `echo ${marker}` }, tool_output: marker, ...extra },+ };+}++function lines(path: string): SpoolRecord[] {+ if (!existsSync(path)) return [];+ return readFileSync(path, "utf-8")+ .split("\n")+ .filter(Boolean)+ .map((l) => JSON.parse(l) as SpoolRecord);+}++describe("capture spool", () => {+ let dir: string;+ const policy: SpoolPolicy = { enabled: true, maxBytes: 64 * 1024, maxAgeMs: 3600_000, maxRecordBytes: 16 * 1024 };++ beforeEach(() => {+ dir = join(mkdtempSync(join(tmpdir(), "am-spool-")), "capture-spool");+ });++ it("names the spool by host and port and treats loopback names alike", () => {+ expect(spoolTargetName("http://localhost:3111")).toBe("local-3111");+ expect(spoolTargetName("http://127.0.0.1:3111/")).toBe("local-3111");+ expect(spoolTargetName("https://mem.example.com")).toBe("mem.example.com-443");+ });++ it("appends records to a private file in a private folder", () => {+ expect(appendSpool(URL, "evc_aaaaaaaaaaaa", body("m1"), "unreachable", { dir, policy }).spooled).toBe(true);+ const paths = spoolPaths(URL, dir);+ expect(statSync(paths.file).mode & 0o777).toBe(0o600);+ expect(statSync(dir).mode & 0o777).toBe(0o700);+ const [rec] = lines(paths.file);+ expect(rec!.eventId).toBe("evc_aaaaaaaaaaaa");+ expect(rec!.reason).toBe("unreachable");+ expect((rec!.body.data as Record<string, unknown>).tool_output).toBe("m1");+ });++ it("redacts secrets before they reach disk", () => {+ const secret = "sk-ant-" + "a".repeat(30);+ appendSpool(URL, "evc_bbbbbbbbbbbb", body(`token ${secret}`), "timeout", { dir, policy });+ const text = readFileSync(spoolPaths(URL, dir).file, "utf-8");+ expect(text).not.toContain(secret);+ expect(text).toContain("[REDACTED_SECRET]");+ });++ it("drops image data before dropping an oversized record, and counts drops", () => {+ const image = "data:image/png;base64," + "A".repeat(20 * 1024);+ expect(appendSpool(URL, "evc_cccccccccccc", body("img", { image_data: image }), "timeout", { dir, policy }).spooled).toBe(true);+ const [rec] = lines(spoolPaths(URL, dir).file);+ expect((rec!.body.data as Record<string, unknown>).image_data).toBe("[image dropped from capture spool]");+ const huge = appendSpool(URL, "evc_dddddddddddd", body("x".repeat(20 * 1024)), "timeout", { dir, policy });+ expect(huge).toEqual({ spooled: false, dropped: "too-large" });+ expect(spoolSummary(URL, { dir, policy }).stats.dropped).toBe(1);+ });++ it("stops at the byte limit and reports the overflow", () => {+ let spooled = 0;+ let dropped = 0;+ for (let i = 0; i < 400; i++) {+ const r = appendSpool(URL, `evc_${String(i).padStart(12, "0")}`, body(`m${i}`), "unreachable", { dir, policy });+ if (r.spooled) spooled++;+ else dropped++;+ }+ const summary = spoolSummary(URL, { dir, policy });+ expect(summary.bytes).toBeLessThanOrEqual(policy.maxBytes);+ expect(summary.records).toBe(spooled);+ expect(dropped).toBeGreaterThan(0);+ expect(summary.stats.dropped).toBe(dropped);+ expect(summary.stats.lastDropReason).toBe("full");+ });++ it("does nothing when the spool is disabled", () => {+ const r = appendSpool(URL, "evc_eeeeeeeeeeee", body("off"), "unreachable", { dir, policy: { ...policy, enabled: false } });+ expect(r).toEqual({ spooled: false, dropped: "disabled" });+ expect(existsSync(dir)).toBe(false);+ });++ it("reads its limits from the environment", () => {+ const p = spoolPolicy({ AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES: "1048576", AGENTMEMORY_CAPTURE_SPOOL_MAX_AGE_HOURS: "2" });+ expect(p).toMatchObject({ enabled: true, maxBytes: 1048576, maxAgeMs: 2 * 3600_000 });+ expect(spoolPolicy({ AGENTMEMORY_CAPTURE_SPOOL: "false" }).enabled).toBe(false);+ expect(spoolPolicy({ AGENTMEMORY_CAPTURE_SPOOL_MAX_BYTES: "10" }).maxBytes).toBe(64 * 1024);+ });++ it("drains in order, removes delivered records and keeps the rest after a failure", async () => {+ for (let i = 0; i < 5; i++) appendSpool(URL, `evc_${String(i).padStart(12, "0")}`, body(`m${i}`), "unreachable", { dir, policy });+ const sent: string[] = [];+ const result = await drainSpool(+ URL,+ async (rec) => {+ sent.push(rec.eventId);+ if (sent.length === 3) return "retry";+ return sent.length === 2 ? "duplicate" : "delivered";+ },+ { dir, policy },+ );+ expect(result).toMatchObject({ claimed: 5, delivered: 1, duplicates: 1, remaining: 3 });+ const left = lines(spoolPaths(URL, dir).file);+ expect(left.map((r) => r.eventId)).toEqual(["evc_000000000002", "evc_000000000003", "evc_000000000004"]);+ expect(left[0]!.attempts).toBe(1);+ const second = await drainSpool(URL, async () => "delivered", { dir, policy });+ expect(second).toMatchObject({ claimed: 3, delivered: 3, remaining: 0 });+ expect(spoolSummary(URL, { dir, policy })).toMatchObject({ records: 0 });+ expect(spoolSummary(URL, { dir, policy }).stats.lastDrainDelivered).toBe(3);+ });++ it("sends a spooled event id once even if it was spooled twice", async () => {+ appendSpool(URL, "evc_ffffffffffff", body("dup"), "timeout", { dir, policy });+ appendSpool(URL, "evc_ffffffffffff", body("dup"), "timeout", { dir, policy });+ let calls = 0;+ const result = await drainSpool(URL, async () => (calls++, "delivered"), { dir, policy });+ expect(calls).toBe(1);+ expect(result).toMatchObject({ delivered: 1, duplicates: 1 });+ });++ it("drops records older than the age limit instead of sending them", async () => {+ appendSpool(URL, "evc_gggggggggggg", body("old"), "timeout", { dir, policy });+ const paths = spoolPaths(URL, dir);+ const [rec] = lines(paths.file);+ writeFileSync(paths.file, JSON.stringify({ ...rec, spooledAt: new Date(Date.now() - 2 * 3600_000).toISOString() }) + "\n");+ let calls = 0;+ const result = await drainSpool(URL, async () => (calls++, "delivered"), { dir, policy });+ expect(calls).toBe(0);+ expect(result.expired).toBe(1);+ });++ it("keeps records appended while a drain is running", async () => {+ appendSpool(URL, "evc_hhhhhhhhhhh1", body("first"), "timeout", { dir, policy });+ const result = await drainSpool(+ URL,+ async () => {+ appendSpool(URL, "evc_hhhhhhhhhhh2", body("during"), "timeout", { dir, policy });+ return "retry";+ },+ { dir, policy },+ );+ expect(result.remaining).toBe(1);+ expect(lines(spoolPaths(URL, dir).file).map((r) => r.eventId).sort()).toEqual(["evc_hhhhhhhhhhh1", "evc_hhhhhhhhhhh2"]);+ });++ it("refuses a second concurrent drain", async () => {+ appendSpool(URL, "evc_iiiiiiiiiiii", body("one"), "timeout", { dir, policy });+ let release: () => void = () => {};+ const gate = new Promise<void>((r) => (release = r));+ const first = drainSpool(URL, async () => (await gate, "delivered"), { dir, policy });+ await new Promise((r) => setTimeout(r, 20));+ const second = await drainSpool(URL, async () => "delivered", { dir, policy });+ expect(second.skipped).toBe("locked");+ release();+ expect((await first).delivered).toBe(1);+ });++ it("recovers records from a drain that crashed mid-way", async () => {+ appendSpool(URL, "evc_jjjjjjjjjjjj", body("orphan"), "timeout", { dir, policy });+ const paths = spoolPaths(URL, dir);+ const orphan = join(dir, `${paths.name}.draining-99999-1.jsonl`);+ writeFileSync(orphan, readFileSync(paths.file, "utf-8"));+ writeFileSync(paths.file, "");+ const old = new Date(Date.now() - 10 * 60_000);+ const { utimesSync } = await import("node:fs");+ utimesSync(orphan, old, old);+ const seen: string[] = [];+ await drainSpool(URL, async (rec) => (seen.push(rec.eventId), "delivered"), { dir, policy });+ expect(seen).toEqual(["evc_jjjjjjjjjjjj"]);+ expect(existsSync(orphan)).toBe(false);+ });+});++describe("capture spool retention until durable", () => {+ let dir: string;+ const policy: SpoolPolicy = { enabled: true, maxBytes: 64 * 1024, maxAgeMs: 3600_000, maxRecordBytes: 16 * 1024 };+ const BOOT_A = "aaaaaaaaaaaaaaaaaaaaaaaa";+ const BOOT_B = "bbbbbbbbbbbbbbbbbbbbbbbb";++ beforeEach(() => {+ dir = join(mkdtempSync(join(tmpdir(), "am-spool-")), "capture-spool");+ });++ function sentFiles(): string[] {+ if (!existsSync(dir)) return [];+ return readdirSync(dir).filter((f) => f.includes(".sent-"));+ }++ it("reads the durability mark from a response and ignores responses without one", () => {+ expect(parseSentMark({ status: "accepted", bootId: BOOT_A, durableAfterMs: 3500 })).toEqual({ bootId: BOOT_A, durableAfterMs: 3500 });+ expect(parseSentMark({ status: "accepted" })).toBeNull();+ expect(parseSentMark({ bootId: "../etc", durableAfterMs: 1 })).toBeNull();+ expect(parseSentMark({ bootId: BOOT_A, durableAfterMs: -1 })).toBeNull();+ expect(parseSentMark({ bootId: BOOT_A, durableAfterMs: 1e12 })!.durableAfterMs).toBe(600_000);+ expect(parseSentMark(null)).toBeNull();+ });++ it("keeps a sent record until the window passes, then releases it without sending", async () => {+ const now = 1_700_000_000_000;+ expect(retainSent(URL, "evc_keep00000001", body("kept"), { bootId: BOOT_A, durableAfterMs: 3500 }, { dir, policy, now })).toBe(true);+ expect(sentFiles()).toHaveLength(1);+ expect(spoolSummary(URL, { dir, policy })).toMatchObject({ records: 0, retained: 1 });+ expect(reconcileSent(URL, BOOT_A, { dir, policy, now: now + 1000 })).toEqual({ released: 0, requeued: 0 });+ expect(sentFiles()).toHaveLength(1);+ expect(reconcileSent(URL, BOOT_A, { dir, policy, now: now + 5000 })).toEqual({ released: 1, requeued: 0 });+ expect(sentFiles()).toHaveLength(0);+ let calls = 0;+ await drainSpool(URL, async () => (calls++, "delivered"), { dir, policy });+ expect(calls).toBe(0);+ });++ it("re-sends a kept record with the same event id once the server restarted", async () => {+ const now = Date.now();+ retainSent(URL, "evc_restart00001", body("before-restart"), { bootId: BOOT_A, durableAfterMs: 60_000 }, { dir, policy, now });+ expect(reconcileSent(URL, BOOT_B, { dir, policy, now })).toEqual({ released: 0, requeued: 1 });+ expect(sentFiles()).toHaveLength(0);+ const sent: SpoolRecord[] = [];+ const result = await drainSpool(URL, async (rec) => (sent.push(rec), "duplicate"), { dir, policy });+ expect(result).toMatchObject({ claimed: 1, duplicates: 1, remaining: 0 });+ expect(sent.map((r) => r.eventId)).toEqual(["evc_restart00001"]);+ expect((sent[0]!.body.data as Record<string, unknown>).tool_output).toBe("before-restart");+ });++ it("does nothing when the spool is disabled", () => {+ const off = { ...policy, enabled: false };+ expect(retainSent(URL, "evc_off000000001", body("off"), { bootId: BOOT_A, durableAfterMs: 0 }, { dir, policy: off })).toBe(false);+ expect(reconcileSent(URL, BOOT_B, { dir, policy: off })).toEqual({ released: 0, requeued: 0 });+ expect(existsSync(dir)).toBe(false);+ });++ it("keeps kept records inside the spool byte limit and gives way to unsent records", () => {+ let retained = 0;+ for (let i = 0; i < 400; i++) {+ if (retainSent(URL, `evc_r${String(i).padStart(11, "0")}`, body(`r${i}`), { bootId: BOOT_A, durableAfterMs: 60_000 }, { dir, policy })) retained++;+ }+ expect(retained).toBeGreaterThan(0);+ expect(retained).toBeLessThan(400);+ const before = spoolSummary(URL, { dir, policy });+ expect(before.retainedBytes).toBeLessThanOrEqual(policy.maxBytes);+ expect(appendSpool(URL, "evc_unsent000001", body("unsent"), "unreachable", { dir, policy }).spooled).toBe(true);+ const after = spoolSummary(URL, { dir, policy });+ expect(after.records).toBe(1);+ expect(after.retained).toBeLessThan(before.retained);+ expect(after.bytes + after.retainedBytes).toBeLessThanOrEqual(policy.maxBytes);+ });+});diff --git a/test/cli-engine-startup.test.ts b/test/cli-engine-startup.test.tsindex 3a2fef81..cdade606 100644--- a/test/cli-engine-startup.test.ts+++ b/test/cli-engine-startup.test.ts@@ -105,7 +105,7 @@ describe("fresh native engine startup", () => {});it("keeps AGENTMEMORY_URL client-only and lets local port flags win", () => {- expect(source).toContain('const URL_CLIENT_COMMANDS = new Set(["status", "doctor", "mcp"])');+ expect(source).toContain('const URL_CLIENT_COMMANDS = new Set(["status", "doctor", "mcp", "capture"])');expect(source).toContain("hasExplicitLocalPortOverride");expect(source).toContain("shouldUseAgentmemoryUrl()");expect(source).toContain('process.env["III_REST_PORT"] = String(base)');diff --git a/test/hook-exit-timing.test.ts b/test/hook-exit-timing.test.tsindex dd20cba9..b0a05408 100644--- a/test/hook-exit-timing.test.ts+++ b/test/hook-exit-timing.test.ts@@ -39,10 +39,10 @@ describe("hook exit timing", () => {"subagent-start.ts","subagent-stop.ts","task-completed.ts",- ])("%s does not exit before its observe request can finish", (name) => {+ ])("%s does not exit before its observe request and spool write can finish", (name) => {const hook = hooks.find((h) => h.name === name)!;const exitCap = hook.source.match(/setTimeout\(\(\) => process\.exit\(0\), (\w+)\)\.unref\(\)/)![1];- const abort = hook.source.match(/AbortSignal\.timeout\((\w+)\)/)![1];- expect(numericValue(hook.source, exitCap)).toBeGreaterThanOrEqual(numericValue(hook.source, abort));+ expect(hook.source).toMatch(/captureObservation\(/);+ expect(numericValue(hook.source, exitCap)).toBeGreaterThan(numericValue(hook.source, "OBSERVE_TIMEOUT_MS"));});});diff --git a/test/hook-shared-capture.test.ts b/test/hook-shared-capture.test.tsnew file mode 100644index 00000000..8e915eaf--- /dev/null+++ b/test/hook-shared-capture.test.ts@@ -0,0 +1,37 @@+import { describe, it, expect } from "vitest";+import { existsSync, readFileSync } from "node:fs";+import { join } from "node:path";++const scriptsDir = join(__dirname, "..", "plugin", "scripts");+const captureHooks = [+ "notification",+ "post-tool-failure",+ "post-tool-use",+ "prompt-submit",+ "session-end",+ "subagent-start",+ "subagent-stop",+ "task-completed",+];++describe("shared capture module in plugin/scripts", () => {+ it("ships the shared module under a stable name", () => {+ const shared = readFileSync(join(scriptsDir, "_capture.mjs"), "utf-8");+ expect(shared).toContain("//#region src/capture/spool.ts");+ expect(shared).toMatch(/export \{[^}]*captureObservation[^}]*\}/);+ });++ it.each(captureHooks)("%s imports the shared module instead of inlining it", (hook) => {+ const source = readFileSync(join(scriptsDir, `${hook}.mjs`), "utf-8");+ expect(source).toMatch(/from "\.\/_capture\.mjs";/);+ expect(source).not.toContain("//#region src/capture/spool.ts");+ expect(source).not.toContain("//#region src/hooks/_capture.ts");+ });++ it("leaves hooks that do not capture self-contained", () => {+ for (const hook of ["session-start", "pre-tool-use", "stop", "pre-compact", "post-commit"]) {+ expect(existsSync(join(scriptsDir, `${hook}.mjs`))).toBe(true);+ expect(readFileSync(join(scriptsDir, `${hook}.mjs`), "utf-8")).not.toContain("_capture.mjs");+ }+ });+});diff --git a/test/observe-dedup-prompt.test.ts b/test/observe-dedup-prompt.test.tsindex efab851b..0b12485f 100644--- a/test/observe-dedup-prompt.test.ts+++ b/test/observe-dedup-prompt.test.ts@@ -84,7 +84,7 @@ describe("observe dedup for hooks without tool_input (#1173)", () => {expect(second.deduplicated).toBe(true);});- it("keeps tool_input as the dedup key for tool hooks (response changes still dedup)", async () => {+ it("keeps a repeated tool call when its output changed", async () => {const { registerObserveFunction } = await import("../src/functions/observe.js");const { DedupMap } = await import("../src/functions/dedup.js");const sdk = mockSdk({ looseTrigger: true });@@ -109,6 +109,31 @@ describe("observe dedup for hooks without tool_input (#1173)", () => {)) as { deduplicated?: boolean };expect(first.observationId).toBeTruthy();- expect(second.deduplicated).toBe(true);+ expect(second.deduplicated).toBeUndefined();+ expect((second as { observationId?: string }).observationId).toBeTruthy();+ });++ it("still skips a repeated tool call with the same input and output, and counts it", async () => {+ const { registerObserveFunction } = await import("../src/functions/observe.js");+ const { DedupMap, getDedupSkippedCount } = await import("../src/functions/dedup.js");+ const sdk = mockSdk({ looseTrigger: true });+ const kv = mockKV();+ registerObserveFunction(sdk as never, kv as never, new DedupMap());+ const before = getDedupSkippedCount();++ for (const field of ["tool_response", "tool_output", "output"]) {+ const data = { tool_name: "Bash", tool_input: { command: `ls ${field}` }, [field]: "a.txt" };+ const first = (await sdk.trigger("mem::observe", observePayload("post_tool_use", data))) as { observationId?: string };+ const second = (await sdk.trigger("mem::observe", observePayload("post_tool_use", data))) as { deduplicated?: boolean };+ const changed = (await sdk.trigger(+ "mem::observe",+ observePayload("post_tool_use", { ...data, [field]: "b.txt" }),+ )) as { observationId?: string; deduplicated?: boolean };+ expect(first.observationId).toBeTruthy();+ expect(second.deduplicated).toBe(true);+ expect(changed.deduplicated).toBeUndefined();+ expect(changed.observationId).toBeTruthy();+ }+ expect(getDedupSkippedCount() - before).toBe(3);});});diff --git a/test/status-observe-dedup.test.ts b/test/status-observe-dedup.test.tsnew file mode 100644index 00000000..eeb3429a--- /dev/null+++ b/test/status-observe-dedup.test.ts@@ -0,0 +1,48 @@+import { describe, it, expect } from "vitest";+import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js";++function inputs(overrides: Partial<StatusInputs> = {}): StatusInputs {+ return {+ now: new Date("2026-09-24T12:00:00.000Z"),+ version: "0.9.29",+ engineVersion: "0.22.1",+ uptimeSeconds: 125,+ stateBackend: "file",+ ports: { rest: 3111, streams: 3112, viewer: 3113 },+ health: { status: "healthy", alerts: [], notes: [], connectionState: "connected" },+ circuitBreaker: { state: "closed", failures: 0 },+ functionMetrics: [],+ provider: "llm",+ embeddingProvider: "embeddings",+ flags: [],+ index: {+ bm25Documents: 117,+ vectorDocuments: 117,+ observationsIndexed: 110,+ missingObservations: 0,+ sessions: 95,+ bm25Incomplete: false,+ pendingVectorBackfill: 0,+ },+ graph: null,+ graphExtractionEnabled: true,+ auditLegacy: null,+ ...overrides,+ };+}++describe("observe repeat counter", () => {+ it("reports and renders skipped repeated tool calls", () => {+ const report = evaluateStatus(inputs({ observeDedup: { skippedSinceStart: 4, windowSeconds: 300 } }));+ expect(report.observeDedup).toEqual({ skippedSinceStart: 4, windowSeconds: 300 });+ const html = renderStatusHtml(report, "n");+ expect(html).toContain("Repeats skipped");+ expect(html).toContain("4 since start");+ });++ it("leaves the counter out when it is not reported", () => {+ const report = evaluateStatus(inputs());+ expect(report.observeDedup).toBeNull();+ expect(renderStatusHtml(report, "n")).not.toContain("Repeats skipped");+ });+});diff --git a/test/status.test.ts b/test/status.test.tsindex 4751c488..cc992a97 100644--- a/test/status.test.ts+++ b/test/status.test.ts@@ -451,7 +451,7 @@ describe("status wiring", () => {it("time-boxes every status probe so a slow store cannot hang the page", () => {const reporter = api.slice(api.indexOf("export function createStatusReporter"), api.indexOf("export function createConsolidationStatusReader"));- expect(reporter.match(/valueWithin\(/g)?.length).toBe(5);+ expect(reporter.match(/valueWithin\(/g)?.length).toBe(6);expect(reporter).toMatch(/valueWithin\(scan\.run\(\), STATUS_CHECK_TIMEOUT_MS\)/);expect(api).toMatch(/run: singleFlight\(async \(\) => \{\s*const value = await findUnindexedObservations\(kv\);/);const handler = api.slice(api.indexOf('registerFunction("api::status"'), api.indexOf('function_id: "api::status"'));diff --git a/tsdown.config.ts b/tsdown.config.tsindex 0c6426c1..7a8be92b 100644--- a/tsdown.config.ts+++ b/tsdown.config.ts@@ -15,8 +15,16 @@ const hookEntries = ["src/hooks/session-end.ts","src/hooks/post-commit.ts","src/hooks/antigravity-bridge.ts",+ "src/hooks/_capture.ts",];+const sharedCaptureModule = {+ name: "agentmemory-shared-capture",+ resolveId(source: string) {+ return source === "./_capture.js" ? { id: "./_capture.mjs", external: true } : null;+ },+};+const shared = {format: ["esm"] as const,target: "node20" as const,@@ -73,6 +81,7 @@ export default defineConfig([entry: [entry],outDir: "dist/hooks",...shared,+ plugins: [sharedCaptureModule],clean: false,sourcemap: false,})),@@ -80,6 +89,7 @@ export default defineConfig([entry: [entry],outDir: "plugin/scripts",...shared,+ plugins: [sharedCaptureModule],clean: false,sourcemap: false,})),