#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix: preserve structured capture redaction (#1569)
Rohit Ghumareea2f629f04d7
diff --git a/plugin/scripts/_capture.mjs b/plugin/scripts/_capture.mjsindex efa2cc79..e4a136a7 100644--- a/plugin/scripts/_capture.mjs+++ b/plugin/scripts/_capture.mjs@@ -246,6 +246,29 @@ function stripPrivateData(input) {}return result;}+function scrubRecord(value, options = {}) {+ if (typeof value === "string") return stripPrivateData(value);+ if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options));+ if (value !== null && typeof value === "object") {+ const proto = Object.getPrototypeOf(value);+ if (proto !== Object.prototype && proto !== null) return value;+ const entries = Object.entries(value).map(([key, item]) => [options.scrubKeys ? stripPrivateData(key) : key, item]);+ const reserved = new Set(entries.map(([key]) => key));+ const used = /* @__PURE__ */ new Set();+ return Object.fromEntries(entries.map(([key, item]) => {+ let unique = key;+ if (used.has(unique)) {+ let suffix = 2;+ do+ unique = `${key}_${suffix++}`;+ while (reserved.has(unique) || used.has(unique));+ }+ used.add(unique);+ return [unique, scrubRecord(item, options)];+ }));+ }+ return value;+}//#endregion//#region src/capture/spool.tsconst DEFAULT_MAX_BYTES = 5 * 1024 * 1024;@@ -391,13 +414,6 @@ function fileSize(path) {return 0;}}-function sanitizeBody(body) {- try {- return JSON.parse(stripPrivateData(JSON.stringify(body)));- } catch {- return body;- }-}function withoutImage(body) {const data = body["data"];if (!data || typeof data !== "object" || Array.isArray(data)) return body;@@ -483,7 +499,7 @@ function appendSpool(url, eventId, body, reason, options = {}) {spooledAt: (/* @__PURE__ */ new Date()).toISOString(),reason,attempts: options.attempts ?? 0,- body: sanitizeBody(body)+ body: scrubRecord(JSON.parse(JSON.stringify(body)), { scrubKeys: true })};let line = JSON.stringify(record) + "\n";if (Buffer.byteLength(line) > policy.maxRecordBytes) {@@ -583,7 +599,7 @@ function retainSent(url, eventId, body, mark, options = {}) {spooledAt: new Date(now).toISOString(),reason: "sent",attempts: 0,- body: sanitizeBody(body),+ body: scrubRecord(JSON.parse(JSON.stringify(body)), { scrubKeys: true }),bootId: mark.bootId};let line = JSON.stringify(record) + "\n";diff --git a/scripts/release-gate/vector-recovery.mjs b/scripts/release-gate/vector-recovery.mjsindex a54fbb00..ad210286 100644--- a/scripts/release-gate/vector-recovery.mjs+++ b/scripts/release-gate/vector-recovery.mjs@@ -112,8 +112,8 @@ async function rest(path, body, timeoutMs = 3_000) {const response = await fetch(`http://127.0.0.1:${port}/agentmemory${path}`, { method: body ? "POST" : "GET", headers: { "content-type": "application/json", ...(secret ? { authorization: `Bearer ${secret}` } : {}) }, body: body ? JSON.stringify(body) : undefined, signal: AbortSignal.timeout(timeoutMs) });assert(response.ok, `${path}: HTTP ${response.status}`); return response.json();}-function persisted() {- try { return decodeRkyvJsonScope(readFileSync(join(data, "state_store.db", `${encodeURIComponent("mem:index:bm25")}.bin`))); } catch { return {}; }+function persisted(scope = "mem:index:bm25") {+ try { return decodeRkyvJsonScope(readFileSync(join(data, "state_store.db", `${encodeURIComponent(scope)}.bin`))); } catch { return {}; }}async function state(functionId, payload) {const client = registerWorker(`ws://127.0.0.1:${port + 46023}`, { workerName: "vector-recovery-fixture", invocationTimeoutMs: 5_000, enableMetricsReporting: false, otel: { enabled: false } });@@ -232,7 +232,12 @@ try {failEmbeddings = true;assert.equal((await rest("/snapshot/restore", { commitHash }, 30_000)).success, true);failEmbeddings = false; releaseHeld(); releaseHeld = undefined;- await check("incremental-snapshot", 7, 1, "paused", 4); await stop();+ await check("incremental-snapshot", 7, 1, "paused", 4);+ await poll("restored observation persisted before restart", () => {+ const restored = persisted("mem:obs:incremental")[target.id];+ return restored?.title === "Onyx restored historical state" && restored?.timestamp === timestamp;+ });+ await stop();await start("incremental-snapshot-gated");await check("incremental-snapshot-gated", 7, 1, "waiting-for-opt-in", 0); await stop();await start("incremental-snapshot-recovered", { AGENTMEMORY_VECTOR_BACKFILL: "all" });diff --git a/src/capture/spool.ts b/src/capture/spool.tsindex 5b71b4a8..4c9a66bc 100644--- a/src/capture/spool.ts+++ b/src/capture/spool.ts@@ -16,7 +16,7 @@ import {} from "node:fs";import { join, resolve } from "node:path";import { resolveDataDir } from "../cli-data-dir.js";-import { stripPrivateData } from "../functions/privacy.js";+import { scrubRecord } from "../functions/privacy.js";export interface SpoolRecord {v: 1;@@ -239,14 +239,6 @@ function fileSize(path: string): number {}}-function sanitizeBody(body: Record<string, unknown>): Record<string, unknown> {- try {- return JSON.parse(stripPrivateData(JSON.stringify(body))) as Record<string, unknown>;- } catch {- return body;- }-}-function withoutImage(body: Record<string, unknown>): Record<string, unknown> {const data = body["data"];if (!data || typeof data !== "object" || Array.isArray(data)) return body;@@ -334,7 +326,7 @@ export function appendSpool(spooledAt: new Date().toISOString(),reason,attempts: options.attempts ?? 0,- body: sanitizeBody(body),+ body: scrubRecord(JSON.parse(JSON.stringify(body)), { scrubKeys: true }),};let line = JSON.stringify(record) + "\n";if (Buffer.byteLength(line) > policy.maxRecordBytes) {@@ -441,7 +433,7 @@ export function retainSent(spooledAt: new Date(now).toISOString(),reason: "sent",attempts: 0,- body: sanitizeBody(body),+ body: scrubRecord(JSON.parse(JSON.stringify(body)), { scrubKeys: true }),bootId: mark.bootId,};let line = JSON.stringify(record) + "\n";diff --git a/src/functions/observe.ts b/src/functions/observe.tsindex dcffd770..0556bf26 100644--- a/src/functions/observe.ts+++ b/src/functions/observe.ts@@ -7,7 +7,7 @@ import { StateKV } from "../state/kv.js";import { trackViewerStreamItem, pruneViewerStreamIfDue } from "../state/viewer-stream.js";import { addSessionToProjectIndex } from "../state/session-index.js";import { indexObservationSession } from "../state/obs-index.js";-import { stripPrivateData } from "./privacy.js";+import { scrubRecord } from "./privacy.js";import { DedupMap, recordDedupSkip } from "./dedup.js";import { afterKeyedLock, withKeyedLock } from "../state/keyed-mutex.js";import { getFinalizeIdleMs, isAutoCompressEnabled } from "../config.js";@@ -123,14 +123,10 @@ export function registerObserveFunction(}}- let sanitizedRaw: unknown = payload.data;- try {- const jsonStr = JSON.stringify(payload.data);- const sanitized = stripPrivateData(jsonStr);- sanitizedRaw = JSON.parse(sanitized);- } catch {- sanitizedRaw = stripPrivateData(String(payload.data));- }+ const serializedData = JSON.stringify(payload.data);+ const sanitizedRaw = serializedData === undefined+ ? undefined+ : scrubRecord(JSON.parse(serializedData), { scrubKeys: true });let originChannel: Origin["channel"] = "agent";if (payload.hookType === "prompt_submit") originChannel = "user";diff --git a/src/functions/privacy.ts b/src/functions/privacy.tsindex 4aff36c0..d20cb792 100644--- a/src/functions/privacy.ts+++ b/src/functions/privacy.ts@@ -36,17 +36,29 @@ export function stripPrivateData(input: string): string {return result;}-export function scrubRecord<T>(value: T): T {+export function scrubRecord<T>(value: T, options: { scrubKeys?: boolean } = {}): T {if (typeof value === "string") return stripPrivateData(value) as T;- if (Array.isArray(value)) return value.map((item) => scrubRecord(item)) as T;+ if (Array.isArray(value)) return value.map((item) => scrubRecord(item, options)) as T;if (value !== null && typeof value === "object") {const proto = Object.getPrototypeOf(value);if (proto !== Object.prototype && proto !== null) return value;- const out: Record<string, unknown> = {};- for (const [key, item] of Object.entries(value as Record<string, unknown>)) {- out[key] = scrubRecord(item);- }- return out as T;+ const entries = Object.entries(value as Record<string, unknown>).map(([key, item]) => [+ options.scrubKeys ? stripPrivateData(key) : key,+ item,+ ] as const);+ const reserved = new Set(entries.map(([key]) => key));+ const used = new Set<string>();+ return Object.fromEntries(+ entries.map(([key, item]) => {+ let unique = key;+ if (used.has(unique)) {+ let suffix = 2;+ do { unique = `${key}_${suffix++}`; } while (reserved.has(unique) || used.has(unique));+ }+ used.add(unique);+ return [unique, scrubRecord(item, options)];+ }),+ ) as T;}return value;}diff --git a/test/capture-redaction.test.ts b/test/capture-redaction.test.tsnew file mode 100644index 00000000..0719d6df--- /dev/null+++ b/test/capture-redaction.test.ts@@ -0,0 +1,190 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { appendSpool, retainSent, type SpoolRecord } from "../src/capture/spool.js";+import { registerObserveFunction } from "../src/functions/observe.js";+import { KV } from "../src/state/schema.js";+import type { CompressedObservation, RawObservation } from "../src/types.js";+import { mockKV, mockSdk } from "./helpers/mocks.js";++vi.mock("../src/logger.js", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++const SECRET = "abcdefghijklmnopqrstuvwxyz";+const PRIVATE_TEXT = "private capture details";+const NORMAL = { count: 3, ok: true, empty: null, text: "quoted \"text\" and a \\ path\nnext line" };+class SerializedToolOutput {+ toJSON(key: string) {+ return { key, [`sk-${SECRET}`]: `password=${SECRET}`, ["__proto__"]: { note: `<private>${PRIVATE_TEXT}</private>` } };+ }+}++const CASES = [+ {+ name: "colliding redacted property names",+ data: { nested: [{ [`sk-${SECRET}`]: "first", [`pk-${SECRET}`]: "second", "[REDACTED_SECRET]_2": "ordinary" }] },+ expected: { nested: [{ "[REDACTED_SECRET]": "first", "[REDACTED_SECRET]_3": "second", "[REDACTED_SECRET]_2": "ordinary" }] },+ },+ {+ name: "nested URL objects and dates",+ data: { tool_name: "Read", tool_input: [{ url: new URL(`https://user:${SECRET}@example.test/path`) }], tool_output: new Date("2026-10-16T00:00:00.000Z") },+ expected: { tool_name: "Read", tool_input: [{ url: "https://[REDACTED_SECRET]@example.test/path" }], tool_output: "2026-10-16T00:00:00.000Z" },+ },+ {+ name: "custom JSON representations and boxed strings",+ data: { tool_name: "Read", tool_input: new String(`api_key=${SECRET}`), tool_output: new SerializedToolOutput() },+ expected: { tool_name: "Read", tool_input: "[REDACTED_SECRET]", tool_output: { key: "tool_output", "[REDACTED_SECRET]": "[REDACTED_SECRET]", ["__proto__"]: { note: "[REDACTED]" } } },+ },+ {+ name: "plain objects with custom JSON representations",+ data: { toJSON() { return { tool_name: "Read", tool_output: `api_key=${SECRET}` }; } },+ expected: { tool_name: "Read", tool_output: "[REDACTED_SECRET]" },+ },+ {+ name: "undefined payloads",+ data: undefined,+ expected: undefined,+ },+ {+ name: "a secret assignment at the end of a string",+ data: { tool_name: "Bash", tool_input: { command: "printenv" }, tool_output: `api_key=${SECRET}` },+ expected: { tool_name: "Bash", tool_input: { command: "printenv" }, tool_output: "[REDACTED_SECRET]" },+ },+ {+ name: "quoted secrets and private blocks in nested objects and arrays",+ data: {+ tool_name: "Bash",+ tool_input: { command: `password="${SECRET}"`, options: NORMAL },+ tool_output: [+ { message: `before <private>${PRIVATE_TEXT}\n\"quoted\"</private> after` },+ [`token='${SECRET}'`, "ordinary result", 7, false, null],+ ],+ },+ expected: {+ tool_name: "Bash",+ tool_input: { command: "[REDACTED_SECRET]", options: NORMAL },+ tool_output: [+ { message: "before [REDACTED] after" },+ ["[REDACTED_SECRET]", "ordinary result", 7, false, null],+ ],+ },+ },+ {+ name: "an array payload",+ data: [`api_key=${SECRET}`, { note: `<private>${PRIVATE_TEXT}</private>`, ...NORMAL }],+ expected: ["[REDACTED_SECRET]", { note: "[REDACTED]", ...NORMAL }],+ },+ {+ name: "ordinary JSON data",+ data: { tool_name: "Read", tool_input: { file_path: "src/main.ts" }, tool_output: NORMAL },+ expected: { tool_name: "Read", tool_input: { file_path: "src/main.ts" }, tool_output: NORMAL },+ },+ {+ name: "secrets in nested object property names",+ data: {+ [`sk-${SECRET}`]: "label",+ tool_name: "Read",+ tool_input: [{ [`<private>${PRIVATE_TEXT}</private>`]: { [`api_key=${SECRET}`]: NORMAL } }],+ tool_output: "ordinary result",+ },+ expected: {+ "[REDACTED_SECRET]": "label",+ tool_name: "Read",+ tool_input: [{ "[REDACTED]": { "[REDACTED_SECRET]": NORMAL } }],+ tool_output: "ordinary result",+ },+ },+ {+ name: "an own __proto__ JSON field",+ data: { tool_name: "Read", tool_input: { ["__proto__"]: { note: `api_key=${SECRET}` } }, tool_output: NORMAL },+ expected: { tool_name: "Read", tool_input: { ["__proto__"]: { note: "[REDACTED_SECRET]" } }, tool_output: NORMAL },+ },+];++function payload(data: unknown) {+ return {+ hookType: "post_tool_use" as const,+ sessionId: "ses_capture_redaction",+ project: "redaction-test",+ cwd: "/work/redaction-test",+ timestamp: "2026-10-16T00:00:00.000Z",+ data,+ };+}++function expectClean(value: unknown): void {+ const text = JSON.stringify(value);+ expect(text).not.toContain(SECRET);+ expect(text).not.toContain(PRIVATE_TEXT);+}++describe.each(["pending", "retained"] as const)("%s capture spool redaction", (mode) => {+ let dir: string;++ beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), "am-capture-redaction-"));+ });++ afterEach(() => {+ rmSync(dir, { recursive: true, force: true });+ });++ it.each(CASES)("preserves structure while scrubbing $name", ({ data, expected }) => {+ const body = payload(data);+ const before = JSON.stringify(body);+ const url = "http://localhost:4811";+ const eventId = "evc_redaction000001";+ if (mode === "pending") {+ expect(appendSpool(url, eventId, body, "unreachable", { dir }).spooled).toBe(true);+ } else {+ expect(retainSent(url, eventId, body, { bootId: "redactionboot001", durableAfterMs: 60_000 }, { dir })).toBe(true);+ }+ const files = readdirSync(dir).filter((file) => file.endsWith(".jsonl"));+ expect(files).toHaveLength(1);+ const text = readFileSync(join(dir, files[0]!), "utf-8");+ expectClean(text);+ const record = JSON.parse(text.trim()) as SpoolRecord;+ expect(record.body).toEqual(JSON.parse(JSON.stringify({ ...payload(expected) })));+ expect(record.eventId).toBe(eventId);+ expect(JSON.stringify(body)).toBe(before);+ });+});++describe("direct observation redaction", () => {+ beforeEach(() => {+ vi.stubEnv("AGENTMEMORY_AUTO_COMPRESS", "true");+ });++ afterEach(() => {+ vi.unstubAllEnvs();+ });++ it.each(CASES)("preserves structure while scrubbing $name", async ({ data, expected }) => {+ const sdk = mockSdk({ looseTrigger: true });+ const kv = mockKV();+ const stream = vi.fn(async (_data: unknown) => null);+ const compress = vi.fn(async (_data: unknown) => null);+ sdk.registerFunction("stream::send", stream);+ sdk.registerFunction("mem::compress", compress);+ registerObserveFunction(sdk as never, kv as never);+ const input = payload(data);+ const before = JSON.stringify(input);++ const result = await sdk.trigger({ function_id: "mem::observe", payload: input }) as { observationId: string };++ expect(result.observationId).toBeTruthy();+ const stored = await kv.get<CompressedObservation>(KV.observations(input.sessionId), result.observationId);+ const streamed = (stream.mock.calls[0]![0] as { data: { observation: RawObservation } }).data.observation;+ expect(streamed.raw).toEqual(expected);+ expect(stored?.source?.toolName).toBe(streamed.toolName);+ expect(stored?.source?.toolInput).toEqual(streamed.toolInput);+ expect(stored?.source?.toolOutput).toEqual(streamed.toolOutput);+ if (!streamed.toolName) expect(stored?.source?.payload).toEqual(expected);+ expectClean(stored);+ expectClean(streamed);+ expect(compress).toHaveBeenCalledWith({ observationId: result.observationId, sessionId: input.sessionId });+ expect(JSON.stringify(input)).toBe(before);+ });+});diff --git a/test/privacy.test.ts b/test/privacy.test.tsindex 0fa17b68..8277045b 100644--- a/test/privacy.test.ts+++ b/test/privacy.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from "vitest";-import { stripPrivateData } from "../src/functions/privacy.js";+import { scrubRecord, stripPrivateData } from "../src/functions/privacy.js";describe("stripPrivateData", () => {it("strips private tags", () => {@@ -138,3 +138,41 @@ describe("stripPrivateData", () => {expect(stripPrivateData(expected)).toBe(expected);});});++describe("scrubRecord property names", () => {+ const secret = "sk-abcdefghijklmnopqrstuvwxyz";++ it("leaves property names unchanged by default", () => {+ expect(scrubRecord({ [secret]: secret })).toEqual({ [secret]: "[REDACTED_SECRET]" });+ });++ it("optionally scrubs property names recursively without changing the input", () => {+ const input = { nested: [{ [secret]: { note: secret, count: 3, ok: true, empty: null } }] };+ const before = structuredClone(input);+ expect(scrubRecord(input, { scrubKeys: true })).toEqual({+ nested: [{ "[REDACTED_SECRET]": { note: "[REDACTED_SECRET]", count: 3, ok: true, empty: null } }],+ });+ expect(input).toEqual(before);+ });++ it.each([false, true])("preserves own __proto__ fields with scrubKeys=%s", (scrubKeys) => {+ const input = { ["__proto__"]: { note: secret }, constructor: "ordinary value" };+ const output = scrubRecord(input, { scrubKeys });+ expect(Object.getPrototypeOf(output)).toBe(Object.prototype);+ expect(Object.hasOwn(output, "__proto__")).toBe(true);+ expect(output).toEqual({ ["__proto__"]: { note: "[REDACTED_SECRET]" }, constructor: "ordinary value" });+ expect(input["__proto__"].note).toBe(secret);+ });++ it("preserves both values when scrubbed property names collide", () => {+ expect(scrubRecord({ [secret]: "first", "[REDACTED_SECRET]": "last" }, { scrubKeys: true }))+ .toEqual({ "[REDACTED_SECRET]": "first", "[REDACTED_SECRET]_2": "last" });+ });++ it("avoids existing names when allocating collision suffixes", () => {+ const input = { [secret]: "first", "pk-abcdefghijklmnopqrstuvwxyz": "second", "[REDACTED_SECRET]_2": "ordinary" };+ const output = scrubRecord(input, { scrubKeys: true });+ expect(output).toEqual({ "[REDACTED_SECRET]": "first", "[REDACTED_SECRET]_3": "second", "[REDACTED_SECRET]_2": "ordinary" });+ expect(scrubRecord(output, { scrubKeys: true })).toEqual(output);+ });+});