Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix: index retained custom event payloads (#1542)

Rohit Ghumaree8b961061ded
diff --git a/src/functions/compress-synthetic.ts b/src/functions/compress-synthetic.ts
index 6b13f932..000199b5 100644
--- a/src/functions/compress-synthetic.ts
+++ b/src/functions/compress-synthetic.ts
@@ -68,7 +68,9 @@ function stringifyForNarrative(v: unknown): string {
}
function truncate(s: string, n: number): string {
- return s.length > n ? s.slice(0, n - 1) + "\u2026" : s;
+ if (s.length <= n) return s;
+ const end = /[\uD800-\uDBFF]/.test(s[n - 2]) ? n - 2 : n - 1;
+ return s.slice(0, end) + "\u2026";
}
export function buildSyntheticCompression(
@@ -78,10 +80,16 @@ export function buildSyntheticCompression(
const inputStr = stringifyForNarrative(raw.toolInput);
const outputStr = stringifyForNarrative(raw.toolOutput);
const promptStr = raw.userPrompt ?? "";
+ const source = createObservationSource(raw);
const narrativeParts = [promptStr, inputStr, outputStr].filter(
(s) => s.length > 0,
);
+ const payload = source.payload;
+ if (narrativeParts.length === 0 && payload != null &&
+ (typeof payload !== "object" || Object.keys(payload).length > 0)) {
+ narrativeParts.push(stringifyForNarrative(payload));
+ }
const result: CompressedObservation = {
id: raw.id,
@@ -97,7 +105,7 @@ export function buildSyntheticCompression(
files: extractFiles(raw.toolName, raw.toolInput),
importance: 5,
confidence: 0.3,
- source: createObservationSource(raw),
+ source,
};
if (raw.modality) result.modality = raw.modality;
if (raw.imageData) result.imageData = raw.imageData;
diff --git a/test/custom-payload-search.test.ts b/test/custom-payload-search.test.ts
new file mode 100644
index 00000000..2545d3d1
--- /dev/null
+++ b/test/custom-payload-search.test.ts
@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+import { buildSyntheticCompression } from "../src/functions/compress-synthetic.js";
+import { createObservationSource, OBSERVATION_SOURCE_MAX_BYTES, rawFromObservationSource } from "../src/functions/observation-source.js";
+import { SearchIndex } from "../src/state/search-index.js";
+import type { HookType, RawObservation } from "../src/types.js";
+
+function raw(payload: unknown, fields: Partial<RawObservation> = {}): RawObservation {
+ return { id: "custom-observation", sessionId: "custom-session", timestamp: "2026-01-01T00:00:00Z",
+ hookType: "custom_event" as HookType, raw: payload, ...fields };
+}
+
+describe("custom payload synthetic search", () => {
+ it("indexes retained payload content without inventing tool or prompt fields", () => {
+ const original = raw({ title: "Quasar dispatch choice", content: "Use quasar routing for the job queue" });
+ const observation = buildSyntheticCompression(original);
+ const index = new SearchIndex();
+ index.add(observation);
+ expect(index.search("quasar").map((hit) => hit.obsId)).toContain(observation.id);
+ expect(observation.narrative).toContain("Use quasar routing");
+ expect(observation.source).toEqual(createObservationSource(original));
+ expect(observation.source?.toolOutput).toBeUndefined();
+ expect(rawFromObservationSource(observation)?.raw).toEqual(original.raw);
+ });
+
+ it.each(["message canary", ["message canary"], { nested: { message: "message canary" } }, 0, false])(
+ "keeps supported primitive and structured payloads: %j", (payload) => {
+ const observation = buildSyntheticCompression(raw(payload));
+ const expected = typeof payload === "string" ? payload : JSON.stringify(payload);
+ expect(observation.narrative).toBe(expected);
+ expect(observation.source?.payload).toEqual(payload);
+ },
+ );
+
+ it.each([undefined, null, {}, [], ""])("does not invent narrative for empty data: %j", (payload) => {
+ expect(buildSyntheticCompression(raw(payload)).narrative).toBe("");
+ });
+
+ it("uses the sanitized source and never indexes private data or credentials", () => {
+ const key = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789";
+ const observation = buildSyntheticCompression(raw({ content: `Public canary <private>hidden-canary</private> ${key}` }));
+ expect(observation.narrative).toContain("Public canary");
+ expect(observation.narrative).toContain("[REDACTED]");
+ expect(observation.narrative).not.toContain("hidden-canary");
+ expect(observation.narrative).not.toContain(key);
+ });
+
+ it("preserves canonical prompt/tool precedence without duplicating envelope metadata", () => {
+ const observation = buildSyntheticCompression(raw({ message: "envelope-only-canary" }, {
+ userPrompt: "Prompt canary", toolInput: { command: "run" }, toolOutput: "Output canary",
+ }));
+ expect(observation.narrative).toBe('Prompt canary | {"command":"run"} | Output canary');
+ expect(observation.source).not.toHaveProperty("payload");
+ });
+
+ it("bounds large Unicode data while leaving the retained source budget intact", () => {
+ const original = raw("x".repeat(398) + "😀" + "界".repeat(20000));
+ const observation = buildSyntheticCompression(original);
+ expect(observation.narrative.length).toBeLessThanOrEqual(400);
+ expect(observation.narrative).toBe("x".repeat(398) + "…");
+ expect(observation.source).toEqual(createObservationSource(original));
+ expect(observation.source?.truncated).toBe(true);
+ expect(Buffer.byteLength(JSON.stringify(observation.source))).toBeLessThanOrEqual(OBSERVATION_SOURCE_MAX_BYTES);
+ });
+});