Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix: clear deleted memories from bm25 and vector indices (#636)

Abhinav Mahajan8c558c6559be
diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts
index 49493572..74c571b9 100644
--- a/src/functions/auto-forget.ts
+++ b/src/functions/auto-forget.ts
@@ -4,6 +4,7 @@ import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { recordAudit } from "./audit.js";
import { deleteAccessLog } from "./access-tracker.js";
+import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js";
import { logger } from "../logger.js";
const MS_PER_DAY = 24 * 60 * 60 * 1000;
@@ -53,6 +54,8 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
timestamp: mem.forgetAfter,
});
await deleteAccessLog(kv, mem.id);
+ getSearchIndex().remove(mem.id);
+ vectorIndexRemove(mem.id);
}
}
}
@@ -179,12 +182,18 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
sessionId: sessions[i].id,
timestamp: obs.timestamp,
});
+ getSearchIndex().remove(obs.id);
+ vectorIndexRemove(obs.id);
}
}
}
}
}
+ if (!dryRun && (result.ttlExpired.length > 0 || result.lowValueObs.length > 0)) {
+ await flushIndexSave();
+ }
+
logger.info("Auto-forget complete", {
ttlExpired: result.ttlExpired.length,
contradictions: result.contradictions.length,
diff --git a/src/functions/governance.ts b/src/functions/governance.ts
index 60813dd4..16d0d1c4 100644
--- a/src/functions/governance.ts
+++ b/src/functions/governance.ts
@@ -4,6 +4,7 @@ import { KV } from "../state/schema.js";
import type { StateKV } from "../state/kv.js";
import { recordAudit, safeAudit, queryAudit } from "./audit.js";
import { deleteAccessLog } from "./access-tracker.js";
+import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js";
import { logger } from "../logger.js";
export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {
@@ -23,10 +24,14 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {
if (mem) {
await kv.delete(KV.memories, id);
await deleteAccessLog(kv, id);
+ getSearchIndex().remove(id);
+ vectorIndexRemove(id);
deleted++;
}
}
+ if (deleted > 0) await flushIndexSave();
+
await recordAudit(
kv,
"delete",
@@ -107,6 +112,8 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {
batch.map(async (mem) => {
await kv.delete(KV.memories, mem.id);
await deleteAccessLog(kv, mem.id);
+ getSearchIndex().remove(mem.id);
+ vectorIndexRemove(mem.id);
}),
);
results.forEach((result, j) => {
@@ -129,6 +136,8 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {
});
}
+ if (successfulIds.length > 0) await flushIndexSave();
+
await safeAudit(
kv,
"delete",
diff --git a/src/functions/remember.ts b/src/functions/remember.ts
index 8d667705..30199ecd 100644
--- a/src/functions/remember.ts
+++ b/src/functions/remember.ts
@@ -6,7 +6,7 @@ import { withKeyedLock } from "../state/keyed-mutex.js";
import { memoryToObservation } from "../state/memory-utils.js";
import { deleteAccessLog } from "./access-tracker.js";
import { recordAudit } from "./audit.js";
-import { getSearchIndex, vectorIndexAddGuarded } from "./search.js";
+import { getSearchIndex, vectorIndexAddGuarded, vectorIndexRemove, flushIndexSave } from "./search.js";
import { logger } from "../logger.js";
export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
@@ -157,6 +157,8 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
await decrementImageRef(kv, sdk, mem.imageRef);
}
await deleteAccessLog(kv, data.memoryId);
+ getSearchIndex().remove(data.memoryId);
+ vectorIndexRemove(data.memoryId);
deletedMemoryIds.push(data.memoryId);
deleted++;
}
@@ -176,6 +178,8 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
if (obs?.imageRef && obs.imageRef !== obs.imageData) {
await decrementImageRef(kv, sdk, obs.imageRef);
}
+ getSearchIndex().remove(obsId);
+ vectorIndexRemove(obsId);
deletedObservationIds.push(obsId);
deleted++;
}
@@ -195,6 +199,8 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
if (obs.imageRef && obs.imageRef !== obs.imageData) {
await decrementImageRef(kv, sdk, obs.imageRef);
}
+ getSearchIndex().remove(obs.id);
+ vectorIndexRemove(obs.id);
deletedObservationIds.push(obs.id);
deleted++;
}
@@ -205,6 +211,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
}
if (deleted > 0) {
+ await flushIndexSave();
await recordAudit(
kv,
"forget",
diff --git a/src/functions/retention.ts b/src/functions/retention.ts
index c9c16573..fd3e6466 100644
--- a/src/functions/retention.ts
+++ b/src/functions/retention.ts
@@ -14,6 +14,7 @@ import {
normalizeAccessLog,
} from "./access-tracker.js";
import { recordAudit } from "./audit.js";
+import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js";
import { logger } from "../logger.js";
const DEFAULT_DECAY: DecayConfig = {
@@ -370,6 +371,8 @@ export function registerRetentionFunctions(
await kv.delete(scope, candidate.memoryId);
await kv.delete(KV.retentionScores, candidate.memoryId);
await deleteAccessLog(kv, candidate.memoryId);
+ getSearchIndex().remove(candidate.memoryId);
+ vectorIndexRemove(candidate.memoryId);
evicted++;
evictedIds.push(candidate.memoryId);
if (resolvedSource === "semantic") evictedSemantic++;
@@ -386,6 +389,7 @@ export function registerRetentionFunctions(
// one record per invocation — per-candidate audits would flood
// the audit log during normal eviction sweeps.
if (evicted > 0) {
+ await flushIndexSave();
await recordAudit(kv, "delete", "mem::retention-evict", evictedIds, {
threshold,
evicted,
diff --git a/src/functions/search.ts b/src/functions/search.ts
index b4444b48..dc629d6d 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -34,6 +34,44 @@ export function getEmbeddingProvider(): EmbeddingProvider | null {
return currentEmbeddingProvider
}
+export function vectorIndexRemove(id: string): void {
+ vectorIndex?.remove(id);
+}
+
+// Persistence sync hook. Without this, index removals only live in
+// memory; a crash/SIGKILL before graceful shutdown reloads a stale
+// snapshot at boot and the deleted entry resurrects in the index.
+// Wired by src/index.ts after IndexPersistence is constructed; no-op
+// until then so unit tests that exercise the delete paths in
+// isolation don't need to wire persistence.
+let indexPersistence: {
+ scheduleSave: () => void;
+ save: () => Promise<void>;
+} | null = null;
+
+export function setIndexPersistence(
+ p: { scheduleSave: () => void; save: () => Promise<void> } | null,
+): void {
+ indexPersistence = p;
+}
+
+export function scheduleIndexSave(): void {
+ indexPersistence?.scheduleSave();
+}
+
+// Synchronous flush variant for delete paths. The debounced
+// scheduleSave is fine for adds (chatty), but a hard process exit
+// inside the 5s debounce window would lose deletes and resurrect
+// removed entries on next boot. Deletes are infrequent enough that
+// awaiting a single write per operation is acceptable. save() catches
+// its own errors via IndexPersistence.logFailure, so this resolves
+// even when persistence fails — callers must not treat a failed
+// flush as a fatal error on the delete itself (the KV delete already
+// committed before this is invoked).
+export async function flushIndexSave(): Promise<void> {
+ await indexPersistence?.save();
+}
+
// Hard cap on embedding input length. Most providers cap input around
// 8k tokens (~32k chars at ~4 chars/token). Truncate defensively so a
// huge memory.content can't 400 the embed call or blow context budget
diff --git a/src/index.ts b/src/index.ts
index 704d4809..d02b3dc7 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -37,6 +37,7 @@ import {
getSearchIndex,
setVectorIndex,
setEmbeddingProvider,
+ setIndexPersistence,
} from "./functions/search.js";
import { registerContextFunction } from "./functions/context.js";
import { registerSummarizeFunction } from "./functions/summarize.js";
@@ -344,6 +345,11 @@ async function main() {
const healthMonitor = registerHealthMonitor(sdk, kv);
const indexPersistence = new IndexPersistence(kv, bm25Index, vectorIndex);
+ // Wire the persistence hook so delete paths can flush BM25/vector
+ // index mutations to disk. Without this, an in-memory remove can be
+ // lost across a hard process exit and the persisted snapshot
+ // restores the deleted entry at next boot.
+ setIndexPersistence(indexPersistence);
const loaded = await indexPersistence.load().catch((err) => {
console.warn(`[agentmemory] Failed to load persisted index:`, err);
diff --git a/src/state/search-index.ts b/src/state/search-index.ts
index f2530078..67f3e2ef 100644
--- a/src/state/search-index.ts
+++ b/src/state/search-index.ts
@@ -51,6 +51,29 @@ export class SearchIndex {
return this.entries.has(id);
}
+ remove(id: string): void {
+ const entry = this.entries.get(id);
+ if (!entry) return;
+
+ const termFreq = this.docTermCounts.get(id);
+ if (termFreq) {
+ for (const term of termFreq.keys()) {
+ const postingList = this.invertedIndex.get(term);
+ if (postingList) {
+ postingList.delete(id);
+ if (postingList.size === 0) {
+ this.invertedIndex.delete(term);
+ }
+ }
+ }
+ this.docTermCounts.delete(id);
+ }
+
+ this.totalDocLength = Math.max(0, this.totalDocLength - entry.termCount);
+ this.entries.delete(id);
+ this.sortedTerms = null;
+ }
+
search(
query: string,
limit = 20,
diff --git a/test/auto-forget.test.ts b/test/auto-forget.test.ts
index 367e6c59..0f104eb3 100644
--- a/test/auto-forget.test.ts
+++ b/test/auto-forget.test.ts
@@ -1,10 +1,15 @@
-import { describe, it, expect, beforeEach, vi } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
import { registerAutoForgetFunction } from "../src/functions/auto-forget.js";
+import {
+ getSearchIndex,
+ setIndexPersistence,
+} from "../src/functions/search.js";
+import { memoryToObservation } from "../src/state/memory-utils.js";
import type { Memory, CompressedObservation, Session } from "../src/types.js";
function mockKV() {
@@ -168,6 +173,85 @@ describe("Auto-Forget Function", () => {
expect(stillExists).not.toBeNull();
});
+ describe("search-index cleanup", () => {
+ beforeEach(() => {
+ getSearchIndex().clear();
+ setIndexPersistence(null);
+ });
+
+ afterEach(() => {
+ setIndexPersistence(null);
+ });
+
+ it("removes TTL-expired memories from the BM25 index and flushes persistence", async () => {
+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };
+ setIndexPersistence(persistence);
+
+ const expired = makeMemory({
+ id: "mem_expired",
+ forgetAfter: "2020-01-01T00:00:00Z",
+ });
+ await kv.set("mem:memories", "mem_expired", expired);
+ getSearchIndex().add(memoryToObservation(expired));
+ expect(getSearchIndex().has("mem_expired")).toBe(true);
+
+ await sdk.trigger("mem::auto-forget", {});
+
+ expect(getSearchIndex().has("mem_expired")).toBe(false);
+ expect(persistence.save).toHaveBeenCalled();
+ });
+
+ it("removes evicted low-value observations from the BM25 index", async () => {
+ const session: Session = {
+ id: "ses_1",
+ project: "my-project",
+ cwd: "/tmp",
+ startedAt: "2025-01-01T00:00:00Z",
+ status: "completed",
+ observationCount: 1,
+ };
+ await kv.set("mem:sessions", "ses_1", session);
+
+ const oldLowObs: CompressedObservation = {
+ id: "obs_old",
+ sessionId: "ses_1",
+ timestamp: "2025-01-01T00:00:00Z",
+ type: "other",
+ title: "trivial event",
+ facts: [],
+ narrative: "nothing important",
+ concepts: [],
+ files: [],
+ importance: 1,
+ };
+ await kv.set("mem:obs:ses_1", "obs_old", oldLowObs);
+ getSearchIndex().add(oldLowObs);
+ expect(getSearchIndex().has("obs_old")).toBe(true);
+
+ await sdk.trigger("mem::auto-forget", {});
+
+ expect(getSearchIndex().has("obs_old")).toBe(false);
+ });
+
+ it("does not flush persistence on dryRun", async () => {
+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };
+ setIndexPersistence(persistence);
+
+ const expired = makeMemory({
+ id: "mem_expired",
+ forgetAfter: "2020-01-01T00:00:00Z",
+ });
+ await kv.set("mem:memories", "mem_expired", expired);
+ getSearchIndex().add(memoryToObservation(expired));
+
+ await sdk.trigger("mem::auto-forget", { dryRun: true });
+
+ // dryRun must not mutate the index or write to disk.
+ expect(getSearchIndex().has("mem_expired")).toBe(true);
+ expect(persistence.save).not.toHaveBeenCalled();
+ });
+ });
+
it("does not flag non-similar memories as contradictions", async () => {
const mem1 = makeMemory({
id: "mem_1",
diff --git a/test/governance.test.ts b/test/governance.test.ts
index 22554164..6390bbee 100644
--- a/test/governance.test.ts
+++ b/test/governance.test.ts
@@ -1,10 +1,15 @@
-import { describe, it, expect, beforeEach, vi } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
import { registerGovernanceFunction } from "../src/functions/governance.js";
+import {
+ getSearchIndex,
+ setIndexPersistence,
+} from "../src/functions/search.js";
+import { memoryToObservation } from "../src/state/memory-utils.js";
import type { Memory, AuditEntry } from "../src/types.js";
function mockKV() {
@@ -133,6 +138,109 @@ describe("Governance Functions", () => {
expect(remaining.length).toBe(3);
});
+ // Delete paths must tear down the BM25 index entry and trigger an
+ // IndexPersistence save so a hard process exit can't restore a stale
+ // snapshot at next boot.
+ describe("search index cleanup on delete", () => {
+ function indexedObs(id: string, title: string) {
+ return memoryToObservation({
+ id,
+ createdAt: "2026-02-01T00:00:00Z",
+ updatedAt: "2026-02-01T00:00:00Z",
+ type: "fact",
+ title,
+ content: title,
+ concepts: [],
+ files: [],
+ sessionIds: ["ses_1"],
+ strength: 5,
+ version: 1,
+ isLatest: true,
+ });
+ }
+
+ function mockPersistence() {
+ return {
+ scheduleSave: vi.fn(),
+ save: vi.fn(async () => {}),
+ };
+ }
+
+ beforeEach(() => {
+ // SearchIndex is a module-level singleton — wipe it so cases
+ // don't bleed into each other.
+ getSearchIndex().clear();
+ setIndexPersistence(null);
+ });
+
+ // The persistence singleton is module-scoped; without this reset
+ // the last test's mock would leak into sibling tests in the outer
+ // suite and trigger unexpected spy invocations.
+ afterEach(() => {
+ setIndexPersistence(null);
+ });
+
+ it("governance-delete removes the memory from the search index", async () => {
+ getSearchIndex().add(indexedObs("mem_1", "alpha"));
+ getSearchIndex().add(indexedObs("mem_2", "beta"));
+ expect(getSearchIndex().has("mem_1")).toBe(true);
+
+ await sdk.trigger("mem::governance-delete", {
+ memoryIds: ["mem_1"],
+ });
+
+ expect(getSearchIndex().has("mem_1")).toBe(false);
+ expect(getSearchIndex().has("mem_2")).toBe(true);
+ });
+
+ it("governance-delete flushes persistence immediately", async () => {
+ const persistence = mockPersistence();
+ setIndexPersistence(persistence);
+ getSearchIndex().add(indexedObs("mem_1", "alpha"));
+
+ await sdk.trigger("mem::governance-delete", { memoryIds: ["mem_1"] });
+
+ // Delete paths must use the synchronous save (not the debounced
+ // scheduleSave) so a process exit immediately after delete can't
+ // resurrect the entry on next boot.
+ expect(persistence.save).toHaveBeenCalled();
+ });
+
+ it("governance-delete skips persistence flush when nothing was deleted", async () => {
+ const persistence = mockPersistence();
+ setIndexPersistence(persistence);
+
+ await sdk.trigger("mem::governance-delete", {
+ memoryIds: ["nonexistent_999"],
+ });
+
+ expect(persistence.save).not.toHaveBeenCalled();
+ });
+
+ it("governance-bulk removes deleted memories from the search index", async () => {
+ getSearchIndex().add(indexedObs("mem_1", "alpha"));
+ getSearchIndex().add(indexedObs("mem_2", "beta"));
+ getSearchIndex().add(indexedObs("mem_3", "gamma"));
+
+ await sdk.trigger("mem::governance-bulk", { type: ["pattern"] });
+
+ // mem_1 and mem_3 are type "pattern" per the outer beforeEach.
+ expect(getSearchIndex().has("mem_1")).toBe(false);
+ expect(getSearchIndex().has("mem_3")).toBe(false);
+ expect(getSearchIndex().has("mem_2")).toBe(true);
+ });
+
+ it("governance-bulk flushes persistence immediately", async () => {
+ const persistence = mockPersistence();
+ setIndexPersistence(persistence);
+ getSearchIndex().add(indexedObs("mem_1", "alpha"));
+
+ await sdk.trigger("mem::governance-bulk", { type: ["pattern"] });
+
+ expect(persistence.save).toHaveBeenCalled();
+ });
+ });
+
it("audit-query returns audit entries", async () => {
await sdk.trigger("mem::governance-delete", {
memoryIds: ["mem_1"],
diff --git a/test/remember-forget-audit.test.ts b/test/remember-forget-audit.test.ts
index 7be571fa..7d17b543 100644
--- a/test/remember-forget-audit.test.ts
+++ b/test/remember-forget-audit.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
@@ -9,6 +9,12 @@ vi.mock("../src/state/keyed-mutex.js", () => ({
}));
import { registerRememberFunction } from "../src/functions/remember.js";
+import {
+ getSearchIndex,
+ setIndexPersistence,
+} from "../src/functions/search.js";
+import { memoryToObservation } from "../src/state/memory-utils.js";
+import type { Memory } from "../src/types.js";
function mockKV() {
const store = new Map<string, Map<string, unknown>>();
@@ -117,3 +123,89 @@ describe("mem::forget audit coverage (issue #125)", () => {
expect(auditRows).toHaveLength(0);
});
});
+
+// Delete paths must tear down the BM25 index entry and synchronously
+// flush the persisted snapshot. Without this, a deleted memory keeps
+// occupying limit-capped search result slots, and an in-memory remove
+// would be lost if the process exits before the debounced save fires.
+describe("mem::forget search-index cleanup", () => {
+ function makeMemory(id: string): Memory {
+ return {
+ id,
+ createdAt: "2026-02-01T00:00:00Z",
+ updatedAt: "2026-02-01T00:00:00Z",
+ type: "fact",
+ title: `title ${id}`,
+ content: `content ${id}`,
+ concepts: [],
+ files: [],
+ sessionIds: ["ses_1"],
+ strength: 5,
+ version: 1,
+ isLatest: true,
+ };
+ }
+
+ beforeEach(() => {
+ getSearchIndex().clear();
+ setIndexPersistence(null);
+ });
+
+ afterEach(() => {
+ setIndexPersistence(null);
+ });
+
+ it("removes a forgotten memory from the BM25 index", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ const mem = makeMemory("mem_a");
+ await kv.set("mem:memories", mem.id, mem);
+ getSearchIndex().add(memoryToObservation(mem));
+ expect(getSearchIndex().has("mem_a")).toBe(true);
+
+ await sdk.trigger({
+ function_id: "mem::forget",
+ payload: { memoryId: "mem_a" },
+ });
+
+ expect(getSearchIndex().has("mem_a")).toBe(false);
+ });
+
+ it("removes forgotten observations from the BM25 index", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ await kv.set("mem:obs:ses_1", "obs_a", { id: "obs_a" });
+ await kv.set("mem:obs:ses_1", "obs_b", { id: "obs_b" });
+ getSearchIndex().add(memoryToObservation(makeMemory("obs_a")));
+ getSearchIndex().add(memoryToObservation(makeMemory("obs_b")));
+
+ await sdk.trigger({
+ function_id: "mem::forget",
+ payload: { sessionId: "ses_1", observationIds: ["obs_a"] },
+ });
+
+ expect(getSearchIndex().has("obs_a")).toBe(false);
+ expect(getSearchIndex().has("obs_b")).toBe(true);
+ });
+
+ it("flushes persistence immediately when a memory is forgotten", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };
+ setIndexPersistence(persistence);
+
+ await kv.set("mem:memories", "mem_a", makeMemory("mem_a"));
+
+ await sdk.trigger({
+ function_id: "mem::forget",
+ payload: { memoryId: "mem_a" },
+ });
+
+ expect(persistence.save).toHaveBeenCalled();
+ });
+});
diff --git a/test/retention.test.ts b/test/retention.test.ts
index 8acfda6b..cf8cc681 100644
--- a/test/retention.test.ts
+++ b/test/retention.test.ts
@@ -1,4 +1,9 @@
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import {
+ getSearchIndex,
+ setIndexPersistence,
+} from "../src/functions/search.js";
+import { memoryToObservation } from "../src/state/memory-utils.js";
import type { Memory, SemanticMemory } from "../src/types.js";
vi.mock("../src/logger.js", () => ({
@@ -503,4 +508,59 @@ describe("RetentionScoring", () => {
const remaining = await kv.list("mem:memories");
expect(remaining).toHaveLength(0);
});
+
+ describe("search-index cleanup on eviction", () => {
+ beforeEach(() => {
+ getSearchIndex().clear();
+ setIndexPersistence(null);
+ });
+
+ afterEach(() => {
+ setIndexPersistence(null);
+ });
+
+ it("removes evicted memories from the BM25 index and flushes persistence", async () => {
+ const { registerRetentionFunctions } = await import(
+ "../src/functions/retention.js"
+ );
+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };
+ setIndexPersistence(persistence);
+
+ const evictee = makeMemory("mem_evict", "fact", 500);
+ const sdk = mockSdk();
+ const kv = mockKV([evictee]);
+ registerRetentionFunctions(sdk as never, kv as never);
+ getSearchIndex().add(memoryToObservation(evictee));
+ expect(getSearchIndex().has("mem_evict")).toBe(true);
+
+ await sdk.trigger({ function_id: "mem::retention-score", payload: {} });
+ await sdk.trigger({
+ function_id: "mem::retention-evict",
+ payload: { threshold: 0.9 },
+ });
+
+ expect(getSearchIndex().has("mem_evict")).toBe(false);
+ expect(persistence.save).toHaveBeenCalled();
+ });
+
+ it("does not flush persistence when nothing is evicted", async () => {
+ const { registerRetentionFunctions } = await import(
+ "../src/functions/retention.js"
+ );
+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };
+ setIndexPersistence(persistence);
+
+ const sdk = mockSdk();
+ const kv = mockKV([makeMemory("mem_fresh", "architecture", 1)]);
+ registerRetentionFunctions(sdk as never, kv as never);
+
+ await sdk.trigger({ function_id: "mem::retention-score", payload: {} });
+ await sdk.trigger({
+ function_id: "mem::retention-evict",
+ payload: { threshold: 0.01 },
+ });
+
+ expect(persistence.save).not.toHaveBeenCalled();
+ });
+ });
});
diff --git a/test/search-index.test.ts b/test/search-index.test.ts
index f1349873..805faec4 100644
--- a/test/search-index.test.ts
+++ b/test/search-index.test.ts
@@ -89,6 +89,81 @@ describe("SearchIndex", () => {
expect(index.search("auth")).toEqual([]);
});
+ // Regression coverage: deleted docs must not keep occupying result
+ // slots after remove(). Without remove(), a limit-capped search can
+ // return fewer live results than requested because the slot is held
+ // by a doc that no longer exists in storage.
+ describe("remove", () => {
+ it("drops a removed doc from search results", () => {
+ index.add(makeObs({ id: "obs_a", title: "jose for JWT" }));
+ index.add(
+ makeObs({ id: "obs_b", title: "JWT tokens expire after 30 minutes" }),
+ );
+ expect(index.search("jose jwt", 1)[0].obsId).toBe("obs_a");
+
+ index.remove("obs_a");
+
+ // With limit=1, the survivor must now surface instead of getting
+ // masked by the deleted doc's slot.
+ const after = index.search("jose jwt", 1);
+ expect(after).toHaveLength(1);
+ expect(after[0].obsId).toBe("obs_b");
+ expect(index.size).toBe(1);
+ });
+
+ it("is a no-op for unknown ids", () => {
+ index.add(makeObs({ id: "obs_a" }));
+ expect(() => index.remove("does_not_exist")).not.toThrow();
+ expect(index.size).toBe(1);
+ expect(index.search("auth")).toHaveLength(1);
+ });
+
+ it("cleans up empty posting lists and the prefix cache", () => {
+ // unique_concept_xyz appears only in obs_a. After removing obs_a,
+ // a prefix search for "unique_" must NOT surface it from the
+ // sortedTerms cache.
+ index.add(
+ makeObs({
+ id: "obs_a",
+ concepts: ["unique_concept_xyz"],
+ }),
+ );
+ index.add(makeObs({ id: "obs_b", title: "completely different" }));
+ // Prime the sortedTerms cache.
+ expect(index.search("unique_")).toHaveLength(1);
+
+ index.remove("obs_a");
+
+ expect(index.search("unique_concept_xyz")).toEqual([]);
+ expect(index.search("unique_")).toEqual([]);
+ });
+
+ it("keeps scoring consistent after add/remove/add cycles", () => {
+ // totalDocLength bookkeeping desync would skew BM25 normalization
+ // (docLen / avgDocLen). Round-trip add → remove → re-add and
+ // confirm size + retrievability hold.
+ const obs = makeObs({ id: "obs_a" });
+ index.add(obs);
+ expect(index.size).toBe(1);
+ index.remove("obs_a");
+ expect(index.size).toBe(0);
+ index.add(obs);
+ expect(index.size).toBe(1);
+ expect(index.search("auth")[0].obsId).toBe("obs_a");
+ });
+
+ it("survives serialize/deserialize after removes", () => {
+ index.add(makeObs({ id: "obs_a", title: "alpha doc" }));
+ index.add(makeObs({ id: "obs_b", title: "beta doc" }));
+ index.remove("obs_a");
+
+ const restored = SearchIndex.deserialize(index.serialize());
+ expect(restored.size).toBe(1);
+ expect(restored.search("alpha")).toEqual([]);
+ expect(restored.search("beta")).toHaveLength(1);
+ });
+ });
+
it("returns empty for empty query", () => {
index.add(makeObs());
expect(index.search("")).toEqual([]);