#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix: clear deleted memories from bm25 and vector indices (#636)
Abhinav Mahajan8c558c6559be
diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.tsindex 49493572..74c571b9 100644--- a/src/functions/auto-forget.ts+++ b/src/functions/auto-forget.ts@@ -4,6 +4,7 @@ import { KV } from "../state/schema.js";import { StateKV } from "../state/kv.js";import { recordAudit } from "./audit.js";import { deleteAccessLog } from "./access-tracker.js";+import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js";import { logger } from "../logger.js";const MS_PER_DAY = 24 * 60 * 60 * 1000;@@ -53,6 +54,8 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {timestamp: mem.forgetAfter,});await deleteAccessLog(kv, mem.id);+ getSearchIndex().remove(mem.id);+ vectorIndexRemove(mem.id);}}}@@ -179,12 +182,18 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {sessionId: sessions[i].id,timestamp: obs.timestamp,});+ getSearchIndex().remove(obs.id);+ vectorIndexRemove(obs.id);}}}}}+ if (!dryRun && (result.ttlExpired.length > 0 || result.lowValueObs.length > 0)) {+ await flushIndexSave();+ }+logger.info("Auto-forget complete", {ttlExpired: result.ttlExpired.length,contradictions: result.contradictions.length,diff --git a/src/functions/governance.ts b/src/functions/governance.tsindex 60813dd4..16d0d1c4 100644--- a/src/functions/governance.ts+++ b/src/functions/governance.ts@@ -4,6 +4,7 @@ import { KV } from "../state/schema.js";import type { StateKV } from "../state/kv.js";import { recordAudit, safeAudit, queryAudit } from "./audit.js";import { deleteAccessLog } from "./access-tracker.js";+import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js";import { logger } from "../logger.js";export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {@@ -23,10 +24,14 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {if (mem) {await kv.delete(KV.memories, id);await deleteAccessLog(kv, id);+ getSearchIndex().remove(id);+ vectorIndexRemove(id);deleted++;}}+ if (deleted > 0) await flushIndexSave();+await recordAudit(kv,"delete",@@ -107,6 +112,8 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {batch.map(async (mem) => {await kv.delete(KV.memories, mem.id);await deleteAccessLog(kv, mem.id);+ getSearchIndex().remove(mem.id);+ vectorIndexRemove(mem.id);}),);results.forEach((result, j) => {@@ -129,6 +136,8 @@ export function registerGovernanceFunction(sdk: ISdk, kv: StateKV): void {});}+ if (successfulIds.length > 0) await flushIndexSave();+await safeAudit(kv,"delete",diff --git a/src/functions/remember.ts b/src/functions/remember.tsindex 8d667705..30199ecd 100644--- a/src/functions/remember.ts+++ b/src/functions/remember.ts@@ -6,7 +6,7 @@ import { withKeyedLock } from "../state/keyed-mutex.js";import { memoryToObservation } from "../state/memory-utils.js";import { deleteAccessLog } from "./access-tracker.js";import { recordAudit } from "./audit.js";-import { getSearchIndex, vectorIndexAddGuarded } from "./search.js";+import { getSearchIndex, vectorIndexAddGuarded, vectorIndexRemove, flushIndexSave } from "./search.js";import { logger } from "../logger.js";export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {@@ -157,6 +157,8 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {await decrementImageRef(kv, sdk, mem.imageRef);}await deleteAccessLog(kv, data.memoryId);+ getSearchIndex().remove(data.memoryId);+ vectorIndexRemove(data.memoryId);deletedMemoryIds.push(data.memoryId);deleted++;}@@ -176,6 +178,8 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {if (obs?.imageRef && obs.imageRef !== obs.imageData) {await decrementImageRef(kv, sdk, obs.imageRef);}+ getSearchIndex().remove(obsId);+ vectorIndexRemove(obsId);deletedObservationIds.push(obsId);deleted++;}@@ -195,6 +199,8 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {if (obs.imageRef && obs.imageRef !== obs.imageData) {await decrementImageRef(kv, sdk, obs.imageRef);}+ getSearchIndex().remove(obs.id);+ vectorIndexRemove(obs.id);deletedObservationIds.push(obs.id);deleted++;}@@ -205,6 +211,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {}if (deleted > 0) {+ await flushIndexSave();await recordAudit(kv,"forget",diff --git a/src/functions/retention.ts b/src/functions/retention.tsindex c9c16573..fd3e6466 100644--- a/src/functions/retention.ts+++ b/src/functions/retention.ts@@ -14,6 +14,7 @@ import {normalizeAccessLog,} from "./access-tracker.js";import { recordAudit } from "./audit.js";+import { getSearchIndex, vectorIndexRemove, flushIndexSave } from "./search.js";import { logger } from "../logger.js";const DEFAULT_DECAY: DecayConfig = {@@ -370,6 +371,8 @@ export function registerRetentionFunctions(await kv.delete(scope, candidate.memoryId);await kv.delete(KV.retentionScores, candidate.memoryId);await deleteAccessLog(kv, candidate.memoryId);+ getSearchIndex().remove(candidate.memoryId);+ vectorIndexRemove(candidate.memoryId);evicted++;evictedIds.push(candidate.memoryId);if (resolvedSource === "semantic") evictedSemantic++;@@ -386,6 +389,7 @@ export function registerRetentionFunctions(// one record per invocation — per-candidate audits would flood// the audit log during normal eviction sweeps.if (evicted > 0) {+ await flushIndexSave();await recordAudit(kv, "delete", "mem::retention-evict", evictedIds, {threshold,evicted,diff --git a/src/functions/search.ts b/src/functions/search.tsindex b4444b48..dc629d6d 100644--- a/src/functions/search.ts+++ b/src/functions/search.ts@@ -34,6 +34,44 @@ export function getEmbeddingProvider(): EmbeddingProvider | null {return currentEmbeddingProvider}+export function vectorIndexRemove(id: string): void {+ vectorIndex?.remove(id);+}++// Persistence sync hook. Without this, index removals only live in+// memory; a crash/SIGKILL before graceful shutdown reloads a stale+// snapshot at boot and the deleted entry resurrects in the index.+// Wired by src/index.ts after IndexPersistence is constructed; no-op+// until then so unit tests that exercise the delete paths in+// isolation don't need to wire persistence.+let indexPersistence: {+ scheduleSave: () => void;+ save: () => Promise<void>;+} | null = null;++export function setIndexPersistence(+ p: { scheduleSave: () => void; save: () => Promise<void> } | null,+): void {+ indexPersistence = p;+}++export function scheduleIndexSave(): void {+ indexPersistence?.scheduleSave();+}++// Synchronous flush variant for delete paths. The debounced+// scheduleSave is fine for adds (chatty), but a hard process exit+// inside the 5s debounce window would lose deletes and resurrect+// removed entries on next boot. Deletes are infrequent enough that+// awaiting a single write per operation is acceptable. save() catches+// its own errors via IndexPersistence.logFailure, so this resolves+// even when persistence fails — callers must not treat a failed+// flush as a fatal error on the delete itself (the KV delete already+// committed before this is invoked).+export async function flushIndexSave(): Promise<void> {+ await indexPersistence?.save();+}+// Hard cap on embedding input length. Most providers cap input around// 8k tokens (~32k chars at ~4 chars/token). Truncate defensively so a// huge memory.content can't 400 the embed call or blow context budgetdiff --git a/src/index.ts b/src/index.tsindex 704d4809..d02b3dc7 100644--- a/src/index.ts+++ b/src/index.ts@@ -37,6 +37,7 @@ import {getSearchIndex,setVectorIndex,setEmbeddingProvider,+ setIndexPersistence,} from "./functions/search.js";import { registerContextFunction } from "./functions/context.js";import { registerSummarizeFunction } from "./functions/summarize.js";@@ -344,6 +345,11 @@ async function main() {const healthMonitor = registerHealthMonitor(sdk, kv);const indexPersistence = new IndexPersistence(kv, bm25Index, vectorIndex);+ // Wire the persistence hook so delete paths can flush BM25/vector+ // index mutations to disk. Without this, an in-memory remove can be+ // lost across a hard process exit and the persisted snapshot+ // restores the deleted entry at next boot.+ setIndexPersistence(indexPersistence);const loaded = await indexPersistence.load().catch((err) => {console.warn(`[agentmemory] Failed to load persisted index:`, err);diff --git a/src/state/search-index.ts b/src/state/search-index.tsindex f2530078..67f3e2ef 100644--- a/src/state/search-index.ts+++ b/src/state/search-index.ts@@ -51,6 +51,29 @@ export class SearchIndex {return this.entries.has(id);}+ remove(id: string): void {+ const entry = this.entries.get(id);+ if (!entry) return;++ const termFreq = this.docTermCounts.get(id);+ if (termFreq) {+ for (const term of termFreq.keys()) {+ const postingList = this.invertedIndex.get(term);+ if (postingList) {+ postingList.delete(id);+ if (postingList.size === 0) {+ this.invertedIndex.delete(term);+ }+ }+ }+ this.docTermCounts.delete(id);+ }++ this.totalDocLength = Math.max(0, this.totalDocLength - entry.termCount);+ this.entries.delete(id);+ this.sortedTerms = null;+ }+search(query: string,limit = 20,diff --git a/test/auto-forget.test.ts b/test/auto-forget.test.tsindex 367e6c59..0f104eb3 100644--- a/test/auto-forget.test.ts+++ b/test/auto-forget.test.ts@@ -1,10 +1,15 @@-import { describe, it, expect, beforeEach, vi } from "vitest";+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";vi.mock("../src/logger.js", () => ({logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));import { registerAutoForgetFunction } from "../src/functions/auto-forget.js";+import {+ getSearchIndex,+ setIndexPersistence,+} from "../src/functions/search.js";+import { memoryToObservation } from "../src/state/memory-utils.js";import type { Memory, CompressedObservation, Session } from "../src/types.js";function mockKV() {@@ -168,6 +173,85 @@ describe("Auto-Forget Function", () => {expect(stillExists).not.toBeNull();});+ describe("search-index cleanup", () => {+ beforeEach(() => {+ getSearchIndex().clear();+ setIndexPersistence(null);+ });++ afterEach(() => {+ setIndexPersistence(null);+ });++ it("removes TTL-expired memories from the BM25 index and flushes persistence", async () => {+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };+ setIndexPersistence(persistence);++ const expired = makeMemory({+ id: "mem_expired",+ forgetAfter: "2020-01-01T00:00:00Z",+ });+ await kv.set("mem:memories", "mem_expired", expired);+ getSearchIndex().add(memoryToObservation(expired));+ expect(getSearchIndex().has("mem_expired")).toBe(true);++ await sdk.trigger("mem::auto-forget", {});++ expect(getSearchIndex().has("mem_expired")).toBe(false);+ expect(persistence.save).toHaveBeenCalled();+ });++ it("removes evicted low-value observations from the BM25 index", async () => {+ const session: Session = {+ id: "ses_1",+ project: "my-project",+ cwd: "/tmp",+ startedAt: "2025-01-01T00:00:00Z",+ status: "completed",+ observationCount: 1,+ };+ await kv.set("mem:sessions", "ses_1", session);++ const oldLowObs: CompressedObservation = {+ id: "obs_old",+ sessionId: "ses_1",+ timestamp: "2025-01-01T00:00:00Z",+ type: "other",+ title: "trivial event",+ facts: [],+ narrative: "nothing important",+ concepts: [],+ files: [],+ importance: 1,+ };+ await kv.set("mem:obs:ses_1", "obs_old", oldLowObs);+ getSearchIndex().add(oldLowObs);+ expect(getSearchIndex().has("obs_old")).toBe(true);++ await sdk.trigger("mem::auto-forget", {});++ expect(getSearchIndex().has("obs_old")).toBe(false);+ });++ it("does not flush persistence on dryRun", async () => {+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };+ setIndexPersistence(persistence);++ const expired = makeMemory({+ id: "mem_expired",+ forgetAfter: "2020-01-01T00:00:00Z",+ });+ await kv.set("mem:memories", "mem_expired", expired);+ getSearchIndex().add(memoryToObservation(expired));++ await sdk.trigger("mem::auto-forget", { dryRun: true });++ // dryRun must not mutate the index or write to disk.+ expect(getSearchIndex().has("mem_expired")).toBe(true);+ expect(persistence.save).not.toHaveBeenCalled();+ });+ });+it("does not flag non-similar memories as contradictions", async () => {const mem1 = makeMemory({id: "mem_1",diff --git a/test/governance.test.ts b/test/governance.test.tsindex 22554164..6390bbee 100644--- a/test/governance.test.ts+++ b/test/governance.test.ts@@ -1,10 +1,15 @@-import { describe, it, expect, beforeEach, vi } from "vitest";+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";vi.mock("../src/logger.js", () => ({logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));import { registerGovernanceFunction } from "../src/functions/governance.js";+import {+ getSearchIndex,+ setIndexPersistence,+} from "../src/functions/search.js";+import { memoryToObservation } from "../src/state/memory-utils.js";import type { Memory, AuditEntry } from "../src/types.js";function mockKV() {@@ -133,6 +138,109 @@ describe("Governance Functions", () => {expect(remaining.length).toBe(3);});+ // Delete paths must tear down the BM25 index entry and trigger an+ // IndexPersistence save so a hard process exit can't restore a stale+ // snapshot at next boot.+ describe("search index cleanup on delete", () => {+ function indexedObs(id: string, title: string) {+ return memoryToObservation({+ id,+ createdAt: "2026-02-01T00:00:00Z",+ updatedAt: "2026-02-01T00:00:00Z",+ type: "fact",+ title,+ content: title,+ concepts: [],+ files: [],+ sessionIds: ["ses_1"],+ strength: 5,+ version: 1,+ isLatest: true,+ });+ }++ function mockPersistence() {+ return {+ scheduleSave: vi.fn(),+ save: vi.fn(async () => {}),+ };+ }++ beforeEach(() => {+ // SearchIndex is a module-level singleton — wipe it so cases+ // don't bleed into each other.+ getSearchIndex().clear();+ setIndexPersistence(null);+ });++ // The persistence singleton is module-scoped; without this reset+ // the last test's mock would leak into sibling tests in the outer+ // suite and trigger unexpected spy invocations.+ afterEach(() => {+ setIndexPersistence(null);+ });++ it("governance-delete removes the memory from the search index", async () => {+ getSearchIndex().add(indexedObs("mem_1", "alpha"));+ getSearchIndex().add(indexedObs("mem_2", "beta"));+ expect(getSearchIndex().has("mem_1")).toBe(true);++ await sdk.trigger("mem::governance-delete", {+ memoryIds: ["mem_1"],+ });++ expect(getSearchIndex().has("mem_1")).toBe(false);+ expect(getSearchIndex().has("mem_2")).toBe(true);+ });++ it("governance-delete flushes persistence immediately", async () => {+ const persistence = mockPersistence();+ setIndexPersistence(persistence);+ getSearchIndex().add(indexedObs("mem_1", "alpha"));++ await sdk.trigger("mem::governance-delete", { memoryIds: ["mem_1"] });++ // Delete paths must use the synchronous save (not the debounced+ // scheduleSave) so a process exit immediately after delete can't+ // resurrect the entry on next boot.+ expect(persistence.save).toHaveBeenCalled();+ });++ it("governance-delete skips persistence flush when nothing was deleted", async () => {+ const persistence = mockPersistence();+ setIndexPersistence(persistence);++ await sdk.trigger("mem::governance-delete", {+ memoryIds: ["nonexistent_999"],+ });++ expect(persistence.save).not.toHaveBeenCalled();+ });++ it("governance-bulk removes deleted memories from the search index", async () => {+ getSearchIndex().add(indexedObs("mem_1", "alpha"));+ getSearchIndex().add(indexedObs("mem_2", "beta"));+ getSearchIndex().add(indexedObs("mem_3", "gamma"));++ await sdk.trigger("mem::governance-bulk", { type: ["pattern"] });++ // mem_1 and mem_3 are type "pattern" per the outer beforeEach.+ expect(getSearchIndex().has("mem_1")).toBe(false);+ expect(getSearchIndex().has("mem_3")).toBe(false);+ expect(getSearchIndex().has("mem_2")).toBe(true);+ });++ it("governance-bulk flushes persistence immediately", async () => {+ const persistence = mockPersistence();+ setIndexPersistence(persistence);+ getSearchIndex().add(indexedObs("mem_1", "alpha"));++ await sdk.trigger("mem::governance-bulk", { type: ["pattern"] });++ expect(persistence.save).toHaveBeenCalled();+ });+ });+it("audit-query returns audit entries", async () => {await sdk.trigger("mem::governance-delete", {memoryIds: ["mem_1"],diff --git a/test/remember-forget-audit.test.ts b/test/remember-forget-audit.test.tsindex 7be571fa..7d17b543 100644--- a/test/remember-forget-audit.test.ts+++ b/test/remember-forget-audit.test.ts@@ -1,4 +1,4 @@-import { describe, it, expect, vi } from "vitest";+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";vi.mock("../src/logger.js", () => ({logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },@@ -9,6 +9,12 @@ vi.mock("../src/state/keyed-mutex.js", () => ({}));import { registerRememberFunction } from "../src/functions/remember.js";+import {+ getSearchIndex,+ setIndexPersistence,+} from "../src/functions/search.js";+import { memoryToObservation } from "../src/state/memory-utils.js";+import type { Memory } from "../src/types.js";function mockKV() {const store = new Map<string, Map<string, unknown>>();@@ -117,3 +123,89 @@ describe("mem::forget audit coverage (issue #125)", () => {expect(auditRows).toHaveLength(0);});});++// Delete paths must tear down the BM25 index entry and synchronously+// flush the persisted snapshot. Without this, a deleted memory keeps+// occupying limit-capped search result slots, and an in-memory remove+// would be lost if the process exits before the debounced save fires.+describe("mem::forget search-index cleanup", () => {+ function makeMemory(id: string): Memory {+ return {+ id,+ createdAt: "2026-02-01T00:00:00Z",+ updatedAt: "2026-02-01T00:00:00Z",+ type: "fact",+ title: `title ${id}`,+ content: `content ${id}`,+ concepts: [],+ files: [],+ sessionIds: ["ses_1"],+ strength: 5,+ version: 1,+ isLatest: true,+ };+ }++ beforeEach(() => {+ getSearchIndex().clear();+ setIndexPersistence(null);+ });++ afterEach(() => {+ setIndexPersistence(null);+ });++ it("removes a forgotten memory from the BM25 index", async () => {+ const sdk = mockSdk();+ const kv = mockKV();+ registerRememberFunction(sdk as never, kv as never);++ const mem = makeMemory("mem_a");+ await kv.set("mem:memories", mem.id, mem);+ getSearchIndex().add(memoryToObservation(mem));+ expect(getSearchIndex().has("mem_a")).toBe(true);++ await sdk.trigger({+ function_id: "mem::forget",+ payload: { memoryId: "mem_a" },+ });++ expect(getSearchIndex().has("mem_a")).toBe(false);+ });++ it("removes forgotten observations from the BM25 index", async () => {+ const sdk = mockSdk();+ const kv = mockKV();+ registerRememberFunction(sdk as never, kv as never);++ await kv.set("mem:obs:ses_1", "obs_a", { id: "obs_a" });+ await kv.set("mem:obs:ses_1", "obs_b", { id: "obs_b" });+ getSearchIndex().add(memoryToObservation(makeMemory("obs_a")));+ getSearchIndex().add(memoryToObservation(makeMemory("obs_b")));++ await sdk.trigger({+ function_id: "mem::forget",+ payload: { sessionId: "ses_1", observationIds: ["obs_a"] },+ });++ expect(getSearchIndex().has("obs_a")).toBe(false);+ expect(getSearchIndex().has("obs_b")).toBe(true);+ });++ it("flushes persistence immediately when a memory is forgotten", async () => {+ const sdk = mockSdk();+ const kv = mockKV();+ registerRememberFunction(sdk as never, kv as never);+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };+ setIndexPersistence(persistence);++ await kv.set("mem:memories", "mem_a", makeMemory("mem_a"));++ await sdk.trigger({+ function_id: "mem::forget",+ payload: { memoryId: "mem_a" },+ });++ expect(persistence.save).toHaveBeenCalled();+ });+});diff --git a/test/retention.test.ts b/test/retention.test.tsindex 8acfda6b..cf8cc681 100644--- a/test/retention.test.ts+++ b/test/retention.test.ts@@ -1,4 +1,9 @@-import { describe, it, expect, vi } from "vitest";+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";+import {+ getSearchIndex,+ setIndexPersistence,+} from "../src/functions/search.js";+import { memoryToObservation } from "../src/state/memory-utils.js";import type { Memory, SemanticMemory } from "../src/types.js";vi.mock("../src/logger.js", () => ({@@ -503,4 +508,59 @@ describe("RetentionScoring", () => {const remaining = await kv.list("mem:memories");expect(remaining).toHaveLength(0);});++ describe("search-index cleanup on eviction", () => {+ beforeEach(() => {+ getSearchIndex().clear();+ setIndexPersistence(null);+ });++ afterEach(() => {+ setIndexPersistence(null);+ });++ it("removes evicted memories from the BM25 index and flushes persistence", async () => {+ const { registerRetentionFunctions } = await import(+ "../src/functions/retention.js"+ );+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };+ setIndexPersistence(persistence);++ const evictee = makeMemory("mem_evict", "fact", 500);+ const sdk = mockSdk();+ const kv = mockKV([evictee]);+ registerRetentionFunctions(sdk as never, kv as never);+ getSearchIndex().add(memoryToObservation(evictee));+ expect(getSearchIndex().has("mem_evict")).toBe(true);++ await sdk.trigger({ function_id: "mem::retention-score", payload: {} });+ await sdk.trigger({+ function_id: "mem::retention-evict",+ payload: { threshold: 0.9 },+ });++ expect(getSearchIndex().has("mem_evict")).toBe(false);+ expect(persistence.save).toHaveBeenCalled();+ });++ it("does not flush persistence when nothing is evicted", async () => {+ const { registerRetentionFunctions } = await import(+ "../src/functions/retention.js"+ );+ const persistence = { scheduleSave: vi.fn(), save: vi.fn(async () => {}) };+ setIndexPersistence(persistence);++ const sdk = mockSdk();+ const kv = mockKV([makeMemory("mem_fresh", "architecture", 1)]);+ registerRetentionFunctions(sdk as never, kv as never);++ await sdk.trigger({ function_id: "mem::retention-score", payload: {} });+ await sdk.trigger({+ function_id: "mem::retention-evict",+ payload: { threshold: 0.01 },+ });++ expect(persistence.save).not.toHaveBeenCalled();+ });+ });});diff --git a/test/search-index.test.ts b/test/search-index.test.tsindex f1349873..805faec4 100644--- a/test/search-index.test.ts+++ b/test/search-index.test.ts@@ -89,6 +89,81 @@ describe("SearchIndex", () => {expect(index.search("auth")).toEqual([]);});+ // Regression coverage: deleted docs must not keep occupying result+ // slots after remove(). Without remove(), a limit-capped search can+ // return fewer live results than requested because the slot is held+ // by a doc that no longer exists in storage.+ describe("remove", () => {+ it("drops a removed doc from search results", () => {+ index.add(makeObs({ id: "obs_a", title: "jose for JWT" }));+ index.add(+ makeObs({ id: "obs_b", title: "JWT tokens expire after 30 minutes" }),+ );+ expect(index.search("jose jwt", 1)[0].obsId).toBe("obs_a");++ index.remove("obs_a");++ // With limit=1, the survivor must now surface instead of getting+ // masked by the deleted doc's slot.+ const after = index.search("jose jwt", 1);+ expect(after).toHaveLength(1);+ expect(after[0].obsId).toBe("obs_b");+ expect(index.size).toBe(1);+ });++ it("is a no-op for unknown ids", () => {+ index.add(makeObs({ id: "obs_a" }));+ expect(() => index.remove("does_not_exist")).not.toThrow();+ expect(index.size).toBe(1);+ expect(index.search("auth")).toHaveLength(1);+ });++ it("cleans up empty posting lists and the prefix cache", () => {+ // unique_concept_xyz appears only in obs_a. After removing obs_a,+ // a prefix search for "unique_" must NOT surface it from the+ // sortedTerms cache.+ index.add(+ makeObs({+ id: "obs_a",+ concepts: ["unique_concept_xyz"],+ }),+ );+ index.add(makeObs({ id: "obs_b", title: "completely different" }));+ // Prime the sortedTerms cache.+ expect(index.search("unique_")).toHaveLength(1);++ index.remove("obs_a");++ expect(index.search("unique_concept_xyz")).toEqual([]);+ expect(index.search("unique_")).toEqual([]);+ });++ it("keeps scoring consistent after add/remove/add cycles", () => {+ // totalDocLength bookkeeping desync would skew BM25 normalization+ // (docLen / avgDocLen). Round-trip add → remove → re-add and+ // confirm size + retrievability hold.+ const obs = makeObs({ id: "obs_a" });+ index.add(obs);+ expect(index.size).toBe(1);+ index.remove("obs_a");+ expect(index.size).toBe(0);+ index.add(obs);+ expect(index.size).toBe(1);+ expect(index.search("auth")[0].obsId).toBe("obs_a");+ });++ it("survives serialize/deserialize after removes", () => {+ index.add(makeObs({ id: "obs_a", title: "alpha doc" }));+ index.add(makeObs({ id: "obs_b", title: "beta doc" }));+ index.remove("obs_a");++ const restored = SearchIndex.deserialize(index.serialize());+ expect(restored.size).toBe(1);+ expect(restored.search("alpha")).toEqual([]);+ expect(restored.search("beta")).toHaveLength(1);+ });+ });+it("returns empty for empty query", () => {index.add(makeObs());expect(index.search("")).toEqual([]);