π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
fix(memory): build the RaBitQ index on demand and use it on the vector path (#4318)
Gssd7f228683521
diff --git a/plugins/ruflo-agentdb/commands/embeddings.md b/plugins/ruflo-agentdb/commands/embeddings.mdindex c2e80b3fb5..bc21460130 100644--- a/plugins/ruflo-agentdb/commands/embeddings.md+++ b/plugins/ruflo-agentdb/commands/embeddings.md@@ -10,7 +10,7 @@ Embedding engine commands:3. If not initialized, suggest calling `mcp__plugin_ruflo-core_ruflo__embeddings_init`.4. For search, use `mcp__plugin_ruflo-core_ruflo__embeddings_search` with the user's query (namespace-filtered).5. For large-corpus search with memory pressure, use the RaBitQ quantized path (32Γ memory reduction):- - `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_build` (one-time, after corpus is loaded)+ - the index builds itself on first use and rebuilds when the corpus drifts >20%; call `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_build` to force one now- `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_search` (Hamming-prefilter to top-N candidates)- `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_status` for index health6. For hierarchical data (taxonomies, code trees), use `mcp__plugin_ruflo-core_ruflo__embeddings_hyperbolic` (Poincare ball model).diff --git a/v3/@claude-flow/cli/__tests__/issue-4203-rabitq-auto-build.test.ts b/v3/@claude-flow/cli/__tests__/issue-4203-rabitq-auto-build.test.tsnew file mode 100644index 0000000000..5db35afe07--- /dev/null+++ b/v3/@claude-flow/cli/__tests__/issue-4203-rabitq-auto-build.test.ts@@ -0,0 +1,290 @@+import { describe, it, expect, beforeEach, vi } from 'vitest';+import { mkdtempSync, rmSync } from 'node:fs';+import { join } from 'node:path';+import { tmpdir } from 'node:os';++// #4203: the RaBitQ index was only ever populated by the explicit+// `embeddings_rabitq_build` tool β `shouldRebuildRabitq()` had no callers, so+// `searchRabitq()` always returned null and the sql.js pre-filter at+// memory-initializer.ts was dead. The bridge path+// (bridgeSearchBruteForceCosine) never consulted the index at all.+//+// These tests exercise the real store (better-sqlite3 is present in this env)+// and assert the two halves of the fix: the index is built on demand by the+// search path, and the bridge search actually consults it.++const DIMENSIONS = 384;+const ROW_COUNT = 12;++function vectorFor(seed: number): number[] {+ const out: number[] = [];+ for (let i = 0; i < DIMENSIONS; i++) {+ out.push(Math.sin(seed * 0.7 + i * 0.11) * Math.cos(seed * 1.3 + i * 0.03));+ }+ return out;+}++function cosine(a: number[], b: number[]): number {+ let dot = 0;+ let na = 0;+ let nb = 0;+ for (let i = 0; i < Math.min(a.length, b.length); i++) {+ dot += a[i] * b[i];+ na += a[i] * a[i];+ nb += b[i] * b[i];+ }+ const mag = Math.sqrt(na * nb);+ return mag === 0 ? 0 : dot / mag;+}++/** Deterministic pseudo-random vector, independent of the `vectorFor` family. */+function randomVectorFor(seed: number): number[] {+ let state = seed | 0;+ const next = () => {+ state = (state + 0x6d2b79f5) | 0;+ let t = Math.imul(state ^ (state >>> 15), 1 | state);+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;+ };+ const out: number[] = [];+ for (let i = 0; i < DIMENSIONS; i++) out.push(next() * 2 - 1);+ return out;+}++/** Seed a real store with `rowCount` embeddings and return its path. */+async function seedStore(+ root: string,+ rowCount = ROW_COUNT,+ vectorAt: (i: number) => number[] = vectorFor,+): Promise<string | null> {+ const dbPath = join(root, 'memory.db');+ const bridge = await import('../src/memory/memory-bridge.js');+ const { initializeMemoryDatabase } = await import('../src/memory/memory-initializer.js');++ await initializeMemoryDatabase({ dbPath, force: true, verbose: false });++ const status = await bridge.bridgeGetVectorSearchStatus(dbPath);+ if (!status) return null; // bridge unavailable in this environment++ for (let i = 0; i < rowCount; i++) {+ const id = `00000000-0000-4000-8000-${String(i).padStart(12, '0')}`;+ const written = await bridge.bridgeAddEmbedding(+ id,+ vectorAt(i),+ { id, key: `emb-${i}`, namespace: 'test', content: `row ${i}` },+ dbPath,+ );+ if (written !== true) return null;+ }++ return dbPath;+}++describe('#4203 RaBitQ index is built on demand instead of only via the manual tool', () => {+ // Each test gets a fresh module graph: the index state, the auto-build+ // throttle and the bridge registry cache are all module-level.+ beforeEach(() => {+ vi.resetModules();+ });++ it('ensureRabitqIndex() builds from stored embeddings and serves searches', async () => {+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-build-'));+ try {+ const dbPath = await seedStore(root);+ if (!dbPath) return; // bridge unavailable in this environment++ const rabitq = await import('../src/memory/rabitq-index.js');++ // Pre-fix: nothing ever built the index, so this was always uninitialized+ // no matter how many embeddings memory_store had written.+ expect(rabitq.getRabitqStatus().initialized).toBe(false);++ expect(await rabitq.ensureRabitqIndex({ dbPath, dimensions: DIMENSIONS })).toBe(true);+ expect(rabitq.getRabitqStatus().vectorCount).toBe(ROW_COUNT);++ // An index that is already current must not be rebuilt.+ expect(await rabitq.shouldRebuildRabitq(ROW_COUNT)).toBe(false);++ const hits = await rabitq.searchRabitq(vectorFor(0), { k: 3 });+ expect(hits).not.toBeNull();+ expect(hits![0].key).toBe('emb-0');+ expect(hits![0].namespace).toBe('test');+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+ });++ it('bridgeSearchBruteForceCosine() consults the RaBitQ index and reranks exactly', async () => {+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-bridge-'));+ try {+ const dbPath = await seedStore(root);+ if (!dbPath) return; // bridge unavailable in this environment++ const bridge = await import('../src/memory/memory-bridge.js');+ const rabitq = await import('../src/memory/rabitq-index.js');++ // Pre-fix: the bridge scanned the whole table and the index stayed empty.+ expect(rabitq.getRabitqStatus().initialized).toBe(false);++ const query = vectorFor(5);+ const results = await bridge.bridgeSearchBruteForceCosine(query, { k: 3 }, dbPath);++ expect(results).not.toBeNull();+ expect(results!.length).toBeGreaterThan(0);+ expect(results![0].key).toBe('emb-5');+ // Reranking uses exact cosine, so the score keeps its brute-force meaning.+ expect(results![0].score).toBeCloseTo(cosine(query, vectorFor(5)), 6);+ expect(results![0].id).toBe('00000000-000');+ expect(results![0].namespace).toBe('test');++ // The index the search used was built by that same search call.+ expect(rabitq.getRabitqStatus().initialized).toBe(true);+ expect(rabitq.getRabitqStatus().vectorCount).toBe(ROW_COUNT);+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+ });++ it('does not rebuild the index on every search', async () => {+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-throttle-'));+ try {+ const dbPath = await seedStore(root);+ if (!dbPath) return; // bridge unavailable in this environment++ const bridge = await import('../src/memory/memory-bridge.js');+ const rabitq = await import('../src/memory/rabitq-index.js');++ const first = await bridge.bridgeSearchBruteForceCosine(vectorFor(3), { k: 2 }, dbPath);+ expect(first![0].key).toBe('emb-3');++ const builtAt = rabitq.getRabitqStatus().builtAt;+ expect(builtAt).not.toBeNull();++ // A second search, plus an explicit ensure for the same store and width,+ // must reuse the index: rebuilding on every query would cost more than+ // the brute-force scan it replaces.+ const second = await bridge.bridgeSearchBruteForceCosine(vectorFor(3), { k: 2 }, dbPath);+ expect(second![0].key).toBe('emb-3');+ expect(await rabitq.ensureRabitqIndex({ dbPath, dimensions: DIMENSIONS })).toBe(true);+ expect(rabitq.getRabitqStatus().builtAt).toBe(builtAt);+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+ });++ // Negative control: the auto-build must not fire (and must not throw) when the+ // corpus is below the floor, and the caller's exhaustive scan must still be+ // the thing that answers.+ it('leaves the exhaustive scan in place when the corpus is below the build floor', async () => {+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-small-'));+ try {+ const dbPath = await seedStore(root, 5);+ if (!dbPath) return; // bridge unavailable in this environment++ const rabitq = await import('../src/memory/rabitq-index.js');+ const bridge = await import('../src/memory/memory-bridge.js');++ expect(await rabitq.ensureRabitqIndex({ dbPath, dimensions: DIMENSIONS })).toBe(false);+ expect(rabitq.getRabitqStatus().initialized).toBe(false);++ const results = await bridge.bridgeSearchBruteForceCosine(vectorFor(3), { k: 2 }, dbPath);+ expect(results).not.toBeNull();+ expect(results![0].key).toBe('emb-3');+ expect(results![0].score).toBeCloseTo(cosine(vectorFor(3), vectorFor(3)), 6);+ expect(rabitq.getRabitqStatus().initialized).toBe(false);+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+ });++ // Positive/negative control pair over the same corpus: the indexed path and+ // the exhaustive path must return the same rows with the same exact scores,+ // so switching RaBitQ on cannot silently change rankings.+ // Note: this corpus (12 rows) fits entirely inside the candidate window+ // (k * 2 * 3 = 30), so equality here is exact by construction. The recall+ // test below covers a corpus that is larger than the window.+ it('indexed search agrees with the exhaustive scan on the same corpus', async () => {+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-parity-'));+ try {+ const dbPath = await seedStore(root);+ if (!dbPath) return; // bridge unavailable in this environment++ // Negative control: a 128-dim probe cannot be served by a 384-dim store,+ // so the first search keeps the exhaustive scan (the retry window opened+ // by that probe also keeps it from building behind our back).+ const rabitqExhaustive = await import('../src/memory/rabitq-index.js');+ const bridgeExhaustive = await import('../src/memory/memory-bridge.js');+ expect(await rabitqExhaustive.ensureRabitqIndex({ dbPath, dimensions: 128 })).toBe(false);+ const exhaustive = await bridgeExhaustive.bridgeSearchBruteForceCosine(vectorFor(7), { k: 5 }, dbPath);+ expect(exhaustive).not.toBeNull();+ expect(exhaustive!.length).toBeGreaterThan(0);+ expect(rabitqExhaustive.getRabitqStatus().initialized).toBe(false);++ // Positive control: a fresh module graph has no retry window, so the same+ // query is served from the index that this search builds.+ vi.resetModules();+ const rabitqIndexed = await import('../src/memory/rabitq-index.js');+ const bridgeIndexed = await import('../src/memory/memory-bridge.js');+ const indexed = await bridgeIndexed.bridgeSearchBruteForceCosine(vectorFor(7), { k: 5 }, dbPath);+ expect(indexed).not.toBeNull();+ expect(rabitqIndexed.getRabitqStatus().initialized).toBe(true);++ expect(indexed!.map(r => r.key)).toEqual(exhaustive!.map(r => r.key));+ expect(indexed!.map(r => r.id)).toEqual(exhaustive!.map(r => r.id));+ for (let i = 0; i < indexed!.length; i++) {+ expect(indexed![i].score).toBeCloseTo(exhaustive![i].score, 6);+ expect(indexed![i].namespace).toBe(exhaustive![i].namespace);+ }+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+ });++ // Recall/rerank check on a corpus that is larger than the candidate window+ // (60 rows, k = 5 β 30 candidates), where the two paths may legitimately rank+ // differently. Two things must still hold: every returned score is that row's+ // exact cosine (the rerank is not an approximation), and top-k recall does not+ // collapse because of the pre-filter.+ it('reranks with exact cosine and keeps top-k recall above the window size', async () => {+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-recall-'));+ try {+ const query = randomVectorFor(999);+ const dbPath = await seedStore(root, 60, i => randomVectorFor(i + 1));+ if (!dbPath) return; // bridge unavailable in this environment++ // Same negative-then-positive control: the first search keeps the+ // exhaustive scan, the second (fresh module graph) uses the index.+ const rabitqExhaustive = await import('../src/memory/rabitq-index.js');+ const bridgeExhaustive = await import('../src/memory/memory-bridge.js');+ expect(await rabitqExhaustive.ensureRabitqIndex({ dbPath, dimensions: 128 })).toBe(false);+ const exhaustive = await bridgeExhaustive.bridgeSearchBruteForceCosine(+ query, { k: 5, threshold: -1 }, dbPath,+ );+ expect(exhaustive).not.toBeNull();+ expect(exhaustive!.length).toBe(5);++ vi.resetModules();+ const rabitqIndexed = await import('../src/memory/rabitq-index.js');+ const bridgeIndexed = await import('../src/memory/memory-bridge.js');+ const indexed = await bridgeIndexed.bridgeSearchBruteForceCosine(+ query, { k: 5, threshold: -1 }, dbPath,+ );+ expect(rabitqIndexed.getRabitqStatus().initialized).toBe(true);+ expect(indexed).not.toBeNull();++ // Rerank correctness: each score equals that row's exact cosine.+ for (const row of indexed!) {+ const seed = Number(row.key.replace('emb-', '')) + 1;+ expect(row.score).toBeCloseTo(cosine(query, randomVectorFor(seed)), 6);+ }++ // Recall: β₯80% of the exhaustive top-5 must survive the pre-filter.+ const indexedKeys = new Set(indexed!.map(r => r.key));+ const recovered = exhaustive!.filter(r => indexedKeys.has(r.key)).length;+ expect(recovered).toBeGreaterThanOrEqual(4);+ expect(indexed![0].key).toBe(exhaustive![0].key);+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+ });+});diff --git a/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts b/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.tsindex 28f27f0d18..6e3bd0696a 100644--- a/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts+++ b/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts@@ -1022,17 +1022,25 @@ export const embeddingsTools: MCPTool[] = [const v = vt(params.query as string, 'query');if (!v.valid) return { success: false, error: v.error };- const { searchRabitq } = await import('../memory/rabitq-index.js');+ const { ensureRabitqIndex, searchRabitq } = await import('../memory/rabitq-index.js');const { generateEmbedding } = await import('../memory/memory-initializer.js');const queryEmb = await generateEmbedding(params.query as string);- const results = await searchRabitq(queryEmb.embedding, {+ const searchOptions = {k: (params.k as number) || 10,namespace: params.namespace as string,- });+ };++ // #4203: the index is built on demand, so a search no longer requires a+ // separate `embeddings_rabitq_build` call first.+ let results = await searchRabitq(queryEmb.embedding, searchOptions);+ if (!results) {+ await ensureRabitqIndex({ dimensions: queryEmb.embedding.length });+ results = await searchRabitq(queryEmb.embedding, searchOptions);+ }if (!results) {- return { success: false, error: 'RaBitQ index not built. Call embeddings_rabitq_build first.' };+ return { success: false, error: 'RaBitQ index not built (needs at least 10 embedded entries). Call embeddings_rabitq_build first.' };}return {diff --git a/v3/@claude-flow/cli/src/memory/memory-bridge.ts b/v3/@claude-flow/cli/src/memory/memory-bridge.tsindex de68248cd7..3c01f27420 100644--- a/v3/@claude-flow/cli/src/memory/memory-bridge.ts+++ b/v3/@claude-flow/cli/src/memory/memory-bridge.ts@@ -2113,10 +2113,95 @@ export async function bridgeGetVectorSearchStatus(}}+/**+ * #4203: RaBitQ pre-filter for the bridge's vector search.+ *+ * The index yields candidate ids from a 32Γ-compressed Hamming scan; each is+ * reranked here with the same exact cosine similarity the exhaustive scan uses,+ * so scores keep their meaning and only the scanned set shrinks (the candidate+ * window `k * 2` is then widened 3Γ by `searchRabitq` itself).+ *+ * Returns null whenever the index cannot answer this query β WASM missing, too+ * few vectors, or an index built for another store/dimension β so the caller+ * runs its unchanged brute-force scan. Keyword/BM25 recall is unaffected: this+ * only ever replaces the cosine scan.+ */+async function rabitqSearchAndRerank(+ db: any,+ queryEmbedding: number[],+ options: { k: number; threshold: number; namespace?: string; dbPath?: string },+): Promise<Array<{ id: string; key: string; content: string; score: number; namespace: string }> | null> {+ try {+ const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');++ const ready = await ensureRabitqIndex({ dbPath: options.dbPath, dimensions: queryEmbedding.length });+ if (!ready) return null;++ const candidates = await searchRabitq(queryEmbedding, {+ k: options.k * 2,+ namespace: options.namespace,+ });+ if (!candidates || candidates.length === 0) return null;++ // Fetch exactly the candidates (id is the primary key) instead of the+ // 10k-row full scan; chunked to stay well inside SQLite's parameter limit.+ const rowsById = new Map<string, { key: string; namespace: string; content: string; score: number }>();+ const ids = candidates.map(c => c.id);+ for (let i = 0; i < ids.length; i += 500) {+ const chunk = ids.slice(i, i + 500);+ const placeholders = chunk.map(() => '?').join(',');+ const rows: any[] = db.prepare(`+ SELECT id, key, namespace, content, embedding+ FROM memory_entries+ WHERE id IN (${placeholders}) AND ${liveMemoryRowSql()} AND embedding IS NOT NULL+ `).all(...chunk);++ for (const row of rows) {+ let embedding: number[];+ try {+ embedding = JSON.parse(row.embedding) as number[];+ } catch {+ continue;+ }+ if (!Array.isArray(embedding) || embedding.length !== queryEmbedding.length) continue;+ rowsById.set(String(row.id), {+ key: row.key || String(row.id),+ namespace: row.namespace || 'default',+ content: row.content || '',+ score: cosineSim(queryEmbedding, embedding),+ });+ }+ }++ const results: Array<{ id: string; key: string; content: string; score: number; namespace: string }> = [];+ for (const candidate of candidates) {+ const row = rowsById.get(candidate.id);+ if (!row || row.score < options.threshold) continue;+ results.push({+ id: candidate.id.substring(0, 12),+ key: row.key || candidate.id.substring(0, 15),+ content: row.content.substring(0, 60) + (row.content.length > 60 ? '...' : ''),+ score: row.score,+ namespace: row.namespace,+ });+ }++ if (results.length === 0) return null;+ results.sort((a, b) => b.score - a.score);+ return results.slice(0, options.k);+ } catch {+ return null;+ }+}+/*** Search AgentDB v3's embedder + SQLite entries via a full-table scan and* brute-force cosine similarity. NOT HNSW-accelerated β see #2922. Returns* null if unavailable.+ *+ * #4203: when a RaBitQ index is available the Hamming-scan candidate set is+ * reranked exactly instead of scanning the whole table; the exhaustive loop+ * below stays the fallback, so results and scores keep their meaning.*/export async function bridgeSearchBruteForceCosine(queryEmbedding: number[],@@ -2143,6 +2228,17 @@ export async function bridgeSearchBruteForceCosine(? `AND namespace = ?`: '';+ // #4203: try the RaBitQ pre-filter before the exhaustive scan below. A null+ // result (no index / unavailable / nothing above threshold) falls through+ // to the unchanged brute-force path.+ const accelerated = await rabitqSearchAndRerank(ctx.db, queryEmbedding, {+ k,+ threshold,+ namespace: options?.namespace,+ dbPath,+ });+ if (accelerated) return accelerated;+let rows: any[];try {// #2982/#2976: same unordered pre-rank truncation as bridgeSearchEntries@@ -3772,6 +3868,47 @@ export async function bridgeGetAllEmbeddings(options?: {}}+/**+ * Count the embeddings `bridgeGetAllEmbeddings()` would export (#4203).+ * RaBitQ's auto-build uses this as its cheap "has the corpus drifted?" probe β+ * a COUNT instead of re-reading every embedding blob. With `dimensions` it+ * counts only rows build() would keep, so the drift ratio compares like with+ * like. Returns null if the bridge is unavailable.+ */+export async function bridgeCountEmbeddings(options?: {+ dimensions?: number;+ dbPath?: string;+}): Promise<number | null> {+ if (!operationContext.getStore()?.active) return withBridgeOperation(() => bridgeCountEmbeddings(options));+ const registry = await getRegistry(options?.dbPath);+ if (!registry) return null;++ const ctx = getDb(registry);+ if (!ctx) return null;++ const dims = options?.dimensions;+ try {+ if (dims != null) {+ try {+ const row = ctx.db.prepare(+ `SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL AND json_array_length(embedding) = ?`,+ ).get(dims);+ return row?.cnt ?? 0;+ } catch {+ // JSON1 unavailable in this SQLite build β fall through to the+ // unfiltered count rather than losing the probe entirely.+ }+ }++ const row = ctx.db.prepare(+ `SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL`,+ ).get();+ return row?.cnt ?? 0;+ } catch {+ return null;+ }+}+// ===== Utility =====function cosineSim(a: number[], b: number[]): number {diff --git a/v3/@claude-flow/cli/src/memory/memory-initializer.ts b/v3/@claude-flow/cli/src/memory/memory-initializer.tsindex 04c9870a99..ba016a2895 100644--- a/v3/@claude-flow/cli/src/memory/memory-initializer.ts+++ b/v3/@claude-flow/cli/src/memory/memory-initializer.ts@@ -3270,7 +3270,11 @@ export async function searchEntries(options: {// Try RaBitQ pre-filter first (32Γ compressed Hamming scan)try {- const { searchRabitq } = await import('./rabitq-index.js');+ const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');+ // #4203: build/rebuild on demand β without this the index was only ever+ // populated by the explicit `embeddings_rabitq_build` tool, so this+ // pre-filter always missed.+ await ensureRabitqIndex({ dbPath, dimensions: queryEmbedding.length });const rabitqCandidates = await searchRabitq(queryEmbedding, { k: limit * 2, namespace: effectiveNamespace });if (rabitqCandidates && rabitqCandidates.length > 0) {// Rerank candidates with exact cosine similarity from SQLitediff --git a/v3/@claude-flow/cli/src/memory/rabitq-index.ts b/v3/@claude-flow/cli/src/memory/rabitq-index.tsindex 9ea096b11b..55d41cc6c2 100644--- a/v3/@claude-flow/cli/src/memory/rabitq-index.ts+++ b/v3/@claude-flow/cli/src/memory/rabitq-index.ts@@ -9,6 +9,11 @@* 1. build() β bulk-load all embeddings from SQLite into the WASM index* 2. search() β fast Hamming scan β candidate ids β caller reranks* 3. rebuild() β called when entry count drifts >20% from last build+ *+ * #4203: step 3 is driven by `ensureRabitqIndex()`, which the search paths+ * call before `searchRabitq()`. Previously the only caller of the build was+ * the explicit `embeddings_rabitq_build` tool, so the index was effectively+ * never populated and every `searchRabitq()` returned null.*/import * as fs from 'fs';@@ -26,14 +31,26 @@ interface RabitqState {dimensions: number;builtAt: number;vectorCount: number;+ /** Resolved source database the index was built from (#4203). An index built+ * from one store must not answer a query aimed at another one β same reason+ * the bridge registry cache is keyed by path (#3196). */+ sourcePath?: string;}const RABITQ_SEED = 42n;const RABITQ_RERANK_FACTOR = 20;const REBUILD_DRIFT_THRESHOLD = 0.2; // rebuild when count drifts >20%+/** #4203: minimum gap between auto-build attempts, so a store that is still+ * too small cannot turn every search into a build attempt. */+const AUTO_BUILD_RETRY_MS = 60_000;+const RABITQ_UNAVAILABLE_ERROR = '@ruvector/rabitq-wasm not available';let rabitqState: RabitqState | null = null;let rabitqInitializing = false;+let lastAutoBuildAttempt = 0;+/** Set once the WASM module is known to be missing for this process: retrying+ * the same failing dynamic import on every search buys nothing. */+let autoBuildDisabled = false;async function loadRabitqModule(): Promise<{RabitqIndex: any;@@ -88,7 +105,7 @@ export async function buildRabitqIndex(options?: {const mod = await loadRabitqModule();if (!mod) {rabitqInitializing = false;- return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: '@ruvector/rabitq-wasm not available' };+ return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: RABITQ_UNAVAILABLE_ERROR };}const dimensions = options?.dimensions ?? 384;@@ -171,6 +188,7 @@ export async function buildRabitqIndex(options?: {dimensions,builtAt: Date.now(),vectorCount: entries.length,+ sourcePath: options?.dbPath ? path.resolve(options.dbPath) : undefined,};// Persist metadata for fast reload hint@@ -290,6 +308,105 @@ export async function shouldRebuildRabitq(currentEntryCount: number): Promise<boreturn drift > REBUILD_DRIFT_THRESHOLD;}+/**+ * Whether the in-memory index can answer a query for this store/dimension.+ * `undefined` dbPath means "the default store", matching build's own default.+ */+function isRabitqUsable(options?: { dbPath?: string; dimensions?: number }): boolean {+ if (!rabitqState) return false;+ if (options?.dimensions != null && rabitqState.dimensions !== options.dimensions) return false;+ const sourcePath = options?.dbPath ? path.resolve(options.dbPath) : undefined;+ return rabitqState.sourcePath === sourcePath;+}++/**+ * Count the rows `buildRabitqIndex()` would load, using the cheapest source+ * available: the bridge's COUNT (WAL-visible, same rows build prefers) with a+ * sql.js COUNT of the same file as fallback. Returns 0 when neither is+ * readable, which simply leaves the caller on its non-RaBitQ path.+ */+async function countIndexableEntries(options?: { dbPath?: string; dimensions?: number }): Promise<number> {+ const dimensions = options?.dimensions ?? 384;++ try {+ const { bridgeCountEmbeddings } = await import('./memory-bridge.js');+ const bridgeCount = await bridgeCountEmbeddings({ dimensions, dbPath: options?.dbPath });+ if (bridgeCount !== null) return bridgeCount;+ } catch { /* bridge unavailable, fall through */ }++ const dbPath = options?.dbPath+ ? path.resolve(options.dbPath)+ : path.join(path.resolve(process.cwd(), '.swarm'), 'memory.db');+ if (!fs.existsSync(dbPath)) return 0;++ let db: any = null;+ try {+ const initSqlJs = (await import('sql.js')).default;+ const SQL = await initSqlJs();+ db = new SQL.Database(fs.readFileSync(dbPath));++ const count = (dimFilter: string) => db.exec(`+ SELECT COUNT(*) FROM memory_entries+ WHERE status = 'active' AND embedding IS NOT NULL${dimFilter}+ `);++ let result;+ try {+ // Same rows build() keeps: only embeddings of the requested width.+ result = count(` AND json_array_length(embedding) = ${dimensions}`);+ } catch {+ result = count('');+ }+ return Number(result[0]?.values?.[0]?.[0] ?? 0);+ } catch {+ return 0;+ } finally {+ try { db?.close(); } catch { /* ignore */ }+ }+}++/**+ * Auto-build / auto-rebuild entry point (#4203).+ *+ * Search paths call this before `searchRabitq()`: it lets `shouldRebuildRabitq()`+ * decide whether the index is missing or has drifted past the 20% threshold and+ * builds it when so. Never throws β a missing WASM module, an unreadable store+ * or an index built for another store/dimension just leaves the caller's+ * existing non-RaBitQ path untouched. Returns whether an index usable for these+ * options exists afterwards.+ */+export async function ensureRabitqIndex(options?: {+ dbPath?: string;+ dimensions?: number;+}): Promise<boolean> {+ if (autoBuildDisabled || rabitqInitializing) return isRabitqUsable(options);++ const dimensions = options?.dimensions ?? 384;+ const now = Date.now();+ if (now - lastAutoBuildAttempt < AUTO_BUILD_RETRY_MS) return isRabitqUsable(options);+ lastAutoBuildAttempt = now;++ let currentEntryCount = 0;+ try {+ currentEntryCount = await countIndexableEntries({ dbPath: options?.dbPath, dimensions });+ } catch {+ currentEntryCount = 0;+ }++ if (isRabitqUsable(options)) {+ // Same store and width: rebuild only once the corpus has drifted >20%.+ if (!(await shouldRebuildRabitq(currentEntryCount))) return true;+ } else if (!rabitqState) {+ // Nothing built yet: honour the "enough vectors to be worth it" floor.+ if (!(await shouldRebuildRabitq(currentEntryCount))) return false;+ }+ // Either the index is stale for this store/width, or the entry count drifted.++ const result = await buildRabitqIndex({ dbPath: options?.dbPath, dimensions });+ if (!result.success && result.error === RABITQ_UNAVAILABLE_ERROR) autoBuildDisabled = true;+ return isRabitqUsable(options);+}+/*** Get RaBitQ index status.*/