Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

fix(memory): build the RaBitQ index on demand and use it on the vector path (#4318)

Gssd7f228683521
diff --git a/plugins/ruflo-agentdb/commands/embeddings.md b/plugins/ruflo-agentdb/commands/embeddings.md
index c2e80b3fb5..bc21460130 100644
--- a/plugins/ruflo-agentdb/commands/embeddings.md
+++ b/plugins/ruflo-agentdb/commands/embeddings.md
@@ -10,7 +10,7 @@ Embedding engine commands:
3. If not initialized, suggest calling `mcp__plugin_ruflo-core_ruflo__embeddings_init`.
4. For search, use `mcp__plugin_ruflo-core_ruflo__embeddings_search` with the user's query (namespace-filtered).
5. For large-corpus search with memory pressure, use the RaBitQ quantized path (32Γ— memory reduction):
- - `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_build` (one-time, after corpus is loaded)
+ - the index builds itself on first use and rebuilds when the corpus drifts >20%; call `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_build` to force one now
- `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_search` (Hamming-prefilter to top-N candidates)
- `mcp__plugin_ruflo-core_ruflo__embeddings_rabitq_status` for index health
6. For hierarchical data (taxonomies, code trees), use `mcp__plugin_ruflo-core_ruflo__embeddings_hyperbolic` (Poincare ball model).
diff --git a/v3/@claude-flow/cli/__tests__/issue-4203-rabitq-auto-build.test.ts b/v3/@claude-flow/cli/__tests__/issue-4203-rabitq-auto-build.test.ts
new file mode 100644
index 0000000000..5db35afe07
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/issue-4203-rabitq-auto-build.test.ts
@@ -0,0 +1,290 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+// #4203: the RaBitQ index was only ever populated by the explicit
+// `embeddings_rabitq_build` tool β€” `shouldRebuildRabitq()` had no callers, so
+// `searchRabitq()` always returned null and the sql.js pre-filter at
+// memory-initializer.ts was dead. The bridge path
+// (bridgeSearchBruteForceCosine) never consulted the index at all.
+//
+// These tests exercise the real store (better-sqlite3 is present in this env)
+// and assert the two halves of the fix: the index is built on demand by the
+// search path, and the bridge search actually consults it.
+
+const DIMENSIONS = 384;
+const ROW_COUNT = 12;
+
+function vectorFor(seed: number): number[] {
+ const out: number[] = [];
+ for (let i = 0; i < DIMENSIONS; i++) {
+ out.push(Math.sin(seed * 0.7 + i * 0.11) * Math.cos(seed * 1.3 + i * 0.03));
+ }
+ return out;
+}
+
+function cosine(a: number[], b: number[]): number {
+ let dot = 0;
+ let na = 0;
+ let nb = 0;
+ for (let i = 0; i < Math.min(a.length, b.length); i++) {
+ dot += a[i] * b[i];
+ na += a[i] * a[i];
+ nb += b[i] * b[i];
+ }
+ const mag = Math.sqrt(na * nb);
+ return mag === 0 ? 0 : dot / mag;
+}
+
+/** Deterministic pseudo-random vector, independent of the `vectorFor` family. */
+function randomVectorFor(seed: number): number[] {
+ let state = seed | 0;
+ const next = () => {
+ state = (state + 0x6d2b79f5) | 0;
+ let t = Math.imul(state ^ (state >>> 15), 1 | state);
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+ };
+ const out: number[] = [];
+ for (let i = 0; i < DIMENSIONS; i++) out.push(next() * 2 - 1);
+ return out;
+}
+
+/** Seed a real store with `rowCount` embeddings and return its path. */
+async function seedStore(
+ root: string,
+ rowCount = ROW_COUNT,
+ vectorAt: (i: number) => number[] = vectorFor,
+): Promise<string | null> {
+ const dbPath = join(root, 'memory.db');
+ const bridge = await import('../src/memory/memory-bridge.js');
+ const { initializeMemoryDatabase } = await import('../src/memory/memory-initializer.js');
+
+ await initializeMemoryDatabase({ dbPath, force: true, verbose: false });
+
+ const status = await bridge.bridgeGetVectorSearchStatus(dbPath);
+ if (!status) return null; // bridge unavailable in this environment
+
+ for (let i = 0; i < rowCount; i++) {
+ const id = `00000000-0000-4000-8000-${String(i).padStart(12, '0')}`;
+ const written = await bridge.bridgeAddEmbedding(
+ id,
+ vectorAt(i),
+ { id, key: `emb-${i}`, namespace: 'test', content: `row ${i}` },
+ dbPath,
+ );
+ if (written !== true) return null;
+ }
+
+ return dbPath;
+}
+
+describe('#4203 RaBitQ index is built on demand instead of only via the manual tool', () => {
+ // Each test gets a fresh module graph: the index state, the auto-build
+ // throttle and the bridge registry cache are all module-level.
+ beforeEach(() => {
+ vi.resetModules();
+ });
+
+ it('ensureRabitqIndex() builds from stored embeddings and serves searches', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-build-'));
+ try {
+ const dbPath = await seedStore(root);
+ if (!dbPath) return; // bridge unavailable in this environment
+
+ const rabitq = await import('../src/memory/rabitq-index.js');
+
+ // Pre-fix: nothing ever built the index, so this was always uninitialized
+ // no matter how many embeddings memory_store had written.
+ expect(rabitq.getRabitqStatus().initialized).toBe(false);
+
+ expect(await rabitq.ensureRabitqIndex({ dbPath, dimensions: DIMENSIONS })).toBe(true);
+ expect(rabitq.getRabitqStatus().vectorCount).toBe(ROW_COUNT);
+
+ // An index that is already current must not be rebuilt.
+ expect(await rabitq.shouldRebuildRabitq(ROW_COUNT)).toBe(false);
+
+ const hits = await rabitq.searchRabitq(vectorFor(0), { k: 3 });
+ expect(hits).not.toBeNull();
+ expect(hits![0].key).toBe('emb-0');
+ expect(hits![0].namespace).toBe('test');
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
+
+ it('bridgeSearchBruteForceCosine() consults the RaBitQ index and reranks exactly', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-bridge-'));
+ try {
+ const dbPath = await seedStore(root);
+ if (!dbPath) return; // bridge unavailable in this environment
+
+ const bridge = await import('../src/memory/memory-bridge.js');
+ const rabitq = await import('../src/memory/rabitq-index.js');
+
+ // Pre-fix: the bridge scanned the whole table and the index stayed empty.
+ expect(rabitq.getRabitqStatus().initialized).toBe(false);
+
+ const query = vectorFor(5);
+ const results = await bridge.bridgeSearchBruteForceCosine(query, { k: 3 }, dbPath);
+
+ expect(results).not.toBeNull();
+ expect(results!.length).toBeGreaterThan(0);
+ expect(results![0].key).toBe('emb-5');
+ // Reranking uses exact cosine, so the score keeps its brute-force meaning.
+ expect(results![0].score).toBeCloseTo(cosine(query, vectorFor(5)), 6);
+ expect(results![0].id).toBe('00000000-000');
+ expect(results![0].namespace).toBe('test');
+
+ // The index the search used was built by that same search call.
+ expect(rabitq.getRabitqStatus().initialized).toBe(true);
+ expect(rabitq.getRabitqStatus().vectorCount).toBe(ROW_COUNT);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
+
+ it('does not rebuild the index on every search', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-throttle-'));
+ try {
+ const dbPath = await seedStore(root);
+ if (!dbPath) return; // bridge unavailable in this environment
+
+ const bridge = await import('../src/memory/memory-bridge.js');
+ const rabitq = await import('../src/memory/rabitq-index.js');
+
+ const first = await bridge.bridgeSearchBruteForceCosine(vectorFor(3), { k: 2 }, dbPath);
+ expect(first![0].key).toBe('emb-3');
+
+ const builtAt = rabitq.getRabitqStatus().builtAt;
+ expect(builtAt).not.toBeNull();
+
+ // A second search, plus an explicit ensure for the same store and width,
+ // must reuse the index: rebuilding on every query would cost more than
+ // the brute-force scan it replaces.
+ const second = await bridge.bridgeSearchBruteForceCosine(vectorFor(3), { k: 2 }, dbPath);
+ expect(second![0].key).toBe('emb-3');
+ expect(await rabitq.ensureRabitqIndex({ dbPath, dimensions: DIMENSIONS })).toBe(true);
+ expect(rabitq.getRabitqStatus().builtAt).toBe(builtAt);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
+
+ // Negative control: the auto-build must not fire (and must not throw) when the
+ // corpus is below the floor, and the caller's exhaustive scan must still be
+ // the thing that answers.
+ it('leaves the exhaustive scan in place when the corpus is below the build floor', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-small-'));
+ try {
+ const dbPath = await seedStore(root, 5);
+ if (!dbPath) return; // bridge unavailable in this environment
+
+ const rabitq = await import('../src/memory/rabitq-index.js');
+ const bridge = await import('../src/memory/memory-bridge.js');
+
+ expect(await rabitq.ensureRabitqIndex({ dbPath, dimensions: DIMENSIONS })).toBe(false);
+ expect(rabitq.getRabitqStatus().initialized).toBe(false);
+
+ const results = await bridge.bridgeSearchBruteForceCosine(vectorFor(3), { k: 2 }, dbPath);
+ expect(results).not.toBeNull();
+ expect(results![0].key).toBe('emb-3');
+ expect(results![0].score).toBeCloseTo(cosine(vectorFor(3), vectorFor(3)), 6);
+ expect(rabitq.getRabitqStatus().initialized).toBe(false);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
+
+ // Positive/negative control pair over the same corpus: the indexed path and
+ // the exhaustive path must return the same rows with the same exact scores,
+ // so switching RaBitQ on cannot silently change rankings.
+ // Note: this corpus (12 rows) fits entirely inside the candidate window
+ // (k * 2 * 3 = 30), so equality here is exact by construction. The recall
+ // test below covers a corpus that is larger than the window.
+ it('indexed search agrees with the exhaustive scan on the same corpus', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-parity-'));
+ try {
+ const dbPath = await seedStore(root);
+ if (!dbPath) return; // bridge unavailable in this environment
+
+ // Negative control: a 128-dim probe cannot be served by a 384-dim store,
+ // so the first search keeps the exhaustive scan (the retry window opened
+ // by that probe also keeps it from building behind our back).
+ const rabitqExhaustive = await import('../src/memory/rabitq-index.js');
+ const bridgeExhaustive = await import('../src/memory/memory-bridge.js');
+ expect(await rabitqExhaustive.ensureRabitqIndex({ dbPath, dimensions: 128 })).toBe(false);
+ const exhaustive = await bridgeExhaustive.bridgeSearchBruteForceCosine(vectorFor(7), { k: 5 }, dbPath);
+ expect(exhaustive).not.toBeNull();
+ expect(exhaustive!.length).toBeGreaterThan(0);
+ expect(rabitqExhaustive.getRabitqStatus().initialized).toBe(false);
+
+ // Positive control: a fresh module graph has no retry window, so the same
+ // query is served from the index that this search builds.
+ vi.resetModules();
+ const rabitqIndexed = await import('../src/memory/rabitq-index.js');
+ const bridgeIndexed = await import('../src/memory/memory-bridge.js');
+ const indexed = await bridgeIndexed.bridgeSearchBruteForceCosine(vectorFor(7), { k: 5 }, dbPath);
+ expect(indexed).not.toBeNull();
+ expect(rabitqIndexed.getRabitqStatus().initialized).toBe(true);
+
+ expect(indexed!.map(r => r.key)).toEqual(exhaustive!.map(r => r.key));
+ expect(indexed!.map(r => r.id)).toEqual(exhaustive!.map(r => r.id));
+ for (let i = 0; i < indexed!.length; i++) {
+ expect(indexed![i].score).toBeCloseTo(exhaustive![i].score, 6);
+ expect(indexed![i].namespace).toBe(exhaustive![i].namespace);
+ }
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
+
+ // Recall/rerank check on a corpus that is larger than the candidate window
+ // (60 rows, k = 5 β†’ 30 candidates), where the two paths may legitimately rank
+ // differently. Two things must still hold: every returned score is that row's
+ // exact cosine (the rerank is not an approximation), and top-k recall does not
+ // collapse because of the pre-filter.
+ it('reranks with exact cosine and keeps top-k recall above the window size', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'ruflo-4203-recall-'));
+ try {
+ const query = randomVectorFor(999);
+ const dbPath = await seedStore(root, 60, i => randomVectorFor(i + 1));
+ if (!dbPath) return; // bridge unavailable in this environment
+
+ // Same negative-then-positive control: the first search keeps the
+ // exhaustive scan, the second (fresh module graph) uses the index.
+ const rabitqExhaustive = await import('../src/memory/rabitq-index.js');
+ const bridgeExhaustive = await import('../src/memory/memory-bridge.js');
+ expect(await rabitqExhaustive.ensureRabitqIndex({ dbPath, dimensions: 128 })).toBe(false);
+ const exhaustive = await bridgeExhaustive.bridgeSearchBruteForceCosine(
+ query, { k: 5, threshold: -1 }, dbPath,
+ );
+ expect(exhaustive).not.toBeNull();
+ expect(exhaustive!.length).toBe(5);
+
+ vi.resetModules();
+ const rabitqIndexed = await import('../src/memory/rabitq-index.js');
+ const bridgeIndexed = await import('../src/memory/memory-bridge.js');
+ const indexed = await bridgeIndexed.bridgeSearchBruteForceCosine(
+ query, { k: 5, threshold: -1 }, dbPath,
+ );
+ expect(rabitqIndexed.getRabitqStatus().initialized).toBe(true);
+ expect(indexed).not.toBeNull();
+
+ // Rerank correctness: each score equals that row's exact cosine.
+ for (const row of indexed!) {
+ const seed = Number(row.key.replace('emb-', '')) + 1;
+ expect(row.score).toBeCloseTo(cosine(query, randomVectorFor(seed)), 6);
+ }
+
+ // Recall: β‰₯80% of the exhaustive top-5 must survive the pre-filter.
+ const indexedKeys = new Set(indexed!.map(r => r.key));
+ const recovered = exhaustive!.filter(r => indexedKeys.has(r.key)).length;
+ expect(recovered).toBeGreaterThanOrEqual(4);
+ expect(indexed![0].key).toBe(exhaustive![0].key);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts b/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts
index 28f27f0d18..6e3bd0696a 100644
--- a/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts
+++ b/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts
@@ -1022,17 +1022,25 @@ export const embeddingsTools: MCPTool[] = [
const v = vt(params.query as string, 'query');
if (!v.valid) return { success: false, error: v.error };
- const { searchRabitq } = await import('../memory/rabitq-index.js');
+ const { ensureRabitqIndex, searchRabitq } = await import('../memory/rabitq-index.js');
const { generateEmbedding } = await import('../memory/memory-initializer.js');
const queryEmb = await generateEmbedding(params.query as string);
- const results = await searchRabitq(queryEmb.embedding, {
+ const searchOptions = {
k: (params.k as number) || 10,
namespace: params.namespace as string,
- });
+ };
+
+ // #4203: the index is built on demand, so a search no longer requires a
+ // separate `embeddings_rabitq_build` call first.
+ let results = await searchRabitq(queryEmb.embedding, searchOptions);
+ if (!results) {
+ await ensureRabitqIndex({ dimensions: queryEmb.embedding.length });
+ results = await searchRabitq(queryEmb.embedding, searchOptions);
+ }
if (!results) {
- return { success: false, error: 'RaBitQ index not built. Call embeddings_rabitq_build first.' };
+ return { success: false, error: 'RaBitQ index not built (needs at least 10 embedded entries). Call embeddings_rabitq_build first.' };
}
return {
diff --git a/v3/@claude-flow/cli/src/memory/memory-bridge.ts b/v3/@claude-flow/cli/src/memory/memory-bridge.ts
index de68248cd7..3c01f27420 100644
--- a/v3/@claude-flow/cli/src/memory/memory-bridge.ts
+++ b/v3/@claude-flow/cli/src/memory/memory-bridge.ts
@@ -2113,10 +2113,95 @@ export async function bridgeGetVectorSearchStatus(
}
}
+/**
+ * #4203: RaBitQ pre-filter for the bridge's vector search.
+ *
+ * The index yields candidate ids from a 32Γ—-compressed Hamming scan; each is
+ * reranked here with the same exact cosine similarity the exhaustive scan uses,
+ * so scores keep their meaning and only the scanned set shrinks (the candidate
+ * window `k * 2` is then widened 3Γ— by `searchRabitq` itself).
+ *
+ * Returns null whenever the index cannot answer this query β€” WASM missing, too
+ * few vectors, or an index built for another store/dimension β€” so the caller
+ * runs its unchanged brute-force scan. Keyword/BM25 recall is unaffected: this
+ * only ever replaces the cosine scan.
+ */
+async function rabitqSearchAndRerank(
+ db: any,
+ queryEmbedding: number[],
+ options: { k: number; threshold: number; namespace?: string; dbPath?: string },
+): Promise<Array<{ id: string; key: string; content: string; score: number; namespace: string }> | null> {
+ try {
+ const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');
+
+ const ready = await ensureRabitqIndex({ dbPath: options.dbPath, dimensions: queryEmbedding.length });
+ if (!ready) return null;
+
+ const candidates = await searchRabitq(queryEmbedding, {
+ k: options.k * 2,
+ namespace: options.namespace,
+ });
+ if (!candidates || candidates.length === 0) return null;
+
+ // Fetch exactly the candidates (id is the primary key) instead of the
+ // 10k-row full scan; chunked to stay well inside SQLite's parameter limit.
+ const rowsById = new Map<string, { key: string; namespace: string; content: string; score: number }>();
+ const ids = candidates.map(c => c.id);
+ for (let i = 0; i < ids.length; i += 500) {
+ const chunk = ids.slice(i, i + 500);
+ const placeholders = chunk.map(() => '?').join(',');
+ const rows: any[] = db.prepare(`
+ SELECT id, key, namespace, content, embedding
+ FROM memory_entries
+ WHERE id IN (${placeholders}) AND ${liveMemoryRowSql()} AND embedding IS NOT NULL
+ `).all(...chunk);
+
+ for (const row of rows) {
+ let embedding: number[];
+ try {
+ embedding = JSON.parse(row.embedding) as number[];
+ } catch {
+ continue;
+ }
+ if (!Array.isArray(embedding) || embedding.length !== queryEmbedding.length) continue;
+ rowsById.set(String(row.id), {
+ key: row.key || String(row.id),
+ namespace: row.namespace || 'default',
+ content: row.content || '',
+ score: cosineSim(queryEmbedding, embedding),
+ });
+ }
+ }
+
+ const results: Array<{ id: string; key: string; content: string; score: number; namespace: string }> = [];
+ for (const candidate of candidates) {
+ const row = rowsById.get(candidate.id);
+ if (!row || row.score < options.threshold) continue;
+ results.push({
+ id: candidate.id.substring(0, 12),
+ key: row.key || candidate.id.substring(0, 15),
+ content: row.content.substring(0, 60) + (row.content.length > 60 ? '...' : ''),
+ score: row.score,
+ namespace: row.namespace,
+ });
+ }
+
+ if (results.length === 0) return null;
+ results.sort((a, b) => b.score - a.score);
+ return results.slice(0, options.k);
+ } catch {
+ return null;
+ }
+}
+
/**
* Search AgentDB v3's embedder + SQLite entries via a full-table scan and
* brute-force cosine similarity. NOT HNSW-accelerated β€” see #2922. Returns
* null if unavailable.
+ *
+ * #4203: when a RaBitQ index is available the Hamming-scan candidate set is
+ * reranked exactly instead of scanning the whole table; the exhaustive loop
+ * below stays the fallback, so results and scores keep their meaning.
*/
export async function bridgeSearchBruteForceCosine(
queryEmbedding: number[],
@@ -2143,6 +2228,17 @@ export async function bridgeSearchBruteForceCosine(
? `AND namespace = ?`
: '';
+ // #4203: try the RaBitQ pre-filter before the exhaustive scan below. A null
+ // result (no index / unavailable / nothing above threshold) falls through
+ // to the unchanged brute-force path.
+ const accelerated = await rabitqSearchAndRerank(ctx.db, queryEmbedding, {
+ k,
+ threshold,
+ namespace: options?.namespace,
+ dbPath,
+ });
+ if (accelerated) return accelerated;
+
let rows: any[];
try {
// #2982/#2976: same unordered pre-rank truncation as bridgeSearchEntries
@@ -3772,6 +3868,47 @@ export async function bridgeGetAllEmbeddings(options?: {
}
}
+/**
+ * Count the embeddings `bridgeGetAllEmbeddings()` would export (#4203).
+ * RaBitQ's auto-build uses this as its cheap "has the corpus drifted?" probe β€”
+ * a COUNT instead of re-reading every embedding blob. With `dimensions` it
+ * counts only rows build() would keep, so the drift ratio compares like with
+ * like. Returns null if the bridge is unavailable.
+ */
+export async function bridgeCountEmbeddings(options?: {
+ dimensions?: number;
+ dbPath?: string;
+}): Promise<number | null> {
+ if (!operationContext.getStore()?.active) return withBridgeOperation(() => bridgeCountEmbeddings(options));
+ const registry = await getRegistry(options?.dbPath);
+ if (!registry) return null;
+
+ const ctx = getDb(registry);
+ if (!ctx) return null;
+
+ const dims = options?.dimensions;
+ try {
+ if (dims != null) {
+ try {
+ const row = ctx.db.prepare(
+ `SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL AND json_array_length(embedding) = ?`,
+ ).get(dims);
+ return row?.cnt ?? 0;
+ } catch {
+ // JSON1 unavailable in this SQLite build β€” fall through to the
+ // unfiltered count rather than losing the probe entirely.
+ }
+ }
+
+ const row = ctx.db.prepare(
+ `SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL`,
+ ).get();
+ return row?.cnt ?? 0;
+ } catch {
+ return null;
+ }
+}
+
// ===== Utility =====
function cosineSim(a: number[], b: number[]): number {
diff --git a/v3/@claude-flow/cli/src/memory/memory-initializer.ts b/v3/@claude-flow/cli/src/memory/memory-initializer.ts
index 04c9870a99..ba016a2895 100644
--- a/v3/@claude-flow/cli/src/memory/memory-initializer.ts
+++ b/v3/@claude-flow/cli/src/memory/memory-initializer.ts
@@ -3270,7 +3270,11 @@ export async function searchEntries(options: {
// Try RaBitQ pre-filter first (32Γ— compressed Hamming scan)
try {
- const { searchRabitq } = await import('./rabitq-index.js');
+ const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');
+ // #4203: build/rebuild on demand β€” without this the index was only ever
+ // populated by the explicit `embeddings_rabitq_build` tool, so this
+ // pre-filter always missed.
+ await ensureRabitqIndex({ dbPath, dimensions: queryEmbedding.length });
const rabitqCandidates = await searchRabitq(queryEmbedding, { k: limit * 2, namespace: effectiveNamespace });
if (rabitqCandidates && rabitqCandidates.length > 0) {
// Rerank candidates with exact cosine similarity from SQLite
diff --git a/v3/@claude-flow/cli/src/memory/rabitq-index.ts b/v3/@claude-flow/cli/src/memory/rabitq-index.ts
index 9ea096b11b..55d41cc6c2 100644
--- a/v3/@claude-flow/cli/src/memory/rabitq-index.ts
+++ b/v3/@claude-flow/cli/src/memory/rabitq-index.ts
@@ -9,6 +9,11 @@
* 1. build() β€” bulk-load all embeddings from SQLite into the WASM index
* 2. search() β€” fast Hamming scan β†’ candidate ids β†’ caller reranks
* 3. rebuild() β€” called when entry count drifts >20% from last build
+ *
+ * #4203: step 3 is driven by `ensureRabitqIndex()`, which the search paths
+ * call before `searchRabitq()`. Previously the only caller of the build was
+ * the explicit `embeddings_rabitq_build` tool, so the index was effectively
+ * never populated and every `searchRabitq()` returned null.
*/
import * as fs from 'fs';
@@ -26,14 +31,26 @@ interface RabitqState {
dimensions: number;
builtAt: number;
vectorCount: number;
+ /** Resolved source database the index was built from (#4203). An index built
+ * from one store must not answer a query aimed at another one β€” same reason
+ * the bridge registry cache is keyed by path (#3196). */
+ sourcePath?: string;
}
const RABITQ_SEED = 42n;
const RABITQ_RERANK_FACTOR = 20;
const REBUILD_DRIFT_THRESHOLD = 0.2; // rebuild when count drifts >20%
+/** #4203: minimum gap between auto-build attempts, so a store that is still
+ * too small cannot turn every search into a build attempt. */
+const AUTO_BUILD_RETRY_MS = 60_000;
+const RABITQ_UNAVAILABLE_ERROR = '@ruvector/rabitq-wasm not available';
let rabitqState: RabitqState | null = null;
let rabitqInitializing = false;
+let lastAutoBuildAttempt = 0;
+/** Set once the WASM module is known to be missing for this process: retrying
+ * the same failing dynamic import on every search buys nothing. */
+let autoBuildDisabled = false;
async function loadRabitqModule(): Promise<{
RabitqIndex: any;
@@ -88,7 +105,7 @@ export async function buildRabitqIndex(options?: {
const mod = await loadRabitqModule();
if (!mod) {
rabitqInitializing = false;
- return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: '@ruvector/rabitq-wasm not available' };
+ return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: RABITQ_UNAVAILABLE_ERROR };
}
const dimensions = options?.dimensions ?? 384;
@@ -171,6 +188,7 @@ export async function buildRabitqIndex(options?: {
dimensions,
builtAt: Date.now(),
vectorCount: entries.length,
+ sourcePath: options?.dbPath ? path.resolve(options.dbPath) : undefined,
};
// Persist metadata for fast reload hint
@@ -290,6 +308,105 @@ export async function shouldRebuildRabitq(currentEntryCount: number): Promise<bo
return drift > REBUILD_DRIFT_THRESHOLD;
}
+/**
+ * Whether the in-memory index can answer a query for this store/dimension.
+ * `undefined` dbPath means "the default store", matching build's own default.
+ */
+function isRabitqUsable(options?: { dbPath?: string; dimensions?: number }): boolean {
+ if (!rabitqState) return false;
+ if (options?.dimensions != null && rabitqState.dimensions !== options.dimensions) return false;
+ const sourcePath = options?.dbPath ? path.resolve(options.dbPath) : undefined;
+ return rabitqState.sourcePath === sourcePath;
+}
+
+/**
+ * Count the rows `buildRabitqIndex()` would load, using the cheapest source
+ * available: the bridge's COUNT (WAL-visible, same rows build prefers) with a
+ * sql.js COUNT of the same file as fallback. Returns 0 when neither is
+ * readable, which simply leaves the caller on its non-RaBitQ path.
+ */
+async function countIndexableEntries(options?: { dbPath?: string; dimensions?: number }): Promise<number> {
+ const dimensions = options?.dimensions ?? 384;
+
+ try {
+ const { bridgeCountEmbeddings } = await import('./memory-bridge.js');
+ const bridgeCount = await bridgeCountEmbeddings({ dimensions, dbPath: options?.dbPath });
+ if (bridgeCount !== null) return bridgeCount;
+ } catch { /* bridge unavailable, fall through */ }
+
+ const dbPath = options?.dbPath
+ ? path.resolve(options.dbPath)
+ : path.join(path.resolve(process.cwd(), '.swarm'), 'memory.db');
+ if (!fs.existsSync(dbPath)) return 0;
+
+ let db: any = null;
+ try {
+ const initSqlJs = (await import('sql.js')).default;
+ const SQL = await initSqlJs();
+ db = new SQL.Database(fs.readFileSync(dbPath));
+
+ const count = (dimFilter: string) => db.exec(`
+ SELECT COUNT(*) FROM memory_entries
+ WHERE status = 'active' AND embedding IS NOT NULL${dimFilter}
+ `);
+
+ let result;
+ try {
+ // Same rows build() keeps: only embeddings of the requested width.
+ result = count(` AND json_array_length(embedding) = ${dimensions}`);
+ } catch {
+ result = count('');
+ }
+ return Number(result[0]?.values?.[0]?.[0] ?? 0);
+ } catch {
+ return 0;
+ } finally {
+ try { db?.close(); } catch { /* ignore */ }
+ }
+}
+
+/**
+ * Auto-build / auto-rebuild entry point (#4203).
+ *
+ * Search paths call this before `searchRabitq()`: it lets `shouldRebuildRabitq()`
+ * decide whether the index is missing or has drifted past the 20% threshold and
+ * builds it when so. Never throws β€” a missing WASM module, an unreadable store
+ * or an index built for another store/dimension just leaves the caller's
+ * existing non-RaBitQ path untouched. Returns whether an index usable for these
+ * options exists afterwards.
+ */
+export async function ensureRabitqIndex(options?: {
+ dbPath?: string;
+ dimensions?: number;
+}): Promise<boolean> {
+ if (autoBuildDisabled || rabitqInitializing) return isRabitqUsable(options);
+
+ const dimensions = options?.dimensions ?? 384;
+ const now = Date.now();
+ if (now - lastAutoBuildAttempt < AUTO_BUILD_RETRY_MS) return isRabitqUsable(options);
+ lastAutoBuildAttempt = now;
+
+ let currentEntryCount = 0;
+ try {
+ currentEntryCount = await countIndexableEntries({ dbPath: options?.dbPath, dimensions });
+ } catch {
+ currentEntryCount = 0;
+ }
+
+ if (isRabitqUsable(options)) {
+ // Same store and width: rebuild only once the corpus has drifted >20%.
+ if (!(await shouldRebuildRabitq(currentEntryCount))) return true;
+ } else if (!rabitqState) {
+ // Nothing built yet: honour the "enough vectors to be worth it" floor.
+ if (!(await shouldRebuildRabitq(currentEntryCount))) return false;
+ }
+ // Either the index is stale for this store/width, or the entry count drifted.
+
+ const result = await buildRabitqIndex({ dbPath: options?.dbPath, dimensions });
+ if (!result.success && result.error === RABITQ_UNAVAILABLE_ERROR) autoBuildDisabled = true;
+ return isRabitqUsable(options);
+}
+
/**
* Get RaBitQ index status.
*/