π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
feat: add RaBitQ 1-bit quantized search + fix HNSW/RRF bugs (v3.6.5)
Reuvenca4d1f0a4d23
diff --git a/v3/@claude-flow/cli/package.json b/v3/@claude-flow/cli/package.jsonindex 14f0a2ceb8..96f976277b 100644--- a/v3/@claude-flow/cli/package.json+++ b/v3/@claude-flow/cli/package.json@@ -1,6 +1,6 @@{"name": "@claude-flow/cli",- "version": "3.6.4",+ "version": "3.6.5","type": "module","description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code","main": "dist/src/index.js",@@ -97,23 +97,24 @@"@claude-flow/mcp": "^3.0.0-alpha.8","@claude-flow/shared": "^3.0.0-alpha.7","@noble/ed25519": "^2.1.0",+ "@ruvector/rabitq-wasm": "^0.1.0","semver": "^7.6.0"},"optionalDependencies": {"@claude-flow/aidefence": "^3.0.2",- "@claude-flow/security": "^3.0.0-alpha.1","@claude-flow/codex": "^3.0.0-alpha.8","@claude-flow/embeddings": "^3.0.0-alpha.12","@claude-flow/guidance": "^3.0.0-alpha.1","@claude-flow/memory": "^3.0.0-alpha.12","@claude-flow/plugin-gastown-bridge": "^0.1.3",+ "@claude-flow/security": "^3.0.0-alpha.1","@ruvector/attention": "^0.1.32","@ruvector/attention-darwin-arm64": "0.1.32",+ "@ruvector/diskann": "^0.1.0","@ruvector/learning-wasm": "^0.1.29","@ruvector/router": "^0.1.30","@ruvector/ruvllm-wasm": "^2.0.2","@ruvector/rvagent-wasm": "^0.1.0",- "@ruvector/diskann": "^0.1.0","@ruvector/sona": "^0.1.5","agentdb": "^3.0.0-alpha.11","agentic-flow": "^3.0.0-alpha.1"diff --git a/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts b/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.tsindex 0dbbaed2d0..a825e66f18 100644--- a/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts+++ b/v3/@claude-flow/cli/src/mcp-tools/embeddings-tools.ts@@ -876,4 +876,80 @@ export const embeddingsTools: MCPTool[] = [};},},++ // --- RaBitQ 1-bit quantized vector index ---++ {+ name: 'embeddings_rabitq_build',+ description: 'Build RaBitQ 1-bit quantized index from stored embeddings (32Γ compression). Pre-filters candidates via Hamming scan before exact rerank.',+ category: 'embeddings',+ inputSchema: {+ type: 'object',+ properties: {+ force: { type: 'boolean', description: 'Force rebuild even if index exists' },+ },+ },+ handler: async (params: Record<string, unknown>) => {+ const { buildRabitqIndex } = await import('../memory/rabitq-index.js');+ return buildRabitqIndex({ force: params.force as boolean });+ },+ },++ {+ name: 'embeddings_rabitq_search',+ description: 'Search via RaBitQ quantized index (fast Hamming scan). Returns candidate IDs for reranking.',+ category: 'embeddings',+ inputSchema: {+ type: 'object',+ properties: {+ query: { type: 'string', description: 'Search query text' },+ k: { type: 'number', description: 'Number of results (default: 10)' },+ namespace: { type: 'string', description: 'Filter by namespace' },+ },+ required: ['query'],+ },+ handler: async (params: Record<string, unknown>) => {+ const { validateText: vt } = await import('./validate-input.js');+ const v = vt(params.query as string, 'query');+ if (!v.valid) return { success: false, error: v.error };++ const { searchRabitq } = await import('../memory/rabitq-index.js');+ const { generateEmbedding } = await import('../memory/memory-initializer.js');++ const queryEmb = await generateEmbedding(params.query as string);+ const results = await searchRabitq(queryEmb.embedding, {+ k: (params.k as number) || 10,+ namespace: params.namespace as string,+ });++ if (!results) {+ return { success: false, error: 'RaBitQ index not built. Call embeddings_rabitq_build first.' };+ }++ return {+ success: true,+ results: results.map(r => ({+ id: r.id.substring(0, 12),+ key: r.key,+ namespace: r.namespace,+ distance: Math.round(r.distance * 10000) / 10000,+ })),+ count: results.length,+ };+ },+ },++ {+ name: 'embeddings_rabitq_status',+ description: 'Get RaBitQ quantized index status β availability, vector count, compression ratio',+ category: 'embeddings',+ inputSchema: {+ type: 'object',+ properties: {},+ },+ handler: async () => {+ const { getRabitqStatus } = await import('../memory/rabitq-index.js');+ return { success: true, ...getRabitqStatus() };+ },+ },];diff --git a/v3/@claude-flow/cli/src/mcp-tools/hooks-tools.ts b/v3/@claude-flow/cli/src/mcp-tools/hooks-tools.tsindex 681edadf69..41e4d78bf4 100644--- a/v3/@claude-flow/cli/src/mcp-tools/hooks-tools.ts+++ b/v3/@claude-flow/cli/src/mcp-tools/hooks-tools.ts@@ -2595,6 +2595,7 @@ export const hooksPatternStore: MCPTool = {const success = reasoningResult?.success || storeResult.success;const controller = reasoningResult?.controller || (storeResult.success ? 'bridge-store' : 'none');+ const hasEmbedding = !!storeResult.embedding || controller === 'reasoningBank' || controller === 'bridge-fallback';return {patternId: reasoningResult?.patternId || storeResult.id || patternId,@@ -2602,14 +2603,18 @@ export const hooksPatternStore: MCPTool = {type,confidence,indexed: success,- hnswIndexed: success && (!!storeResult.embedding || controller === 'reasoningBank'),+ hnswIndexed: success && hasEmbedding,embedding: storeResult.embedding,timestamp,controller,- implementation: controller === 'reasoningBank' ? 'reasoning-bank-controller' : (storeResult.success ? 'real-hnsw-indexed' : 'memory-only'),+ implementation: (controller === 'reasoningBank' || controller === 'bridge-fallback')+ ? 'reasoning-bank-controller'+ : (storeResult.success ? 'real-hnsw-indexed' : 'memory-only'),note: controller === 'reasoningBank'? 'Pattern stored via ReasoningBank controller with HNSW indexing'- : (storeResult.success ? 'Pattern stored with vector embedding for semantic search' : (storeResult.error || 'Store function unavailable')),+ : controller === 'bridge-fallback'+ ? 'Pattern stored via bridge with embedding and HNSW indexing'+ : (storeResult.success ? 'Pattern stored with vector embedding for semantic search' : (storeResult.error || 'Store function unavailable')),};},};diff --git a/v3/@claude-flow/cli/src/memory/memory-bridge.ts b/v3/@claude-flow/cli/src/memory/memory-bridge.tsindex 82ba1a9976..c0659766e1 100644--- a/v3/@claude-flow/cli/src/memory/memory-bridge.ts+++ b/v3/@claude-flow/cli/src/memory/memory-bridge.ts@@ -365,6 +365,7 @@ export async function bridgeStoreEntry(options: {success: boolean;id: string;embedding?: { dimensions: number; model: string };+ rawEmbedding?: number[];guarded?: boolean;cached?: boolean;attested?: boolean;@@ -444,6 +445,7 @@ export async function bridgeStoreEntry(options: {success: true,id,embedding: embeddingJson ? { dimensions, model } : undefined,+ rawEmbedding: embeddingJson ? JSON.parse(embeddingJson) as number[] : undefined,guarded: true,cached: true,attested: true,@@ -548,10 +550,11 @@ export async function bridgeSearchEntries(options: {}// Reciprocal rank fusion: combine semantic and BM25- // Weight: 0.7 semantic + 0.3 BM25 (semantic preferred when embeddings available)- const score = queryEmbedding+ // Weight: 0.7 semantic + 0.3 BM25 when both embeddings present+ // Fall back to BM25-only when either query or row lacks an embedding+ const score = semanticScore > 0? (0.7 * semanticScore + 0.3 * bm25ScoreVal)- : bm25ScoreVal; // BM25-only when no embeddings+ : bm25ScoreVal;if (score >= threshold) {// Phase 4: ExplainableRecall provenance@@ -1189,16 +1192,32 @@ export async function bridgeStorePattern(options: {}// Fallback: store via bridge SQL+ const patternValue = JSON.stringify({ pattern: options.pattern, type: options.type, confidence: options.confidence, metadata: options.metadata });const result = await bridgeStoreEntry({key: patternId,- value: JSON.stringify({ pattern: options.pattern, type: options.type, confidence: options.confidence, metadata: options.metadata }),+ value: patternValue,namespace: 'pattern',generateEmbeddingFlag: true,tags: [options.type, 'reasoning-pattern'],dbPath: options.dbPath,});- return result ? { success: true, patternId: result.id, controller: 'bridge-fallback' } : null;+ if (!result) return null;++ // Add to HNSW index for fast semantic search (bridgeStoreEntry stores SQL only)+ if (result.rawEmbedding) {+ try {+ const { addToHNSWIndex } = await import('./memory-initializer.js');+ await addToHNSWIndex(result.id, result.rawEmbedding, {+ id: result.id,+ key: patternId,+ namespace: 'pattern',+ content: patternValue,+ });+ } catch { /* HNSW is best-effort */ }+ }++ return { success: true, patternId: result.id, controller: 'bridge-fallback' };} catch {return null;}diff --git a/v3/@claude-flow/cli/src/memory/memory-initializer.ts b/v3/@claude-flow/cli/src/memory/memory-initializer.tsindex 3fabea2164..3cc82ce9b6 100644--- a/v3/@claude-flow/cli/src/memory/memory-initializer.ts+++ b/v3/@claude-flow/cli/src/memory/memory-initializer.ts@@ -2043,7 +2043,19 @@ export async function storeEntry(options: {const bridge = await getBridge();if (bridge) {const bridgeResult = await bridge.bridgeStoreEntry(options);- if (bridgeResult) return bridgeResult;+ if (bridgeResult) {+ // Keep HNSW index in sync with bridge-stored entries+ if (bridgeResult.rawEmbedding && bridgeResult.success) {+ const ns = options.namespace || 'default';+ await addToHNSWIndex(bridgeResult.id, bridgeResult.rawEmbedding, {+ id: bridgeResult.id,+ key: options.key,+ namespace: ns,+ content: options.value,+ }).catch(() => {});+ }+ return bridgeResult;+ }}// Fallback: raw sql.js@@ -2203,7 +2215,52 @@ export async function searchEntries(options: {const queryEmb = await generateEmbedding(query);const queryEmbedding = queryEmb.embedding;- // Try HNSW search first (150x faster)+ // Try RaBitQ pre-filter first (32Γ compressed Hamming scan)+ try {+ const { searchRabitq } = await import('./rabitq-index.js');+ const rabitqCandidates = await searchRabitq(queryEmbedding, { k: limit * 2, namespace: effectiveNamespace });+ if (rabitqCandidates && rabitqCandidates.length > 0) {+ // Rerank candidates with exact cosine similarity from SQLite+ const initSqlJs = (await import('sql.js')).default;+ const SQL = await initSqlJs();+ const fileBuffer = fs.readFileSync(dbPath);+ const db = new SQL.Database(fileBuffer);+ const reranked: { id: string; key: string; content: string; score: number; namespace: string }[] = [];++ for (const candidate of rabitqCandidates) {+ const stmt = db.prepare('SELECT content, embedding FROM memory_entries WHERE id = ? AND status = ?');+ stmt.bind([candidate.id, 'active']);+ if (stmt.step()) {+ const [content, embeddingJson] = stmt.get() as [string, string | null];+ let score = 0;+ if (embeddingJson) {+ try {+ const embedding = JSON.parse(embeddingJson) as number[];+ score = cosineSim(queryEmbedding, embedding);+ } catch { /* skip */ }+ }+ if (score >= threshold) {+ reranked.push({+ id: candidate.id.substring(0, 12),+ key: candidate.key || candidate.id.substring(0, 15),+ content: (content || '').substring(0, 60) + ((content || '').length > 60 ? '...' : ''),+ score,+ namespace: candidate.namespace,+ });+ }+ }+ stmt.free();+ }+ db.close();++ if (reranked.length > 0) {+ reranked.sort((a, b) => b.score - a.score);+ return { success: true, results: reranked.slice(0, limit), searchTime: Date.now() - startTime };+ }+ }+ } catch { /* RaBitQ unavailable, fall through */ }++ // Try HNSW search (150x faster than brute-force)const hnswResults = await searchHNSWIndex(queryEmbedding, { k: limit, namespace: effectiveNamespace });if (hnswResults && hnswResults.length > 0) {// Filter by thresholddiff --git a/v3/@claude-flow/cli/src/memory/rabitq-index.ts b/v3/@claude-flow/cli/src/memory/rabitq-index.tsnew file mode 100644index 0000000000..6bbbeb3141--- /dev/null+++ b/v3/@claude-flow/cli/src/memory/rabitq-index.ts@@ -0,0 +1,296 @@+/**+ * RaBitQ Index β 1-bit quantized vector pre-filter (32Γ compression)+ *+ * Wraps @ruvector/rabitq-wasm to provide Hamming-scan pre-filtering+ * over quantized embeddings. Candidates are reranked with exact cosine+ * similarity from the full-precision source (HNSW or SQLite).+ *+ * Lifecycle:+ * 1. build() β bulk-load all embeddings from SQLite into the WASM index+ * 2. search() β fast Hamming scan β candidate ids β caller reranks+ * 3. rebuild() β called when entry count drifts >20% from last build+ */++import * as fs from 'fs';+import * as path from 'path';++interface RabitqEntry {+ id: string;+ key: string;+ namespace: string;+}++interface RabitqState {+ index: any; // RabitqIndex from WASM+ entries: RabitqEntry[]; // positional: entries[i] β row i in build()+ dimensions: number;+ builtAt: number;+ vectorCount: number;+}++const RABITQ_SEED = 42n;+const RABITQ_RERANK_FACTOR = 20;+const REBUILD_DRIFT_THRESHOLD = 0.2; // rebuild when count drifts >20%++let rabitqState: RabitqState | null = null;+let rabitqInitializing = false;++async function loadRabitqModule(): Promise<{+ RabitqIndex: any;+ initSync: (m: any) => any;+ version: () => string;+} | null> {+ try {+ const mod = await import('@ruvector/rabitq-wasm');++ // Node.js: use initSync with the WASM bytes+ const { createRequire } = await import('module');+ const require = createRequire(import.meta.url);+ const wasmPath = require.resolve('@ruvector/rabitq-wasm/ruvector_rabitq_wasm_bg.wasm');+ const wasmBytes = fs.readFileSync(wasmPath);+ mod.initSync({ module: wasmBytes });++ return {+ RabitqIndex: mod.RabitqIndex,+ initSync: mod.initSync,+ version: mod.version,+ };+ } catch {+ return null;+ }+}++/**+ * Build or rebuild the RaBitQ index from SQLite embeddings.+ * Returns entry count or 0 if RaBitQ is unavailable.+ */+export async function buildRabitqIndex(options?: {+ dbPath?: string;+ dimensions?: number;+ force?: boolean;+}): Promise<{+ success: boolean;+ vectorCount: number;+ dimensions: number;+ compressionRatio: number;+ buildTimeMs: number;+ wasmVersion?: string;+ error?: string;+}> {+ if (rabitqInitializing) {+ return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: 'Build already in progress' };+ }++ rabitqInitializing = true;+ const startTime = Date.now();++ try {+ const mod = await loadRabitqModule();+ if (!mod) {+ rabitqInitializing = false;+ return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: '@ruvector/rabitq-wasm not available' };+ }++ const dimensions = options?.dimensions ?? 384;+ const swarmDir = path.resolve(process.cwd(), '.swarm');+ const dbPath = options?.dbPath ? path.resolve(options.dbPath) : path.join(swarmDir, 'memory.db');++ if (!fs.existsSync(dbPath)) {+ rabitqInitializing = false;+ return { success: false, vectorCount: 0, dimensions, compressionRatio: 0, buildTimeMs: 0, error: 'Database not found' };+ }++ // Load embeddings from SQLite+ const initSqlJs = (await import('sql.js')).default;+ const SQL = await initSqlJs();+ const fileBuffer = fs.readFileSync(dbPath);+ const db = new SQL.Database(fileBuffer);++ const result = db.exec(`+ SELECT id, key, namespace, embedding+ FROM memory_entries+ WHERE status = 'active' AND embedding IS NOT NULL+ LIMIT 50000+ `);++ const entries: RabitqEntry[] = [];+ const vectors: number[] = [];++ if (result[0]?.values) {+ for (const row of result[0].values) {+ const [id, key, ns, embeddingJson] = row as [string, string, string, string];+ if (!embeddingJson) continue;++ try {+ const embedding = JSON.parse(embeddingJson) as number[];+ if (embedding.length !== dimensions) continue;++ entries.push({ id: String(id), key: key || String(id), namespace: ns || 'default' });+ vectors.push(...embedding);+ } catch {+ // skip invalid+ }+ }+ }++ db.close();++ if (entries.length < 2) {+ rabitqInitializing = false;+ return { success: false, vectorCount: entries.length, dimensions, compressionRatio: 0, buildTimeMs: Date.now() - startTime, error: 'Need at least 2 vectors to build RaBitQ index' };+ }++ // Build the RaBitQ index+ const flatVectors = new Float32Array(vectors);+ const index = mod.RabitqIndex.build(flatVectors, dimensions, RABITQ_SEED, RABITQ_RERANK_FACTOR);++ // Free old index if exists+ if (rabitqState?.index) {+ try { rabitqState.index.free(); } catch { /* already freed */ }+ }++ rabitqState = {+ index,+ entries,+ dimensions,+ builtAt: Date.now(),+ vectorCount: entries.length,+ };++ // Persist metadata for fast reload hint+ try {+ const metaPath = path.join(swarmDir, 'rabitq.meta.json');+ fs.writeFileSync(metaPath, JSON.stringify({+ vectorCount: entries.length,+ dimensions,+ builtAt: rabitqState.builtAt,+ wasmVersion: mod.version(),+ }));+ } catch { /* best-effort */ }++ const rawBytes = entries.length * dimensions * 4; // f32 = 4 bytes+ const quantizedBytes = entries.length * Math.ceil(dimensions / 8); // 1 bit per dim+ const compressionRatio = rawBytes / Math.max(quantizedBytes, 1);++ rabitqInitializing = false;+ return {+ success: true,+ vectorCount: entries.length,+ dimensions,+ compressionRatio: Math.round(compressionRatio * 10) / 10,+ buildTimeMs: Date.now() - startTime,+ wasmVersion: mod.version(),+ };+ } catch (error) {+ rabitqInitializing = false;+ return {+ success: false,+ vectorCount: 0,+ dimensions: 0,+ compressionRatio: 0,+ buildTimeMs: Date.now() - startTime,+ error: error instanceof Error ? error.message : String(error),+ };+ }+}++/**+ * Search the RaBitQ index for candidate IDs.+ * Returns null if index not built or unavailable.+ * Caller is responsible for reranking with exact similarity.+ */+export async function searchRabitq(+ queryEmbedding: number[],+ options?: { k?: number; namespace?: string }+): Promise<Array<{+ id: string;+ key: string;+ namespace: string;+ distance: number;+ position: number;+}> | null> {+ if (!rabitqState?.index) return null;++ try {+ const query = new Float32Array(queryEmbedding);+ if (query.length !== rabitqState.dimensions) return null;++ const k = options?.k ?? 10;+ // Get more candidates than needed for namespace filtering + rerank+ const expandedK = Math.min(k * 3, rabitqState.vectorCount);++ const rawResults = rabitqState.index.search(query, expandedK);++ const results: Array<{+ id: string;+ key: string;+ namespace: string;+ distance: number;+ position: number;+ }> = [];++ for (const hit of rawResults) {+ const pos = hit.id; // row index from build()+ const entry = rabitqState.entries[pos];+ if (!entry) continue;++ // Namespace filter+ if (options?.namespace && options.namespace !== 'all' && entry.namespace !== options.namespace) {+ continue;+ }++ results.push({+ id: entry.id,+ key: entry.key,+ namespace: entry.namespace,+ distance: hit.distance,+ position: pos,+ });++ // Free WASM SearchResult to prevent leak+ try { hit.free(); } catch { /* already freed */ }++ if (results.length >= k) break;+ }++ // Free remaining SearchResults+ for (const hit of rawResults) {+ try { hit.free(); } catch { /* already freed or used */ }+ }++ return results;+ } catch {+ return null;+ }+}++/**+ * Check if the RaBitQ index needs rebuilding.+ */+export async function shouldRebuildRabitq(currentEntryCount: number): Promise<boolean> {+ if (!rabitqState) return currentEntryCount >= 10; // Build if we have enough vectors++ const drift = Math.abs(currentEntryCount - rabitqState.vectorCount) / Math.max(rabitqState.vectorCount, 1);+ return drift > REBUILD_DRIFT_THRESHOLD;+}++/**+ * Get RaBitQ index status.+ */+export function getRabitqStatus(): {+ available: boolean;+ initialized: boolean;+ vectorCount: number;+ dimensions: number;+ builtAt: number | null;+ compressionRatio: number;+} {+ return {+ available: rabitqState !== null,+ initialized: rabitqState !== null,+ vectorCount: rabitqState?.vectorCount ?? 0,+ dimensions: rabitqState?.dimensions ?? 384,+ builtAt: rabitqState?.builtAt ?? null,+ compressionRatio: rabitqState ? Math.round((rabitqState.dimensions * 4) / Math.ceil(rabitqState.dimensions / 8) * 10) / 10 : 0,+ };+}diff --git a/v3/package-lock.json b/v3/package-lock.jsonindex 4fb543854a..ca750f4eb5 100644--- a/v3/package-lock.json+++ b/v3/package-lock.json@@ -1796,6 +1796,7 @@"@claude-flow/mcp": "^3.0.0-alpha.8","@claude-flow/shared": "^3.0.0-alpha.7","@noble/ed25519": "^2.1.0",+ "@ruvector/rabitq-wasm": "^0.1.0","semver": "^7.6.0"},"bin": {@@ -9201,6 +9202,12 @@"license": "MIT OR Apache-2.0","optional": true},+ "node_modules/@ruvector/rabitq-wasm": {+ "version": "0.1.0",+ "resolved": "https://registry.npmjs.org/@ruvector/rabitq-wasm/-/rabitq-wasm-0.1.0.tgz",+ "integrity": "sha512-Uu7fupcJw8x+QBhJLKAfIK0CyH55fQ8niexJwbnSrYEtSX93ScWSOJBr/zTLR/hd0Md9Dq7ZaatzIDB7uJqazg==",+ "license": "MIT OR Apache-2.0"+ },"node_modules/@ruvector/router": {"version": "0.1.30","resolved": "https://registry.npmjs.org/@ruvector/router/-/router-0.1.30.tgz",