Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

fix(memory): wire real AgentDB HNSWIndex with safe fallback (#4249) (#4322)

rUv462c573cb748
diff --git a/v3/@claude-flow/cli/package.json b/v3/@claude-flow/cli/package.json
index f49c08187d..646ce87050 100644
--- a/v3/@claude-flow/cli/package.json
+++ b/v3/@claude-flow/cli/package.json
@@ -127,11 +127,11 @@
"ws": "^8.21.0",
"yaml": "^2.8.0",
"zod": "^3.22.0",
- "@claude-flow/memory": "3.0.7"
+ "@claude-flow/memory": "3.0.8"
},
"optionalDependencies": {
"@agntcy/slim-bindings": "2.0.0-alpha.5",
- "@claude-flow/memory": "3.0.7",
+ "@claude-flow/memory": "3.0.8",
"@metaharness/darwin": "~0.12.0",
"@metaharness/flywheel": "~0.1.10",
"@metaharness/radio": "~0.1.0",
diff --git a/v3/@claude-flow/memory/benchmarks/agentdb-hnsw.mjs b/v3/@claude-flow/memory/benchmarks/agentdb-hnsw.mjs
new file mode 100644
index 0000000000..6dd0a4c977
--- /dev/null
+++ b/v3/@claude-flow/memory/benchmarks/agentdb-hnsw.mjs
@@ -0,0 +1,81 @@
+/**
+ * Reproducible, bounded AgentDBBackend comparison for #4249.
+ * Build memory first, then: AGENTDB_DISABLE_TRANSFORMERS=1 node benchmarks/agentdb-hnsw.mjs
+ * Run this identical file in a baseline checkout and the candidate checkout.
+ * No model, network embeddings, random wall-clock seeds, or speed assertions.
+ */
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync } from 'node:fs';
+import { cpus, platform, arch } from 'node:os';
+import { AgentDBBackend } from '../dist/agentdb-backend.js';
+import { createDefaultEntry } from '../dist/types.js';
+
+const dimensions = 64;
+const count = 1024;
+const queryCount = 100;
+const k = 10;
+let seed = 4249;
+const random = () => {
+ seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
+ return seed / 0x100000000 * 2 - 1;
+};
+const vector = () => Float32Array.from({ length: dimensions }, random);
+const entries = Array.from({ length: count }, (_, i) => ({
+ ...createDefaultEntry({ key: `v${i}`, content: `v${i}` }),
+ id: `v${i}`,
+ embedding: vector(),
+}));
+const queries = Array.from({ length: queryCount }, vector);
+const cosine = (a, b) => {
+ let dot = 0, aa = 0, bb = 0;
+ for (let i = 0; i < a.length; i++) {
+ dot += a[i] * b[i]; aa += a[i] ** 2; bb += b[i] ** 2;
+ }
+ return dot / Math.sqrt(aa * bb);
+};
+const expected = queries.map((q) => new Set(entries.map((entry) => ({
+ id: entry.id, score: cosine(q, entry.embedding),
+})).sort((a, b) => b.score - a.score).slice(0, k).map((entry) => entry.id)));
+const backend = new AgentDBBackend({
+ dbPath: ':memory:', vectorDimension: dimensions, vectorBackend: 'hnswlib',
+});
+try {
+ await backend.initialize();
+ if (!backend.isAvailable()) throw new Error('Real AgentDB failed to initialize');
+ await backend.bulkInsert(entries);
+ const coldStart = performance.now();
+ await backend.search(queries[0], { k });
+ const coldSearchMs = performance.now() - coldStart;
+ for (let i = 0; i < 10; i++) await backend.search(queries[i], { k });
+ const times = [];
+ let matched = 0;
+ for (let i = 0; i < queryCount; i++) {
+ const start = performance.now();
+ const results = await backend.search(queries[i], { k });
+ times.push(performance.now() - start);
+ matched += results.filter((r) => expected[i].has(r.entry.id)).length;
+ }
+ times.sort((a, b) => a - b);
+ const stats = await backend.getStats();
+ console.log(JSON.stringify({
+ node: process.version,
+ platform: `${platform()}-${arch()}`,
+ cpu: cpus()[0]?.model,
+ dimensions, count, queryCount, k, seed: 4249,
+ sourceSha256: Object.fromEntries([
+ '../src/agentdb-backend.ts', '../src/agentdb-hnsw-adapter.ts',
+ './agentdb-hnsw.mjs', '../dist/agentdb-backend.js', '../dist/agentdb-hnsw-adapter.js',
+ ].map((name) => {
+ const path = new URL(name, import.meta.url);
+ return [name, existsSync(path) ? createHash('sha256').update(readFileSync(path)).digest('hex') : null];
+ })),
+ nativeIndexBuilt: !!stats.hnswStats,
+ coldSearchMs,
+ warmP50Ms: times[Math.floor(times.length * 0.5)],
+ warmP95Ms: times[Math.floor(times.length * 0.95)],
+ recallAt10: matched / (queryCount * k),
+ limitation: 'Single-process bounded run, precomputed seeded vectors, not a production or universal speedup claim.',
+ }, null, 2));
+} finally {
+ await backend.shutdown();
+}
diff --git a/v3/@claude-flow/memory/docs/AGENTDB-HNSW-4249-VERIFICATION.md b/v3/@claude-flow/memory/docs/AGENTDB-HNSW-4249-VERIFICATION.md
new file mode 100644
index 0000000000..7bb8cf328e
--- /dev/null
+++ b/v3/@claude-flow/memory/docs/AGENTDB-HNSW-4249-VERIFICATION.md
@@ -0,0 +1,160 @@
+# AgentDB HNSW adapter verification
+
+Date: 2026-10-20. Scope: issue [#4249](https://github.com/ruvnet/ruflo/issues/4249).
+Baseline: `19fb5c6e7cbf0ce37c0269ca2c2fca3dda760851`.
+Rebased onto `ab40a99f3a0ff03442759fe082037d6a7630ad53` before publication;
+its intervening changes were confined to swarm, not memory or its lockfile.
+The existing cleanup-only draft [#4250](https://github.com/ruvnet/ruflo/pull/4250)
+was inspected at `88b90fdfd6ad66c266777503fb7c957028829c87` and left unchanged.
+
+## What is implemented
+
+- Replace the three unsupported `getController('hnsw')` calls with the actual
+ separately exported `agentdb.HNSWIndex`.
+- Build a fresh index from copied, normalized in-memory vector snapshots.
+ The snapshot adapter permits only one fixed, read-only query. No user SQL,
+ arbitrary filter keys, controller storage, or persistence sidecars are used.
+- Lazily rebuild after store/update/replacement/delete. Reuse an unchanged
+ index, reject stale asynchronous builds/searches by revision, serialize
+ per-query `ef` changes, and use collision-free snapshot IDs.
+- Check native tuning before uint32 coercion. Normalize with JS double
+ precision before native float32 cosine normalization to handle extreme
+ finite values. Clamp native `k` to indexed count.
+- Preserve exact cosine fallback for optional dependency/export/addon absence,
+ unsupported configuration, dimensions, filtered searches and native errors.
+ Do not report HNSW stats/health unless the current index really is built.
+- Detach entry/vector inputs and outputs and snapshot query vectors before any
+ await, preventing caller mutations from bypassing index invalidation. Keep
+ TTL-bearing semantic queries exact so expiry eligibility cannot diverge.
+- Remove unsupported performance claims from this backend's integration guide.
+
+The use of fresh snapshots is deliberate. In alpha.20, construction/initialize
+alone does not build an empty index, `addVector` is not an upsert, and
+`removeVector` removes mappings without deleting the native point. Rebuilding
+avoids duplicate labels and tombstone underfill. It is a read-heavy optimization,
+not an optimized incremental-write implementation.
+
+## Environment and actual dependency
+
+- Linux x64; AMD EPYC 9V74; Node `v24.19.0`.
+- `[email protected]`, checked-in lockfile; Vitest `4.1.8`; TypeScript from lockfile.
+ `[email protected]`. Native addon compiled with `[email protected]`.
+- AgentDB tarball SHA512:
+ `ixjySLNWFDe3G6EiM1+0xKvVB9nWgeSMM7+hkgnQ4C5+trWfukDRYj3G/yU3l4xc/9vpAkLFQS389glsGbH0BQ==`
+- `AGENTDB_DISABLE_TRANSFORMERS=1`: precomputed vectors, no model download or
+ paid embedding service. An initial run without this flag timed out in the
+ existing 10-second initialization hook; deterministic reruns below passed.
+
+The native suite spies on the real class to assert `HNSWIndex.search()` actually
+executes. It does not replace the implementation on happy paths. Fault tests
+inject build/search errors; optional-capability tests explicitly simulate
+missing exports/addons/package. `forceWasm` fallback uses a real AgentDB
+instance. `RUFLO_REQUIRE_NATIVE_HNSW=1` makes native absence fail, not skip.
+
+## Results
+
+| Check | Baseline | Candidate |
+| --- | ---: | ---: |
+| Existing full memory suite | 626/626 pass | 626/626 pass |
+| New targeted regressions | 4 pass, 15 fail | 19/19 pass |
+| Candidate full memory suite | n/a | 645/645 pass, 37 files |
+| Focused backend + new native/optional + hybrid + provider | n/a | 81/81 pass |
+| TypeScript `tsc --noEmit` | not separately run | pass |
+| TypeScript package build | pass | pass |
+| `git diff --check` | n/a | pass |
+
+A full-suite run during final review had one unrelated existing timing guard
+miss: `AgentMemoryScope: resolve 10k paths` took 200.63 ms against a 200 ms
+ceiling. No test or threshold was changed; the unchanged full rerun passed
+645/645. This transient result is disclosed rather than omitted.
+
+Coverage includes actual nearest-neighbor IDs/scores, repeated reads, appends,
+replacement, regenerated-embedding updates, deletion, namespace clearing, empty
+indexes, same-instance shutdown/reinitialize, exact filtered top-k, zero
+thresholds/vectors, oversized `k`, dimension mismatch, extreme finite vectors,
+unsafe native configuration, concurrent `ef`, mutation during asynchronous
+build, expiry parity, mutable input/result/query vectors, and degraded stats/health on missing or failed native capability.
+
+## Bounded benchmark
+
+Identical `benchmarks/agentdb-hnsw.mjs` run in baseline and candidate checkouts
+against built package code. Seed 4249; 1,024 vectors × 64 dimensions; 100 queries;
+`k=10`; 10 warmups. Query vectors differ from corpus vectors. Recall is compared
+with a separately computed exact cosine top-10 for every query.
+
+| Measurement | Baseline dead-controller + exact fallback | Candidate real native HNSW |
+| --- | ---: | ---: |
+| Native index built | no | yes |
+| First search, including index build | 3.671 ms | 97.916 ms |
+| Warm search p50 | 0.433 ms | 0.104 ms |
+| Warm search p95 | 0.722 ms | 0.267 ms |
+| Recall@10 | 1.000 | 1.000 |
+
+This is one bounded run, not a production or universal speedup claim.
+The baseline's existing false-error logging remains enabled. No performance
+threshold is asserted in tests. Cold searches and searches after mutations
+pay full rebuild cost; ties/ANN ordering need not match exact search.
+Native allocation size is not exposed, so `hnswStats.memoryUsage` is unknown
+(represented as zero by the existing stats interface), not a measured zero.
+
+Source/build SHA256 for the reported candidate:
+
+- `src/agentdb-backend.ts`:
+ `c64ac41f2a2e5fcc7a3c62206c52a9692f5a067a132a6d923ac565276c187100`
+- `src/agentdb-hnsw-adapter.ts`:
+ `f41524d5c4b63c0c9023faae4f5feb0e5692117b6e8a1aaebe48dd97f62e1670`
+- `benchmarks/agentdb-hnsw.mjs`:
+ `011d43ac9492c36e53c7379a36afda6f6f4f67426166c06778d33aa7736177d8`
+- `dist/agentdb-backend.js`:
+ `6b4889f95ade01a6de29bb91d2c0b71448dcf15b55cb97ca952c2ac57882f60d`
+- `dist/agentdb-hnsw-adapter.js`:
+ `8233518c024999675a8651fe393e09a5323cc749fad8233cebd37308a00d7122`
+
+The benchmark prints these hashes, environment, parameters and metrics itself.
+
+## Reproduction
+
+From `v3`, install the checked-in lockfile with pnpm 8:
+
+```sh
+npx --yes [email protected] --filter @claude-flow/memory... install --frozen-lockfile --ignore-scripts
+```
+
+Build optional `hnswlib-node` and `better-sqlite3` before requiring native tests.
+In this environment hnswlib was built from its installed package directory with
+the npm-distributed `[email protected]` and writable `--devdir`; better-sqlite3 was
+rebuilt with `npm rebuild --prefix .` in its installed package directory.
+Do not treat a skipped native suite as verification of the native path.
+Build `@claude-flow/security` first for its workspace declarations.
+
+From `v3/@claude-flow/memory`:
+
+```sh
+RUFLO_REQUIRE_NATIVE_HNSW=1 AGENTDB_DISABLE_TRANSFORMERS=1 ../../node_modules/.bin/vitest run
+../../node_modules/.bin/tsc --noEmit
+../../node_modules/.bin/tsc
+AGENTDB_DISABLE_TRANSFORMERS=1 node benchmarks/agentdb-hnsw.mjs
+```
+
+For red/green comparison, copy only the two new test files into an isolated
+baseline checkout and run them against unchanged baseline source. Run the
+existing baseline suite excluding those two files. For benchmark comparison,
+copy the same benchmark script into baseline, build, then run it there.
+
+## Explicit unresolved boundary
+
+This change does **not** fix the separate pre-existing AgentDBBackend persistence
+contract. The real dependency exposes `database.prepare(...).run/get/all`, while
+the old storage paths require `database.run/get`; initialization also does not
+hydrate entries. A real disk-backed probe returned:
+
+| Probe | Baseline | Candidate |
+| --- | ---: | ---: |
+| In-memory count before shutdown | 1 | 1 |
+| Count after new instance opens the same database path | 0 | 0 |
+
+The same-instance lifecycle test is not a persisted-restart test. This PR does
+not claim process-restart durability, ship an npm package, update CLI pins,
+merge the cleanup PR, or cover the unrelated capability-signing scan in #4249.
+Whole-repository CI is separate from the package verification above.
diff --git a/v3/@claude-flow/memory/docs/AGENTDB-INTEGRATION.md b/v3/@claude-flow/memory/docs/AGENTDB-INTEGRATION.md
index 0ec76c19a4..c6f3717025 100644
--- a/v3/@claude-flow/memory/docs/AGENTDB-INTEGRATION.md
+++ b/v3/@claude-flow/memory/docs/AGENTDB-INTEGRATION.md
@@ -2,7 +2,23 @@
## Overview
-The V3 memory module now integrates with **[email protected]** to provide high-performance vector search capabilities with HNSW indexing (150x-12,500x faster than brute-force approaches).
+`AgentDBBackend` integrates with **[email protected]**. Its optional native
+search path uses the separately exported `HNSWIndex`, not
+`AgentDB.getController('hnsw')` (which does not exist in that version).
+
+The index is built lazily from this backend's in-memory entries. Store, update,
+replacement and deletion invalidate it; the next unfiltered search rebuilds a
+fresh index. This avoids the dependency's duplicate-ID and deletion-tombstone
+semantics. Repeated read-only searches reuse the index. Write-heavy workloads
+pay rebuild cost; no universal speedup is promised.
+
+**Persistence boundary:** this adapter does not add disk persistence or hydrate
+`AgentDBBackend` after a process restart. The pre-existing storage adapter uses
+`database.run/get` rather than the dependency's `prepare(...).run/get/all`, and
+startup does not load entries. A same-instance shutdown/reinitialize retains
+its in-memory entries, but reopening the same database in a new instance does
+not restore them. Do not use this change as evidence of durable memory.
+See the [verification receipt](./AGENTDB-HNSW-4249-VERIFICATION.md).
## Features
@@ -10,27 +26,36 @@ The V3 memory module now integrates with **[email protected]** to provide
The `AgentDBBackend` class provides:
-- **HNSW Vector Search**: Approximate nearest neighbor search with sub-millisecond query times
+- **HNSW Vector Search**: Optional approximate cosine nearest-neighbor search
- **Graceful Fallback**: Works without native dependencies (hnswlib-node)
-- **Optional Dependency Handling**: Automatically falls back to pure JavaScript/WASM if native bindings unavailable
+- **Optional Dependency Handling**: Exact in-memory cosine search when native search is unavailable
- **Hybrid Integration**: Seamlessly works with `HybridBackend` for combined SQLite + AgentDB queries
-### Performance Targets
-
-Based on ADR-006 and ADR-009:
-
-- **150x-12,500x** faster vector search compared to brute-force
-- **Sub-millisecond** query latency for k-NN search
-- **Automatic backend selection**: Native hnswlib → ruvector → WASM fallback
+### Search behavior
+
+- Native HNSW is used in `auto`/`hnswlib` mode when the export and addon are available.
+- `forceWasm: true` and explicit `vectorBackend: 'ruvector'` disable this adapter;
+ the AgentDB controllers' vector backend is separate from these memory entries.
+- Filtered searches, including semantic queries with TTL-bearing entries, are
+ exact to avoid top-k underfill or expiry differences between backends. Filter keys are never
+ interpolated into the dependency's SQL filter builder.
+- Missing exports/addons, unsafe tuning, dimension mismatch and native failures
+ fall back to exact cosine search. A failed native instance is retried only on
+ reinitialization, not on every query.
+- Entry objects and embedding/tag/reference arrays are detached on input and
+ output. Use `store`/`update` to change indexed data; arbitrary nested metadata
+ values are not deep-cloned. Query vectors are copied before asynchronous work.
+- HNSW is approximate; equal-score ordering is not guaranteed. Scores and
+ thresholds use this backend's cosine implementation.
## Installation
```bash
# Core package (required)
-npm install [email protected]
+npm install [email protected]
# Optional native dependencies for maximum performance
-npm install hnswlib-node@^3.0.0 better-sqlite3@^11.0.0
+npm install hnswlib-node@^3.0.0 better-sqlite3@^12.9.0
```
## Usage
@@ -86,7 +111,7 @@ await memory.initialize();
// Structured queries go to SQLite
const user = await memory.getByKey('users', '[email protected]');
-// Semantic queries go to AgentDB (150x faster)
+// Semantic queries use native HNSW when available, otherwise exact cosine
const similar = await memory.querySemantic({
content: 'authentication patterns',
k: 10,
@@ -206,63 +231,26 @@ interface AgentDBBackendConfig {
## Graceful Degradation
-The backend handles missing dependencies gracefully:
+The backend handles missing dependencies gracefully. `isAvailable()` reports
+AgentDB initialization, **not** whether a native memory index exists.
```typescript
-// 1. Try native hnswlib (fastest)
-// 2. Fallback to ruvector (fast, pure JS)
-// 3. Fallback to WASM (compatible)
-// 4. Fallback to in-memory brute-force (always works)
-
const backend = new AgentDBBackend();
await backend.initialize();
-
-// Check availability
-if (backend.isAvailable()) {
- console.log('Using AgentDB with HNSW');
-} else {
- console.log('Using fallback in-memory storage');
-}
+// hnswStats is present only for a current, successfully built native index.
+const stats = await backend.getStats();
+console.log(stats.hnswStats ? 'Native HNSW index built' : 'Exact cosine fallback available');
```
-## Performance Metrics
-
-### Benchmarks (from [email protected])
-
-| Operation | Brute Force | HNSW (hnswlib) | Speedup |
-|-----------|-------------|----------------|---------|
-| 10k vectors, k=10 | 150ms | 1ms | 150x |
-| 100k vectors, k=10 | 1500ms | 2ms | 750x |
-| 1M vectors, k=10 | 15000ms | 3ms | 5000x |
-
-### Memory Usage
-
-- **No quantization**: ~4 bytes per dimension per vector
-- **8-bit quantization**: ~1 byte per dimension (4x reduction)
-- **4-bit quantization**: ~0.5 bytes per dimension (8x reduction)
-
-## Advanced Features
+## Performance and supported metrics
-### Vector Quantization
+Use `node benchmarks/agentdb-hnsw.mjs` after building the package for a bounded,
+seeded end-to-end comparison. The [verification receipt](./AGENTDB-HNSW-4249-VERIFICATION.md)
+reports both warm search latency and first-search build cost, alongside recall.
+Historical 150x–12,500x figures are not evidence for this backend.
-```typescript
-const backend = new AgentDBBackend({
- // Enable quantization for 50-75% memory reduction
- quantization: {
- type: 'scalar',
- bits: 8, // 4, 8, or 16
- },
-});
-```
-
-### Custom Distance Metrics
-
-```typescript
-const backend = new AgentDBBackend({
- vectorBackend: 'hnswlib',
- distanceMetric: 'cosine', // 'cosine', 'euclidean', 'dot'
-});
-```
+This adapter uses cosine only. It does not implement quantization or additional
+distance metrics; the separate local `HNSWIndex` API is a different component.
### Health Monitoring
@@ -321,21 +309,21 @@ console.log('Failed:', result.totalFailed);
### AgentDB not available
```
-AgentDB not available. Install [email protected] for vector search support.
+AgentDB not available. Install [email protected] for vector search support.
```
**Solution**: Install agentdb:
```bash
-npm install [email protected]
+npm install [email protected]
```
### Native bindings failed
```
-Failed to load hnswlib-node, falling back to WASM
+Native HNSW index is not built; exact cosine fallback is available
```
-**Solution**: This is normal. The system automatically falls back to WASM. For maximum performance, install build tools:
+**Solution**: This is normal. This memory search path falls back to exact in-memory cosine search. For maximum performance, install build tools:
```bash
# Ubuntu/Debian
@@ -352,15 +340,15 @@ npm install hnswlib-node@^3.0.0
```typescript
const backend = new AgentDBBackend({
- forceWasm: true, // Skip native bindings
+ forceWasm: true, // Disable this native memory-search adapter
});
```
## Testing
```bash
-# Run AgentDB backend tests
-npm test -- agentdb-backend.test.ts
+# Run focused tests; require the real native addon rather than silently skipping
+RUFLO_REQUIRE_NATIVE_HNSW=1 AGENTDB_DISABLE_TRANSFORMERS=1 npm test -- agentdb-backend.test.ts agentdb-backend-hnsw.test.ts agentdb-backend-optional.test.ts
# Run all memory tests
npm test
diff --git a/v3/@claude-flow/memory/package.json b/v3/@claude-flow/memory/package.json
index d4c8a5ea59..afa4e0837c 100644
--- a/v3/@claude-flow/memory/package.json
+++ b/v3/@claude-flow/memory/package.json
@@ -1,6 +1,6 @@
{
"name": "@claude-flow/memory",
- "version": "3.0.7",
+ "version": "3.0.8",
"type": "module",
"description": "Memory module - AgentDB unification, HNSW indexing, vector search, hybrid SQLite+AgentDB backend (ADR-009)",
"main": "dist/index.js",
diff --git a/v3/@claude-flow/memory/src/agentdb-backend-hnsw.test.ts b/v3/@claude-flow/memory/src/agentdb-backend-hnsw.test.ts
new file mode 100644
index 0000000000..412fe727de
--- /dev/null
+++ b/v3/@claude-flow/memory/src/agentdb-backend-hnsw.test.ts
@@ -0,0 +1,289 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { HNSWIndex, isHnswlibAvailable } from 'agentdb';
+import { AgentDBBackend } from './agentdb-backend.js';
+import { createDefaultEntry } from './types.js';
+
+const nativeAvailable = await isHnswlibAvailable();
+const required = process.env.RUFLO_REQUIRE_NATIVE_HNSW === '1';
+const vector = (...values: number[]) => new Float32Array(values);
+const entry = (id: string, embedding: Float32Array, namespace = 'test') =>
+ ({ ...createDefaultEntry({ key: id, content: id, namespace }), id, embedding });
+
+// A normal install can omit the optional native addon. The verification job
+// sets RUFLO_REQUIRE_NATIVE_HNSW=1 so absence is a failure, never a silent skip.
+describe.skipIf(!nativeAvailable && !required)('AgentDBBackend real native HNSW', () => {
+ let backend: AgentDBBackend;
+
+ beforeEach(async () => {
+ expect(nativeAvailable).toBe(true);
+ vi.stubEnv('AGENTDB_DISABLE_TRANSFORMERS', '1');
+ backend = new AgentDBBackend({
+ dbPath: ':memory:',
+ vectorDimension: 3,
+ vectorBackend: 'hnswlib',
+ embeddingGenerator: async (content) =>
+ content === 'changed' ? vector(0, 1, 0) : vector(1, 0, 0),
+ });
+ await backend.initialize();
+ expect(backend.isAvailable()).toBe(true);
+ });
+
+ afterEach(async () => {
+ await backend?.shutdown();
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ });
+
+ it('builds and searches the real export without asking for an imaginary controller', async () => {
+ const nativeSearch = vi.spyOn(HNSWIndex.prototype, 'search');
+ const getController = vi.spyOn(backend.getAgentDB(), 'getController');
+ const errors = vi.spyOn(console, 'error');
+ await backend.bulkInsert([
+ entry('east', vector(1, 0, 0)),
+ entry('near-east', vector(0.9, 0.1, 0)),
+ entry('north', vector(0, 1, 0)),
+ entry('west', vector(-1, 0, 0)),
+ ]);
+ const results = await backend.search(vector(1, 0, 0), { k: 2 });
+ expect(results.map((r) => r.entry.id)).toEqual(['east', 'near-east']);
+ expect(results[0]).toMatchObject({ score: 1, distance: 0 });
+ expect(results[1].score).toBeCloseTo(0.993883734, 6);
+ expect(nativeSearch).toHaveBeenCalledOnce();
+ expect(getController).not.toHaveBeenCalledWith('hnsw');
+ expect(errors.mock.calls.some(([message]) => String(message).includes('HNSW search failed'))).toBe(false);
+ expect((await backend.getStats()).hnswStats?.vectorCount).toBe(4);
+ expect((await backend.healthCheck()).components.index.status).toBe('healthy');
+ });
+
+ it('reuses a built index, but rebuilds on append, replacement, update and deletion', async () => {
+ const build = vi.spyOn(HNSWIndex.prototype, 'buildIndex');
+ await backend.store(entry('a', vector(1, 0, 0)));
+ await backend.store(entry('b', vector(0.9, 0.1, 0)));
+ await backend.search(vector(1, 0, 0), { k: 1 });
+ await backend.search(vector(1, 0, 0), { k: 1 });
+ expect(build).toHaveBeenCalledTimes(1);
+ await backend.store(entry('c', vector(0, 0, 1)));
+ expect((await backend.search(vector(0, 0, 1), { k: 1 }))[0].entry.id).toBe('c');
+ await backend.store(entry('a', vector(-1, 0, 0), 'moved'));
+ expect(await backend.getByKey('test', 'a')).toBeNull();
+ expect((await backend.search(vector(1, 0, 0), { k: 1 }))[0].entry.id).toBe('b');
+ await backend.update('b', { content: 'changed' });
+ expect((await backend.search(vector(0, 1, 0), { k: 1 }))[0].entry.id).toBe('b');
+ await backend.delete('b');
+ const results = await backend.search(vector(0, 1, 0), { k: 99 });
+ expect(results.map((r) => r.entry.id).sort()).toEqual(['a', 'c']);
+ expect(new Set(results.map((r) => r.entry.id)).size).toBe(2);
+ expect((await backend.getStats()).hnswStats?.vectorCount).toBe(2);
+ expect(build).toHaveBeenCalledTimes(5);
+ });
+
+ it('clears the last point and rebuilds from entries on same-instance reinitialize', async () => {
+ await backend.store(entry('a', vector(1, 0, 0)));
+ await backend.search(vector(1, 0, 0), { k: 1 });
+ await backend.clearNamespace('test');
+ expect(await backend.search(vector(1, 0, 0), { k: 1 })).toEqual([]);
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ await backend.store(entry('loaded', vector(0, 1, 0)));
+ await backend.shutdown();
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ await backend.initialize();
+ expect((await backend.search(vector(0, 1, 0), { k: 1 }))[0].entry.id).toBe('loaded');
+ });
+
+ it('keeps filtered top-k exact and honors zero thresholds', async () => {
+ const search = vi.spyOn(HNSWIndex.prototype, 'search');
+ await backend.bulkInsert([
+ entry('outside-1', vector(1, 0, 0), 'other'),
+ entry('outside-2', vector(0.99, 0.01, 0), 'other'),
+ entry('inside', vector(0.8, 0.2, 0)),
+ entry('opposite', vector(-1, 0, 0)),
+ ]);
+ const filtered = await backend.search(vector(1, 0, 0), {
+ k: 1, filters: { type: 'semantic', namespace: 'test', limit: 1 },
+ });
+ expect(filtered.map((r) => r.entry.id)).toEqual(['inside']);
+ expect(search).not.toHaveBeenCalled();
+ const threshold = await backend.search(vector(1, 0, 0), { k: 10, threshold: 0 });
+ expect(threshold.map((r) => r.entry.id)).not.toContain('opposite');
+ expect(search).toHaveBeenCalledOnce();
+ });
+
+ it('uses native search for semantic queries without filtering constraints', async () => {
+ const search = vi.spyOn(HNSWIndex.prototype, 'search');
+ await backend.store(entry('a', vector(1, 0, 0)));
+ expect((await backend.query({
+ type: 'semantic', embedding: vector(1, 0, 0), limit: 1,
+ }))[0].id).toBe('a');
+ expect(search).toHaveBeenCalledOnce();
+ });
+
+ it('keeps default expiry eligibility consistent with exact semantic search', async () => {
+ const search = vi.spyOn(HNSWIndex.prototype, 'search');
+ await backend.store({ ...entry('expired', vector(1, 0, 0)), expiresAt: Date.now() - 1 });
+ await backend.store(entry('live', vector(0.9, 0.1, 0)));
+ const query = { type: 'semantic' as const, embedding: vector(1, 0, 0), limit: 1 };
+ expect((await backend.query(query)).map((r) => r.id)).toEqual(['live']);
+ expect(search).not.toHaveBeenCalled();
+ expect((await backend.query({ ...query, includeExpired: true })).map((r) => r.id)).toEqual(['expired']);
+ await backend.delete('expired');
+ expect((await backend.query(query)).map((r) => r.id)).toEqual(['live']);
+ expect(search).toHaveBeenCalledOnce();
+ await backend.update('live', { expiresAt: Date.now() - 1 });
+ expect(await backend.query(query)).toEqual([]);
+ // Replacing a TTL entry with an entry without expiry restores native use.
+ await backend.store(entry('live', vector(0.9, 0.1, 0)));
+ expect((await backend.query(query)).map((r) => r.id)).toEqual(['live']);
+ expect(search).toHaveBeenCalledTimes(2);
+ });
+
+ it('owns indexed vectors instead of exposing mutable input or result aliases', async () => {
+ const a = entry('a', vector(1, 0, 0));
+ const b = entry('b', vector(0, 1, 0));
+ await backend.bulkInsert([a, b]);
+ const east = () => backend.search(vector(1, 0, 0), { k: 1 });
+ expect((await east())[0].entry.id).toBe('a');
+ a.embedding.set([-1, 0, 0]);
+ b.embedding.set([1, 0, 0]);
+ a.expiresAt = 0;
+ (await backend.get('a'))!.embedding!.set([-1, 0, 0]);
+ (await east())[0].entry.embedding!.set([-1, 0, 0]);
+ (await backend.query({ type: 'exact', key: 'a', limit: 1 }))[0].embedding!.set([-1, 0, 0]);
+ const result = await east();
+ expect(result[0]).toMatchObject({ entry: { id: 'a' }, score: 1 });
+ expect((await backend.query({ type: 'semantic', embedding: vector(1, 0, 0), limit: 1 }))[0].id).toBe('a');
+ await backend.delete('b');
+ const updated = await backend.update('a', { content: 'changed' });
+ updated!.embedding!.set([0, -1, 0]);
+ expect((await backend.search(vector(0, 1, 0), { k: 1 }))[0].score).toBe(1);
+ });
+
+ it('normalizes extreme finite values before native cosine search', async () => {
+ await backend.bulkInsert([
+ entry('huge', vector(3e38, 1e38, 0)),
+ entry('tiny', vector(0, 1e-38, 0)),
+ entry('other', vector(0, 0, 1)),
+ ]);
+ expect((await backend.search(vector(3e38, 1e38, 0), { k: 1 }))[0].entry.id).toBe('huge');
+ expect((await backend.search(vector(0, 1e-38, 0), { k: 1 }))[0].entry.id).toBe('tiny');
+ });
+
+ it('rejects unsafe native tuning without constructing an index', async () => {
+ await backend.shutdown();
+ backend = new AgentDBBackend({
+ dbPath: ':memory:', vectorDimension: 3, hnswM: 0,
+ });
+ await backend.initialize();
+ const build = vi.spyOn(HNSWIndex.prototype, 'buildIndex');
+ await backend.store(entry('a', vector(1, 0, 0)));
+ expect((await backend.search(vector(1, 0, 0), { k: 1 }))[0].entry.id).toBe('a');
+ expect(build).not.toHaveBeenCalled();
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ });
+
+ it('serializes per-query ef changes across concurrent native searches', async () => {
+ await backend.store(entry('a', vector(1, 0, 0)));
+ await backend.search(vector(1, 0, 0), { k: 1 });
+ const original = HNSWIndex.prototype.search;
+ const observed: number[] = [];
+ vi.spyOn(HNSWIndex.prototype, 'search').mockImplementation(async function (this: HNSWIndex, ...args) {
+ await Promise.resolve();
+ observed.push(this.getStats().efSearch);
+ return original.apply(this, args);
+ });
+ await Promise.all([
+ backend.search(vector(1, 0, 0), { k: 1, ef: 20 }),
+ backend.search(vector(1, 0, 0), { k: 1, ef: 200 }),
+ ]);
+ expect(observed).toEqual([20, 200]);
+ });
+
+ it('handles empty, oversized-k, zero vectors and mismatched dimensions safely', async () => {
+ expect(await backend.search(vector(1, 0, 0), { k: 1 })).toEqual([]);
+ await backend.store(entry('zero', vector(0, 0, 0)));
+ const results = await backend.search(vector(1, 0, 0), { k: 100 });
+ expect(results).toHaveLength(1);
+ expect(results[0]).toMatchObject({ score: 0, distance: 1 });
+ expect(await backend.search(vector(1, 0, 0), { k: 0 })).toEqual([]);
+ await backend.store(entry('legacy-dimension', vector(1, 0)));
+ const legacy = await backend.search(vector(1, 0), { k: 2 });
+ expect(legacy.some((r) => r.entry.id === 'legacy-dimension')).toBe(true);
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ });
+
+ it('falls back after native build failure, without retrying on every query', async () => {
+ const build = vi.spyOn(HNSWIndex.prototype, 'buildIndex').mockRejectedValueOnce(new Error('native unavailable'));
+ await backend.store(entry('a', vector(1, 0, 0)));
+ for (let i = 0; i < 2; i++) {
+ expect((await backend.search(vector(1, 0, 0), { k: 1 }))[0].entry.id).toBe('a');
+ }
+ expect(build).toHaveBeenCalledOnce();
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ expect((await backend.healthCheck()).components.index.status).toBe('degraded');
+ });
+
+ it('falls back after native search failure and stops advertising a live index', async () => {
+ const search = vi.spyOn(HNSWIndex.prototype, 'search').mockRejectedValueOnce(new Error('native failure'));
+ await backend.store(entry('a', vector(1, 0, 0)));
+ for (let i = 0; i < 2; i++) {
+ expect((await backend.search(vector(1, 0, 0), { k: 1 }))[0].entry.id).toBe('a');
+ }
+ expect(search).toHaveBeenCalledOnce();
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ });
+
+ it('rejects snapshots invalidated during asynchronous index construction', async () => {
+ const original = HNSWIndex.prototype.buildIndex;
+ let release!: () => void;
+ const gate = new Promise<void>((resolve) => { release = resolve; });
+ const build = vi.spyOn(HNSWIndex.prototype, 'buildIndex').mockImplementationOnce(async function (this: HNSWIndex, ...args) {
+ await gate;
+ return original.apply(this, args);
+ });
+ await backend.store(entry('old', vector(1, 0, 0)));
+ const pending = backend.search(vector(1, 0, 0), { k: 1 });
+ await vi.waitFor(() => expect(build).toHaveBeenCalledOnce());
+ await backend.delete('old');
+ await backend.store(entry('new', vector(0.9, 0.1, 0)));
+ release();
+ expect((await pending).map((r) => r.entry.id)).toEqual(['new']);
+ expect((await backend.search(vector(1, 0, 0), { k: 1 }))[0].entry.id).toBe('new');
+ });
+
+ it('snapshots query vectors before asynchronous native construction', async () => {
+ const original = HNSWIndex.prototype.buildIndex;
+ let release!: () => void;
+ const gate = new Promise<void>((resolve) => { release = resolve; });
+ const build = vi.spyOn(HNSWIndex.prototype, 'buildIndex').mockImplementationOnce(async function (this: HNSWIndex, ...args) {
+ await gate;
+ return original.apply(this, args);
+ });
+ await backend.store(entry('a', vector(1, 0, 0)));
+ await backend.store(entry('b', vector(0, 1, 0)));
+ const query = vector(1, 0, 0);
+ const pending = backend.search(query, { k: 1 });
+ await vi.waitFor(() => expect(build).toHaveBeenCalledOnce());
+ query.fill(Number.NaN);
+ release();
+ expect((await pending)[0]).toMatchObject({ entry: { id: 'a' }, score: 1, distance: 0 });
+ });
+});
+
+describe('AgentDBBackend explicit non-native mode', () => {
+ it('uses exact fallback under forceWasm with no HNSW capability claim', async () => {
+ vi.stubEnv('AGENTDB_DISABLE_TRANSFORMERS', '1');
+ const search = vi.spyOn(HNSWIndex.prototype, 'search');
+ const backend = new AgentDBBackend({ dbPath: ':memory:', forceWasm: true, vectorDimension: 3 });
+ try {
+ await backend.initialize();
+ await backend.store(entry('a', vector(1, 0, 0)));
+ expect((await backend.search(vector(1, 0, 0), { k: 1 }))[0].entry.id).toBe('a');
+ expect(search).not.toHaveBeenCalled();
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ expect((await backend.healthCheck()).components.index.status).toBe('degraded');
+ } finally {
+ await backend.shutdown();
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ }
+ });
+});
diff --git a/v3/@claude-flow/memory/src/agentdb-backend-optional.test.ts b/v3/@claude-flow/memory/src/agentdb-backend-optional.test.ts
new file mode 100644
index 0000000000..2d09e36637
--- /dev/null
+++ b/v3/@claude-flow/memory/src/agentdb-backend-optional.test.ts
@@ -0,0 +1,52 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { createDefaultEntry } from './types.js';
+
+const optional = vi.hoisted(() => ({ mode: 'missing-export' }));
+vi.mock('agentdb', async (importOriginal) => {
+ if (optional.mode === 'missing-package') throw new Error('agentdb is not installed');
+ const actual = await importOriginal<typeof import('agentdb')>();
+ return {
+ ...actual,
+ HNSWIndex: optional.mode === 'missing-export' ? undefined : actual.HNSWIndex,
+ isHnswlibAvailable: async () => false,
+ };
+});
+
+describe('AgentDBBackend optional native capability', () => {
+ beforeEach(() => {
+ vi.resetModules();
+ vi.stubEnv('AGENTDB_DISABLE_TRANSFORMERS', '1');
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ });
+
+ for (const mode of ['missing-export', 'unavailable-addon', 'missing-package']) {
+ it(`returns exact neighbors with ${mode} rather than empty results or false HNSW stats`, async () => {
+ optional.mode = mode;
+ const { AgentDBBackend } = await import('./agentdb-backend.js');
+ const backend = new AgentDBBackend({ dbPath: ':memory:', vectorDimension: 2 });
+ try {
+ await backend.initialize();
+ const entry = {
+ ...createDefaultEntry({ key: 'kept', content: 'kept' }),
+ embedding: new Float32Array([1, 0]),
+ };
+ await backend.store(entry);
+ const errors = vi.spyOn(console, 'error');
+ expect((await backend.search(new Float32Array([1, 0]), { k: 1 }))[0]).toMatchObject({
+ entry: { id: entry.id }, score: 1, distance: 0,
+ });
+ expect((await backend.getStats()).hnswStats).toBeUndefined();
+ expect((await backend.healthCheck()).components.index.status).toBe('degraded');
+ expect(errors.mock.calls.some(([message]) =>
+ String(message).includes('HNSW search failed'),
+ )).toBe(false);
+ } finally {
+ await backend.shutdown();
+ }
+ });
+ }
+});
diff --git a/v3/@claude-flow/memory/src/agentdb-backend.ts b/v3/@claude-flow/memory/src/agentdb-backend.ts
index 894d066e66..60c0473a1f 100644
--- a/v3/@claude-flow/memory/src/agentdb-backend.ts
+++ b/v3/@claude-flow/memory/src/agentdb-backend.ts
@@ -1,9 +1,9 @@
/**
- * AgentDB Backend - Integration with [email protected]
+ * AgentDB Backend - Integration with [email protected]
*
* Provides IMemoryBackend implementation using AgentDB with:
- * - HNSW vector search (150x-12,500x faster than brute-force)
- * - Native or WASM backend support with graceful fallback
+ * - Optional native HNSW vector search through the real HNSWIndex export
+ * - Exact cosine fallback when native search is unavailable
* - Optional dependency handling (works without hnswlib-node)
* - Seamless integration with HybridBackend
*
@@ -12,6 +12,7 @@
import { withAgentdbLockRecovery, closeWithAgentdbLockRecovery } from './agentdb-lock-guard.js';
import { EventEmitter } from 'node:events';
+import { AgentDBHnswAdapter } from './agentdb-hnsw-adapter.js';
import { safeJsonParse } from './json-security.js';
import { useHostSqliteDriver } from './agentdb-native-driver.js';
import {
@@ -30,7 +31,6 @@ import {
generateMemoryId,
createDefaultEntry,
CacheStats,
- HNSWStats,
} from './types.js';
import {
AgentDbRetrievalGuard,
@@ -78,7 +78,7 @@ export interface AgentDBBackendConfig {
/** Force WASM backend (skip native hnswlib) */
forceWasm?: boolean;
- /** Vector backend: 'auto', 'ruvector', 'hnswlib' */
+ /** AgentDB controller backend. Native memory search uses hnswlib in auto/hnswlib mode. */
vectorBackend?: 'auto' | 'ruvector' | 'hnswlib';
/** Vector dimensions (default: 1536) */
@@ -134,11 +134,11 @@ const DEFAULT_CONFIG: Required<
* AgentDB Backend
*
* Integrates AgentDB for vector search with the V3 memory system.
- * Provides 150x-12,500x faster search compared to brute-force approaches.
+ * Native HNSW is an approximate search optimization, not a persistence layer.
*
* Features:
* - HNSW indexing for fast approximate nearest neighbor search
- * - Automatic fallback: native hnswlib → ruvector → WASM
+ * - Exact cosine fallback without native hnswlib, in forceWasm mode, or for filters
* - Graceful handling of optional native dependencies
* - Semantic search with filtering
* - Compatible with HybridBackend for combined SQLite+AgentDB queries
@@ -153,16 +153,14 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
private agentdb: any;
private initialized: boolean = false;
private available: boolean = false;
+ private hnsw?: AgentDBHnswAdapter;
private readonly retrievalGuard: AgentDbRetrievalGuard | null;
// In-memory storage for compatibility
private entries: Map<string, MemoryEntry> = new Map();
private namespaceIndex: Map<string, Set<string>> = new Map();
private keyIndex: Map<string, string> = new Map();
-
- // O(1) bidirectional lookup for numeric ID <-> string ID (fixes O(n) linear scan)
- private numericToStringIdMap: Map<number, string> = new Map();
- private stringToNumericIdMap: Map<string, number> = new Map();
+ private expiringEntryIds: Set<string> = new Set();
// Performance tracking
private stats = {
@@ -234,6 +232,18 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
// Create memory_entries table if it doesn't exist
await this.createSchema();
+ // agentdb has no getController('hnsw'). Its standalone HNSWIndex must be
+ // built from our own entries; its controller vector backend is unrelated.
+ if (!this.config.forceWasm && this.config.vectorBackend !== 'ruvector' &&
+ HNSWIndex && isHnswlibAvailable && await isHnswlibAvailable()) {
+ this.hnsw = new AgentDBHnswAdapter(HNSWIndex, {
+ dimension: this.config.vectorDimension,
+ M: this.config.hnswM,
+ efConstruction: this.config.hnswEfConstruction,
+ efSearch: this.config.hnswEfSearch,
+ });
+ }
+
this.initialized = true;
this.emit('initialized', {
backend: this.agentdb.vectorBackendName,
@@ -253,6 +263,9 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
async shutdown(): Promise<void> {
if (!this.initialized) return;
+ this.hnsw?.invalidate();
+ this.hnsw = undefined;
+
if (this.agentdb) {
// Clears a provably stale lock first; a persist failure is surfaced
// (AgentdbLockError), never swallowed.
@@ -260,6 +273,8 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
await closeWithAgentdbLockRecovery(this.config.dbPath, () => this.agentdb.close());
} finally {
this.initialized = false;
+ this.agentdb = undefined;
+ this.available = false;
this.emit('shutdown');
}
return;
@@ -273,16 +288,24 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
* Store a memory entry
*/
async store(entry: MemoryEntry): Promise<void> {
+ // Own vectors and index-relevant fields. Callers must not be able to mutate
+ // an indexed snapshot through the original entry or a returned result.
+ entry = this.copyEntry(entry);
// Generate embedding if needed
if (entry.content && !entry.embedding && this.config.embeddingGenerator) {
- entry.embedding = await this.config.embeddingGenerator(entry.content);
+ entry.embedding = new Float32Array(await this.config.embeddingGenerator(entry.content));
}
- // Store in-memory for quick access
+ // Replacements must not leave old namespace/key or native vector entries.
+ const previous = this.entries.get(entry.id);
+ if (previous) {
+ this.namespaceIndex.get(previous.namespace)?.delete(entry.id);
+ this.keyIndex.delete(`${previous.namespace}:${previous.key}`);
+ }
this.entries.set(entry.id, entry);
-
- // Register ID mapping for O(1) reverse lookup
- this.registerIdMapping(entry.id);
+ if (entry.expiresAt === undefined) this.expiringEntryIds.delete(entry.id);
+ else this.expiringEntryIds.add(entry.id);
+ this.hnsw?.invalidate();
// Update indexes
this.updateIndexes(entry);
@@ -301,11 +324,12 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
async get(id: string): Promise<MemoryEntry | null> {
// Check in-memory first
const cached = this.entries.get(id);
- if (cached) return cached;
+ if (cached) return this.copyEntry(cached);
// Query AgentDB if available
if (this.agentdb) {
- return this.getFromAgentDB(id);
+ const entry = await this.getFromAgentDB(id);
+ return entry ? this.copyEntry(entry) : null;
}
return null;
@@ -328,17 +352,19 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
const entry = this.entries.get(id);
if (!entry) return null;
+ this.hnsw?.invalidate();
+
// Apply updates
if (update.content !== undefined) {
entry.content = update.content;
// Regenerate embedding if needed
if (this.config.embeddingGenerator) {
- entry.embedding = await this.config.embeddingGenerator(entry.content);
+ entry.embedding = new Float32Array(await this.config.embeddingGenerator(entry.content));
}
}
if (update.tags !== undefined) {
- entry.tags = update.tags;
+ entry.tags = [...update.tags];
}
if (update.metadata !== undefined) {
@@ -351,14 +377,16 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
if (update.expiresAt !== undefined) {
entry.expiresAt = update.expiresAt;
+ this.expiringEntryIds.add(id);
}
if (update.references !== undefined) {
- entry.references = update.references;
+ entry.references = [...update.references];
}
entry.updatedAt = Date.now();
entry.version++;
+ this.hnsw?.invalidate();
// Update in AgentDB
if (this.agentdb) {
@@ -366,7 +394,7 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
}
this.emit('entry:updated', { id });
- return entry;
+ return this.copyEntry(entry);
}
/**
@@ -378,7 +406,8 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
// Remove from indexes
this.entries.delete(id);
- this.unregisterIdMapping(id); // Clean up reverse lookup map
+ this.expiringEntryIds.delete(id);
+ this.hnsw?.invalidate();
this.namespaceIndex.get(entry.namespace)?.delete(id);
const keyIndexKey = `${entry.namespace}:${entry.key}`;
this.keyIndex.delete(keyIndexKey);
@@ -412,7 +441,7 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
this.stats.queryCount++;
this.stats.totalQueryTime += duration;
- return results;
+ return results.map((entry) => this.copyEntry(entry));
}
/**
@@ -422,6 +451,9 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
embedding: Float32Array,
options: SearchOptions
): Promise<SearchResult[]> {
+ // Keep native validation, search and JS score recomputation on the same
+ // vector even if the caller changes its Float32Array during an await.
+ embedding = new Float32Array(embedding);
const startTime = performance.now();
if (!this.agentdb) {
@@ -454,6 +486,7 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
* `CLAUDE_FLOW_RETRIEVAL_GUARD_STRICT`).
*/
private applyRetrievalGuard(results: SearchResult[]): SearchResult[] {
+ results = results.map((result) => ({ ...result, entry: this.copyEntry(result.entry) }));
if (!this.retrievalGuard || !isRetrievalGuardEnabled() || results.length === 0) {
return results;
}
@@ -545,25 +578,7 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
entriesByType[entry.type]++;
}
- // Get HNSW stats if available
- let hnswStats: HNSWStats | undefined;
- if (this.agentdb && HNSWIndex) {
- try {
- const hnsw = this.agentdb.getController('hnsw');
- if (hnsw) {
- const stats = hnsw.getStats();
- hnswStats = {
- vectorCount: stats.numElements || 0,
- memoryUsage: 0,
- avgSearchTime: stats.avgSearchTimeMs || 0,
- buildTime: stats.lastBuildTime || 0,
- compressionRatio: 1.0,
- };
- }
- } catch {
- // HNSW not available
- }
- }
+ const hnswStats = this.hnsw?.getStats();
return {
totalEntries: this.entries.size,
@@ -600,10 +615,9 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
// Check index health
const indexHealth: ComponentHealth = { status: 'healthy', latency: 0 };
- if (!this.agentdb) {
+ if (!this.hnsw?.getStats()) {
indexHealth.status = 'degraded';
- indexHealth.message = 'HNSW index not available';
- recommendations.push('Install agentdb for 150x-12,500x faster vector search');
+ indexHealth.message = 'Native HNSW index is not built; exact cosine fallback is available';
}
// Check cache health
@@ -631,6 +645,16 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
// ===== Private Methods =====
+ private copyEntry(entry: MemoryEntry): MemoryEntry {
+ return {
+ ...entry,
+ embedding: entry.embedding ? new Float32Array(entry.embedding) : undefined,
+ tags: [...entry.tags],
+ metadata: { ...entry.metadata },
+ references: [...entry.references],
+ };
+ }
+
/**
* Create database schema
*/
@@ -739,19 +763,6 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
// AgentDB storage failed - entry is already in-memory
}
- // Add to vector index if HNSW is available
- if (entry.embedding && HNSWIndex) {
- try {
- const hnsw = this.agentdb.getController('hnsw');
- if (hnsw) {
- // Convert string ID to number for HNSW (use hash)
- const numericId = this.stringIdToNumeric(entry.id);
- hnsw.addVector(numericId, entry.embedding);
- }
- } catch {
- // HNSW not available
- }
- }
}
/**
@@ -842,39 +853,27 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
embedding: Float32Array,
options: SearchOptions
): Promise<SearchResult[]> {
- if (!this.agentdb || !HNSWIndex) {
- return [];
+ // Post-filtering native top-k can hide eligible neighbors. Keep filtered
+ // searches exact, and do not pass user filters to HNSWIndex's SQL builder.
+ if (!this.hnsw || this.hasSearchFilters(options.filters)) {
+ return this.bruteForceSearch(embedding, options);
}
+ const ids = await this.hnsw.search(
+ () => Array.from(this.entries.values()), embedding, options.k, options.ef,
+ );
+ if (!ids) return this.bruteForceSearch(embedding, options);
- try {
- const hnsw = this.agentdb.getController('hnsw');
- if (!hnsw) {
- return this.bruteForceSearch(embedding, options);
- }
-
- const results = await hnsw.search(embedding, options.k, {
- threshold: options.threshold,
- });
-
- const searchResults: SearchResult[] = [];
-
- for (const result of results) {
- const id = this.numericIdToString(result.id);
- const entry = await this.get(id);
- if (!entry) continue;
-
- searchResults.push({
- entry,
- score: result.similarity,
- distance: result.distance,
- });
- }
-
- return searchResults;
- } catch (error) {
- console.error('HNSW search failed:', error);
- return this.bruteForceSearch(embedding, options);
+ // Preserve our cosine score/threshold contract (including zero vectors)
+ // instead of exposing native floating-point rounding as a different score.
+ const results: SearchResult[] = [];
+ for (const id of ids) {
+ const entry = this.entries.get(id);
+ if (!entry?.embedding) continue;
+ const score = this.cosineSimilarity(embedding, entry.embedding);
+ if (options.threshold !== undefined && score < options.threshold) continue;
+ results.push({ entry, score, distance: 1 - score });
}
+ return results.sort((a, b) => b.score - a.score);
}
/**
@@ -887,12 +886,12 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
const results: SearchResult[] = [];
for (const entry of this.entries.values()) {
- if (!entry.embedding) continue;
+ if (!entry.embedding || !this.matchesFilters(entry, options.filters)) continue;
const score = this.cosineSimilarity(embedding, entry.embedding);
const distance = 1 - score;
- if (options.threshold && score < options.threshold) continue;
+ if (options.threshold !== undefined && score < options.threshold) continue;
results.push({ entry, score, distance });
}
@@ -994,91 +993,45 @@ export class AgentDBBackend extends EventEmitter implements IMemoryBackend {
};
}
- /**
- * Convert string ID to numeric for HNSW
- */
- private stringIdToNumeric(id: string): number {
- // Check reverse map first — if this ID was collision-probed, the map has the actual numeric ID
- const mapped = this.stringToNumericIdMap.get(id);
- if (mapped !== undefined) return mapped;
-
- // HIGH-05: Dual hash (djb2 + sdbm) for 53-bit space, ~94M entries before collision
- return this.hashStringId(id);
- }
-
- /**
- * Pure hash function for string ID (no map lookup).
- * Used by registerIdMapping for initial hash before collision probing.
- */
- private hashStringId(id: string): number {
- // Hash A: djb2
- let hashA = 5381;
- for (let i = 0; i < id.length; i++) {
- hashA = ((hashA << 5) + hashA + id.charCodeAt(i)) | 0;
- }
-
- // Hash B: sdbm (independent seed/algorithm)
- let hashB = 0;
- for (let i = 0; i < id.length; i++) {
- hashB = id.charCodeAt(i) + ((hashB << 6) + (hashB << 16) - hashB) | 0;
- }
-
- // Combine into a 53-bit safe integer:
- // Use 26 bits from hashA and 27 bits from hashB
- const upper = (Math.abs(hashA) & 0x3FFFFFF); // 26 bits
- const lower = (Math.abs(hashB) & 0x7FFFFFF); // 27 bits
- return upper * 0x8000000 + lower;
+ private hasSearchFilters(filters?: MemoryQuery): boolean {
+ if (!filters) return false;
+ // MemoryQuery excludes expired entries by default. Time can change
+ // eligibility without a write, so keep TTL-bearing semantic queries exact.
+ // Track presence on writes rather than scanning every native read.
+ if (!filters.includeExpired && this.expiringEntryIds.size > 0) return true;
+ // Semantic queries carry type/content/embedding/limit/threshold even when
+ // there are no filtering constraints. Do not disable native search for them.
+ return [
+ filters.namespace, filters.key, filters.keyPrefix, filters.memoryType,
+ filters.accessLevel, filters.ownerId, filters.createdAfter, filters.createdBefore,
+ filters.updatedAfter, filters.updatedBefore, filters.includeExpired,
+ ].some((value) => value !== undefined) ||
+ !!filters.tags?.length || !!(filters.metadata && Object.keys(filters.metadata).length);
}
- /**
- * Convert numeric ID back to string using O(1) reverse lookup
- * PERFORMANCE FIX: Uses pre-built reverse map instead of O(n) linear scan
- */
- private numericIdToString(numericId: number): string {
- // Use O(1) reverse lookup map
- const stringId = this.numericToStringIdMap.get(numericId);
- if (stringId) {
- return stringId;
- }
- // Fallback for unmapped IDs
- return String(numericId);
- }
-
- /**
- * Register string ID in reverse lookup map
- * Called when storing entries to maintain bidirectional mapping
- */
- private registerIdMapping(stringId: string): void {
- const numericId = this.hashStringId(stringId);
- const existing = this.numericToStringIdMap.get(numericId);
- if (existing && existing !== stringId) {
- // HIGH-05: Collision detected — use linear probing fallback
- console.warn(`[HNSW] Hash collision detected: "${stringId}" collides with "${existing}" (numeric: ${numericId})`);
- let fallbackId = numericId + 1;
- while (this.numericToStringIdMap.has(fallbackId)) {
- fallbackId++;
- }
- this.numericToStringIdMap.set(fallbackId, stringId);
- this.stringToNumericIdMap.set(stringId, fallbackId);
- return;
- }
- this.numericToStringIdMap.set(numericId, stringId);
- this.stringToNumericIdMap.set(stringId, numericId);
- }
-
- /**
- * Unregister string ID from reverse lookup map
- * Called when deleting entries
- */
- private unregisterIdMapping(stringId: string): void {
- // Use reverse map for correct numeric ID (may differ from hash due to collision fallback)
- const numericId = this.stringToNumericIdMap.get(stringId) ?? this.stringIdToNumeric(stringId);
- this.numericToStringIdMap.delete(numericId);
- this.stringToNumericIdMap.delete(stringId);
+ /** Apply semantic-search filters before selecting top-k, not after. */
+ private matchesFilters(entry: MemoryEntry, filters?: MemoryQuery): boolean {
+ if (!filters) return true;
+ if (filters.namespace !== undefined && entry.namespace !== filters.namespace) return false;
+ if (filters.key !== undefined && entry.key !== filters.key) return false;
+ if (filters.keyPrefix !== undefined && !entry.key.startsWith(filters.keyPrefix)) return false;
+ if (filters.tags?.some((tag) => !entry.tags.includes(tag))) return false;
+ if (filters.memoryType !== undefined && entry.type !== filters.memoryType) return false;
+ if (filters.accessLevel !== undefined && entry.accessLevel !== filters.accessLevel) return false;
+ if (filters.ownerId !== undefined && entry.ownerId !== filters.ownerId) return false;
+ if (filters.createdAfter !== undefined && entry.createdAt <= filters.createdAfter) return false;
+ if (filters.createdBefore !== undefined && entry.createdAt >= filters.createdBefore) return false;
+ if (filters.updatedAfter !== undefined && entry.updatedAt <= filters.updatedAfter) return false;
+ if (filters.updatedBefore !== undefined && entry.updatedAt >= filters.updatedBefore) return false;
+ if (!filters.includeExpired && entry.expiresAt !== undefined && entry.expiresAt <= Date.now()) return false;
+ if (filters.metadata && Object.entries(filters.metadata).some(
+ ([key, value]) => entry.metadata[key] !== value,
+ )) return false;
+ return true;
}
/**
- * Cosine similarity (returns value in range [0, 1] where 1 = identical)
+ * Cosine similarity (returns value in range [-1, 1] where 1 = identical)
*/
private cosineSimilarity(a: Float32Array, b: Float32Array): number {
let dotProduct = 0;
diff --git a/v3/@claude-flow/memory/src/agentdb-hnsw-adapter.ts b/v3/@claude-flow/memory/src/agentdb-hnsw-adapter.ts
new file mode 100644
index 0000000000..3f96e6be97
--- /dev/null
+++ b/v3/@claude-flow/memory/src/agentdb-hnsw-adapter.ts
@@ -0,0 +1,173 @@
+/**
+ * Adapter for agentdb's separately exported HNSWIndex, not a controller.
+ *
+ * alpha.20 requires buildIndex() before search/add, does not upsert duplicate
+ * IDs, and does not remove native points on delete. Rebuild a disposable index
+ * lazily from the authoritative in-memory entries after mutations. This trades
+ * write-heavy performance for correctness; it is not a persistence layer.
+ */
+import type { HNSWIndex as NativeIndex } from 'agentdb';
+import type { HNSWStats, MemoryEntry } from './types.js';
+
+type IndexConstructor = new (
+ db: unknown,
+ config: ConstructorParameters<typeof NativeIndex>[1],
+) => NativeIndex;
+
+interface AdapterConfig {
+ dimension: number;
+ M: number;
+ efConstruction: number;
+ efSearch: number;
+}
+
+export class AgentDBHnswAdapter {
+ private index?: NativeIndex;
+ private ids = new Map<number, string>();
+ private revision = 0;
+ private building?: Promise<boolean>;
+ private searching: Promise<unknown> = Promise.resolve();
+ private failed = false;
+ private buildTime = 0;
+
+ constructor(
+ private readonly Index: IndexConstructor,
+ private readonly config: AdapterConfig,
+ ) {
+ // hnswlib-node coerces these values to uint32, rather than validating them.
+ const uint = (value: number) => Number.isInteger(value) && value > 0 && value <= 0xffffffff;
+ this.failed = !uint(config.dimension) || !uint(config.efConstruction) ||
+ !uint(config.efSearch) || !uint(config.M) || config.M < 2 || config.M > 10000;
+ }
+
+ private normalize(vector: Float32Array): Float32Array {
+ // Accumulate in JS double precision: the native cosine implementation can
+ // overflow/underflow its float32 norm even when all coordinates are finite.
+ let squaredNorm = 0;
+ for (const value of vector) squaredNorm += value * value;
+ const norm = Math.sqrt(squaredNorm);
+ return Float32Array.from(vector, (value) => norm === 0 ? 0 : value / norm);
+ }
+
+ invalidate(): void {
+ this.revision++;
+ // An in-flight search owns its old instance until it completes. Do not
+ // clear that instance beneath it; the revision check rejects stale results.
+ this.index = undefined;
+ this.ids = new Map();
+ }
+
+ private async build(getEntries: () => readonly MemoryEntry[]): Promise<boolean> {
+ if (this.failed) return false;
+ if (this.index) return true;
+ if (this.building) return this.building;
+
+ const revision = this.revision;
+ const vectors = getEntries().filter((entry) => entry.embedding);
+ if (vectors.length === 0) return false;
+ // Never feed malformed data to a native addon. Preserve fallback behavior
+ // for callers whose embedding dimensions do not match the configured index.
+ if (vectors.some(({ embedding }) =>
+ embedding!.length !== this.config.dimension ||
+ !embedding!.every(Number.isFinite),
+ )) return false;
+
+ const ids = new Map(vectors.map((entry, id) => [id, entry.id]));
+ const rows = vectors.map((entry, id) => ({
+ id,
+ embedding: this.normalize(entry.embedding!),
+ }));
+ this.building = (async () => {
+ try {
+ // HNSWIndex reads this one fixed query. No user-controlled SQL, tables,
+ // filters, native persistence sidecars, or shared AgentDB controller.
+ const db = {
+ prepare(sql: string) {
+ if (sql.replace(/\s+/g, ' ').trim() !==
+ 'SELECT pattern_id as id, embedding FROM pattern_embeddings') {
+ throw new Error('Unsupported HNSW snapshot query');
+ }
+ return { all: () => rows };
+ },
+ };
+ const candidate = new this.Index(db, {
+ ...this.config,
+ metric: 'cosine',
+ persistIndex: false,
+ // Do not preallocate the backend's million-entry default capacity.
+ maxElements: rows.length,
+ });
+ const start = performance.now();
+ await candidate.buildIndex();
+ if (revision !== this.revision || !candidate.isReady()) return false;
+ this.buildTime = performance.now() - start;
+ this.index = candidate;
+ this.ids = ids;
+ return true;
+ } catch {
+ // A missing/incompatible native addon must not break semantic search or
+ // throw/log the same exception on every request. Retry on reinitialize.
+ this.failed = true;
+ return false;
+ }
+ })();
+ try {
+ return await this.building;
+ } finally {
+ this.building = undefined;
+ }
+ }
+
+ async search(
+ getEntries: () => readonly MemoryEntry[],
+ embedding: Float32Array,
+ k: number,
+ ef?: number,
+ ): Promise<string[] | undefined> {
+ embedding = new Float32Array(embedding);
+ if (!Number.isInteger(k) || k <= 0 || k > 0xffffffff ||
+ embedding.length !== this.config.dimension ||
+ !embedding.every(Number.isFinite)) return undefined;
+ const query = this.normalize(embedding);
+ if (!await this.build(getEntries)) return undefined;
+ const index = this.index;
+ const ids = this.ids;
+ const revision = this.revision;
+ if (!index) return undefined;
+ // Upstream search awaits initialize() before touching the native index.
+ // Serialize calls so per-query ef cannot be changed by another request.
+ const result = this.searching.then(async () => {
+ if (revision !== this.revision || this.failed) return undefined;
+ try {
+ const expansion = ef ?? this.config.efSearch;
+ if (Number.isInteger(expansion) && expansion > 0 && expansion <= 0xffffffff &&
+ expansion !== index.getStats().efSearch) {
+ index.setEfSearch(expansion);
+ }
+ const results = await index.search(query, Math.min(k, ids.size));
+ if (revision !== this.revision) return undefined;
+ return results.map((result) => ids.get(result.id)).filter(
+ (id): id is string => id !== undefined,
+ );
+ } catch {
+ this.failed = true;
+ this.invalidate();
+ return undefined;
+ }
+ });
+ this.searching = result;
+ return result;
+ }
+
+ getStats(): HNSWStats | undefined {
+ if (!this.index?.isReady() || this.failed) return undefined;
+ const stats = this.index.getStats();
+ return {
+ vectorCount: stats.numElements,
+ memoryUsage: 0, // Native allocation size is not exposed by agentdb.
+ avgSearchTime: stats.avgSearchTimeMs,
+ buildTime: this.buildTime,
+ compressionRatio: 1,
+ };
+ }
+}
diff --git a/v3/pnpm-lock.yaml b/v3/pnpm-lock.yaml
index e8be0e2132..5743a3da94 100644
--- a/v3/pnpm-lock.yaml
+++ b/v3/pnpm-lock.yaml
@@ -212,7 +212,7 @@ importers:
specifier: 2.0.0-alpha.5
version: 2.0.0-alpha.5
'@claude-flow/memory':
- specifier: 3.0.7
+ specifier: 3.0.8
version: link:../memory
'@metaharness/darwin':
specifier: ~0.12.0