Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

feat(memory): #2061 ADR-125 Phase 3 β€” persistent HNSW snapshot + restore

ruv81a2b23eb980
diff --git a/v3/@claude-flow/memory/src/agentdb-adapter.ts b/v3/@claude-flow/memory/src/agentdb-adapter.ts
index 3af2ee16f1..7d0e94870b 100644
--- a/v3/@claude-flow/memory/src/agentdb-adapter.ts
+++ b/v3/@claude-flow/memory/src/agentdb-adapter.ts
@@ -8,6 +8,8 @@
*/
import { EventEmitter } from 'node:events';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
import {
IMemoryBackend,
MemoryEntry,
@@ -1022,16 +1024,208 @@ export class AgentDBAdapter extends EventEmitter implements IMemoryBackend {
return { status: 'healthy', latency: 0 };
}
+ /**
+ * Path to the HNSW snapshot sidecar.
+ * Convention: `<persistencePath>.hnsw`
+ */
+ private getHnswSidecarPath(): string | null {
+ if (!this.config.persistencePath) return null;
+ return `${this.config.persistencePath}.hnsw`;
+ }
+
+ /**
+ * Path to the in-memory Maps (entries/namespaceIndex/keyIndex/tagIndex) sidecar.
+ * Convention: `<persistencePath>.meta.json`
+ */
+ private getMetaSidecarPath(): string | null {
+ if (!this.config.persistencePath) return null;
+ return `${this.config.persistencePath}.meta.json`;
+ }
+
+ /**
+ * Persist a snapshot of the in-memory state to disk.
+ *
+ * Writes two sidecar files alongside `persistencePath`:
+ * - `<persistencePath>.hnsw` β€” binary HNSW snapshot via {@link HNSWIndex.serialize}
+ * - `<persistencePath>.meta.json` β€” entries + indices in stable JSON
+ *
+ * Public so {@link MemoryService} can trigger periodic snapshots (ADR-125 Phase 3).
+ */
+ async saveSnapshot(): Promise<void> {
+ await this.saveToDisk();
+ }
+
+ /**
+ * ADR-125 Phase 3 β€” real persistence implementation.
+ *
+ * Loads two sidecar files alongside `persistencePath` (when both exist):
+ * - `<persistencePath>.hnsw` β€” binary HNSW snapshot
+ * - `<persistencePath>.meta.json` β€” entries + namespaceIndex + keyIndex + tagIndex
+ *
+ * Emits `persistence:loaded` with `{ status: 'restored' | 'fresh' | 'corrupt' }`.
+ * Falls back to a fresh state on any deserialize / IO error so callers don't throw.
+ */
private async loadFromDisk(): Promise<void> {
- // Placeholder for persistence implementation
- // Would use SQLite or file-based storage
- this.emit('persistence:loaded');
+ const hnswPath = this.getHnswSidecarPath();
+ const metaPath = this.getMetaSidecarPath();
+
+ if (!hnswPath || !metaPath) {
+ this.emit('persistence:loaded', { status: 'fresh', reason: 'no-path' });
+ return;
+ }
+
+ let hnswExists = false;
+ let metaExists = false;
+ try { hnswExists = fs.existsSync(hnswPath); } catch { /* ignore */ }
+ try { metaExists = fs.existsSync(metaPath); } catch { /* ignore */ }
+
+ if (!hnswExists || !metaExists) {
+ this.emit('persistence:loaded', { status: 'fresh', reason: 'no-sidecar' });
+ return;
+ }
+
+ try {
+ const metaRaw = fs.readFileSync(metaPath, 'utf-8');
+ const meta = JSON.parse(metaRaw) as PersistedMeta;
+
+ // Restore entries (rehydrate Float32Array embeddings if present)
+ this.entries.clear();
+ this.namespaceIndex.clear();
+ this.keyIndex.clear();
+ this.tagIndex.clear();
+
+ for (const persisted of meta.entries) {
+ const entry: MemoryEntry = {
+ ...persisted,
+ embedding: persisted.embedding
+ ? Float32Array.from(persisted.embedding)
+ : undefined,
+ };
+ this.entries.set(entry.id, entry);
+ }
+ for (const [ns, ids] of Object.entries(meta.namespaceIndex)) {
+ this.namespaceIndex.set(ns, new Set(ids));
+ }
+ for (const [key, id] of Object.entries(meta.keyIndex)) {
+ this.keyIndex.set(key, id);
+ }
+ for (const [tag, ids] of Object.entries(meta.tagIndex)) {
+ this.tagIndex.set(tag, new Set(ids));
+ }
+
+ // Restore HNSW
+ const hnswBuf = fs.readFileSync(hnswPath);
+ const restored = HNSWIndex.deserialize(hnswBuf);
+ // Swap pointers β€” preserves forwarded events because we only re-listen
+ // when the adapter is reconstructed (which happens on a fresh instance).
+ this.index = restored;
+ // Re-forward HNSW events
+ this.index.on('point:added', (data) => this.emit('index:added', data));
+
+ this.emit('persistence:loaded', { status: 'restored', count: this.entries.size });
+ } catch (err) {
+ // Corrupt sidecar β€” start fresh, leave existing files in place for
+ // operator inspection.
+ this.entries.clear();
+ this.namespaceIndex.clear();
+ this.keyIndex.clear();
+ this.tagIndex.clear();
+ this.emit('persistence:loaded', {
+ status: 'corrupt',
+ reason: err instanceof Error ? err.message : String(err),
+ });
+ }
}
+ /**
+ * ADR-125 Phase 3 β€” real snapshot implementation.
+ *
+ * Writes both sidecars atomically via a temp-file-and-rename dance so a
+ * crash mid-write doesn't leave half-baked state on disk.
+ */
private async saveToDisk(): Promise<void> {
- // Placeholder for persistence implementation
- this.emit('persistence:saved');
+ const hnswPath = this.getHnswSidecarPath();
+ const metaPath = this.getMetaSidecarPath();
+
+ if (!hnswPath || !metaPath) {
+ this.emit('persistence:saved', { status: 'skipped', reason: 'no-path' });
+ return;
+ }
+
+ try {
+ // Ensure parent dir exists
+ const dir = path.dirname(hnswPath);
+ if (!fs.existsSync(dir)) {
+ fs.mkdirSync(dir, { recursive: true });
+ }
+
+ // Build stable JSON representation
+ const meta = this.buildPersistedMeta();
+ const metaText = JSON.stringify(meta);
+
+ // Build HNSW snapshot
+ const hnswBuf = this.index.serialize();
+
+ // Atomic write via temp + rename
+ const hnswTmp = `${hnswPath}.tmp`;
+ const metaTmp = `${metaPath}.tmp`;
+ fs.writeFileSync(hnswTmp, hnswBuf);
+ fs.writeFileSync(metaTmp, metaText);
+ fs.renameSync(hnswTmp, hnswPath);
+ fs.renameSync(metaTmp, metaPath);
+
+ this.emit('persistence:saved', {
+ status: 'ok',
+ bytes: hnswBuf.length + Buffer.byteLength(metaText, 'utf-8'),
+ });
+ } catch (err) {
+ this.emit('persistence:saved', {
+ status: 'failed',
+ reason: err instanceof Error ? err.message : String(err),
+ });
+ }
+ }
+
+ /**
+ * Build a stable, diff-friendly JSON representation of the in-memory Maps.
+ * Keys are sorted; embeddings are serialized as plain number arrays.
+ */
+ private buildPersistedMeta(): PersistedMeta {
+ const entriesArr = [...this.entries.values()].sort((a, b) =>
+ a.id.localeCompare(b.id)
+ );
+ const persistedEntries = entriesArr.map((e) => ({
+ ...e,
+ embedding: e.embedding ? Array.from(e.embedding) : undefined,
+ }));
+
+ const namespaceIndex: Record<string, string[]> = {};
+ for (const ns of [...this.namespaceIndex.keys()].sort()) {
+ namespaceIndex[ns] = [...this.namespaceIndex.get(ns)!].sort();
+ }
+ const keyIndex: Record<string, string> = {};
+ for (const k of [...this.keyIndex.keys()].sort()) {
+ keyIndex[k] = this.keyIndex.get(k)!;
+ }
+ const tagIndex: Record<string, string[]> = {};
+ for (const t of [...this.tagIndex.keys()].sort()) {
+ tagIndex[t] = [...this.tagIndex.get(t)!].sort();
+ }
+
+ return { version: 1, entries: persistedEntries, namespaceIndex, keyIndex, tagIndex };
}
}
+/**
+ * Wire-format for the meta sidecar (`<persistencePath>.meta.json`).
+ * `embedding` is stored as a plain number[] to keep the JSON canonical.
+ */
+interface PersistedMeta {
+ version: 1;
+ entries: Array<Omit<MemoryEntry, 'embedding'> & { embedding?: number[] }>;
+ namespaceIndex: Record<string, string[]>;
+ keyIndex: Record<string, string>;
+ tagIndex: Record<string, string[]>;
+}
+
export default AgentDBAdapter;
diff --git a/v3/@claude-flow/memory/src/consolidator.ts b/v3/@claude-flow/memory/src/consolidator.ts
new file mode 100644
index 0000000000..be855b611e
--- /dev/null
+++ b/v3/@claude-flow/memory/src/consolidator.ts
@@ -0,0 +1,272 @@
+/**
+ * MemoryConsolidator β€” ADR-125 Phase 4
+ *
+ * Periodic maintenance for the in-memory state owned by {@link UnifiedMemoryService}.
+ *
+ * Operations:
+ * - `sweepExpired()` β€” drop entries past `expiresAt` from all indexes (including HNSW).
+ * - `dedup(strategy)` β€” collapse content-hash duplicates per strategy.
+ * - `compactHnsw()` β€” rebuild the HNSW index from current `entries`.
+ * - `runAll()` β€” sweep β†’ dedup β†’ compact in order.
+ *
+ * Invoked from two paths:
+ * 1. {@link UnifiedMemoryService}'s lifecycle (background timer when
+ * `consolidator.autoRun === true`, plus `close()`).
+ * 2. The AgentDB `nightlyLearner` controller (`src/controller-registry.ts`)
+ * delegates to `runAll()` instead of hitting AgentDB directly.
+ *
+ * Phase 3 placeholder lives in this file too β€” it provides the typed
+ * surface that {@link UnifiedMemoryService.getConsolidator} resolves to.
+ * The real implementation lands with Phase 4.
+ *
+ * @module v3/memory/consolidator
+ */
+
+import { createHash } from 'node:crypto';
+import { HNSWIndex } from './hnsw-index.js';
+import type { MemoryEntry } from './types.js';
+
+/**
+ * Strategy for resolving content-hash duplicates inside {@link MemoryConsolidator.dedup}.
+ */
+export type DedupStrategy = 'keep-newest' | 'keep-oldest' | 'merge-tags';
+
+export interface ConsolidatorOptions {
+ /** Default strategy when `dedup()` is invoked with no argument. */
+ dedupStrategy?: DedupStrategy;
+ /** Used by `MemoryService` when scheduling automatic runs (ms). */
+ intervalMs?: number;
+}
+
+export interface SweepResult {
+ removed: number;
+ remaining: number;
+ hnswRemoved: number;
+}
+
+export interface DedupResult {
+ merged: number;
+ groups: number;
+}
+
+export interface CompactResult {
+ before: number;
+ after: number;
+ durationMs: number;
+}
+
+export interface ConsolidationResult {
+ sweep: SweepResult;
+ dedup: DedupResult;
+ compact: CompactResult;
+ totalDurationMs: number;
+}
+
+/**
+ * Minimal surface the consolidator needs from `UnifiedMemoryService`. Avoids
+ * a circular type import.
+ */
+interface ServiceLike {
+ getAdapter(): {
+ // Internals we mutate. Cast-safe at runtime.
+ entries: Map<string, MemoryEntry>;
+ namespaceIndex: Map<string, Set<string>>;
+ keyIndex: Map<string, string>;
+ tagIndex: Map<string, Set<string>>;
+ // Public HNSW accessor exists on AgentDBAdapter; cast through.
+ [key: string]: any;
+ };
+}
+
+export class MemoryConsolidator {
+ constructor(
+ private readonly service: ServiceLike,
+ private readonly opts: ConsolidatorOptions = {}
+ ) {}
+
+ /**
+ * Drop all entries whose `expiresAt` is in the past.
+ */
+ async sweepExpired(): Promise<SweepResult> {
+ const adapter = this.service.getAdapter() as any;
+ const entries: Map<string, MemoryEntry> = adapter.entries;
+ const namespaceIndex: Map<string, Set<string>> = adapter.namespaceIndex;
+ const keyIndex: Map<string, string> = adapter.keyIndex;
+ const tagIndex: Map<string, Set<string>> = adapter.tagIndex;
+ const index: HNSWIndex = adapter.index;
+
+ const now = Date.now();
+ const toRemove: MemoryEntry[] = [];
+ for (const entry of entries.values()) {
+ if (entry.expiresAt != null && entry.expiresAt < now) {
+ toRemove.push(entry);
+ }
+ }
+
+ let hnswRemoved = 0;
+ for (const entry of toRemove) {
+ entries.delete(entry.id);
+ namespaceIndex.get(entry.namespace)?.delete(entry.id);
+ keyIndex.delete(`${entry.namespace}:${entry.key}`);
+ for (const tag of entry.tags) tagIndex.get(tag)?.delete(entry.id);
+ if (entry.embedding) {
+ const removed = await index.removePoint(entry.id);
+ if (removed) hnswRemoved += 1;
+ }
+ }
+
+ // Clean up empty namespace/tag sets to keep memory bounded
+ for (const [ns, ids] of namespaceIndex) {
+ if (ids.size === 0) namespaceIndex.delete(ns);
+ }
+ for (const [tag, ids] of tagIndex) {
+ if (ids.size === 0) tagIndex.delete(tag);
+ }
+
+ return {
+ removed: toRemove.length,
+ remaining: entries.size,
+ hnswRemoved,
+ };
+ }
+
+ /**
+ * Collapse content-hash duplicates across all namespaces per `strategy`.
+ *
+ * - `keep-newest`: keep the entry with the highest `updatedAt`, drop the rest.
+ * - `keep-oldest`: keep the entry with the lowest `createdAt`, drop the rest.
+ * - `merge-tags`: keep newest, but union the tag sets of the duplicates first.
+ */
+ async dedup(strategy?: DedupStrategy): Promise<DedupResult> {
+ const adapter = this.service.getAdapter() as any;
+ const entries: Map<string, MemoryEntry> = adapter.entries;
+ const namespaceIndex: Map<string, Set<string>> = adapter.namespaceIndex;
+ const keyIndex: Map<string, string> = adapter.keyIndex;
+ const tagIndex: Map<string, Set<string>> = adapter.tagIndex;
+ const index: HNSWIndex = adapter.index;
+
+ const effective = strategy ?? this.opts.dedupStrategy ?? 'keep-newest';
+
+ // Bucket by content hash
+ const buckets = new Map<string, MemoryEntry[]>();
+ for (const entry of entries.values()) {
+ const hash = createHash('sha256').update(entry.content).digest('hex');
+ const bucket = buckets.get(hash);
+ if (bucket) bucket.push(entry);
+ else buckets.set(hash, [entry]);
+ }
+
+ let merged = 0;
+ let dupGroups = 0;
+ for (const bucket of buckets.values()) {
+ if (bucket.length <= 1) continue;
+ dupGroups += 1;
+
+ // Choose keeper
+ let keeper: MemoryEntry;
+ if (effective === 'keep-oldest') {
+ keeper = bucket.reduce((acc, e) =>
+ e.createdAt < acc.createdAt ? e : acc
+ );
+ } else {
+ // keep-newest + merge-tags share this branch
+ keeper = bucket.reduce((acc, e) =>
+ e.updatedAt > acc.updatedAt ? e : acc
+ );
+ }
+
+ if (effective === 'merge-tags') {
+ const tagSet = new Set<string>(keeper.tags);
+ for (const e of bucket) for (const t of e.tags) tagSet.add(t);
+ const oldTags = keeper.tags;
+ keeper.tags = [...tagSet];
+ // Update tagIndex membership for newly-added tags
+ for (const t of keeper.tags) {
+ if (!oldTags.includes(t)) {
+ if (!tagIndex.has(t)) tagIndex.set(t, new Set());
+ tagIndex.get(t)!.add(keeper.id);
+ }
+ }
+ }
+
+ // Drop everyone except the keeper
+ for (const e of bucket) {
+ if (e.id === keeper.id) continue;
+ entries.delete(e.id);
+ namespaceIndex.get(e.namespace)?.delete(e.id);
+ const compositeKey = `${e.namespace}:${e.key}`;
+ if (keyIndex.get(compositeKey) === e.id) {
+ keyIndex.delete(compositeKey);
+ }
+ for (const tag of e.tags) tagIndex.get(tag)?.delete(e.id);
+ if (e.embedding) {
+ await index.removePoint(e.id);
+ }
+ merged += 1;
+ }
+ }
+
+ return { merged, groups: dupGroups };
+ }
+
+ /**
+ * Rebuild the HNSW index from the current set of entries with embeddings.
+ * Returns a count snapshot + duration.
+ */
+ async compactHnsw(): Promise<CompactResult> {
+ const adapter = this.service.getAdapter() as any;
+ const entries: Map<string, MemoryEntry> = adapter.entries;
+ const index: HNSWIndex = adapter.index;
+
+ const before = index.size;
+ const t0 = Date.now();
+
+ // Build a fresh HNSW from current entries that have embeddings.
+ const dims = (index as any).config?.dimensions ?? 0;
+ const fresh = new HNSWIndex({
+ dimensions: dims,
+ M: (index as any).config?.M,
+ efConstruction: (index as any).config?.efConstruction,
+ maxElements: (index as any).config?.maxElements,
+ metric: (index as any).config?.metric,
+ });
+
+ for (const entry of entries.values()) {
+ if (entry.embedding) {
+ try {
+ await fresh.addPoint(entry.id, entry.embedding);
+ } catch {
+ // Tolerate dimension mismatches in malformed datasets.
+ }
+ }
+ }
+
+ // Swap pointers atomically and re-forward events
+ adapter.index = fresh;
+ if (typeof adapter.emit === 'function') {
+ fresh.on('point:added', (data: any) => adapter.emit('index:added', data));
+ }
+
+ const durationMs = Date.now() - t0;
+ return { before, after: fresh.size, durationMs };
+ }
+
+ /**
+ * Sweep β†’ dedup β†’ compact. Used by the background timer and by the
+ * `nightlyLearner` AgentDB controller.
+ */
+ async runAll(): Promise<ConsolidationResult> {
+ const start = Date.now();
+ const sweep = await this.sweepExpired();
+ const dedup = await this.dedup();
+ const compact = await this.compactHnsw();
+ return {
+ sweep,
+ dedup,
+ compact,
+ totalDurationMs: Date.now() - start,
+ };
+ }
+}
+
+export default MemoryConsolidator;
diff --git a/v3/@claude-flow/memory/src/hnsw-index.ts b/v3/@claude-flow/memory/src/hnsw-index.ts
index 35accf0e21..124850bdc5 100644
--- a/v3/@claude-flow/memory/src/hnsw-index.ts
+++ b/v3/@claude-flow/memory/src/hnsw-index.ts
@@ -528,6 +528,205 @@ export class HNSWIndex extends EventEmitter {
return this.nodes.size;
}
+ // ===== Persistence (ADR-125 Phase 3) =====
+
+ /**
+ * Magic header for serialized HNSW snapshots.
+ * Format: "HNSW" + version byte (0x01).
+ */
+ static readonly SERIALIZATION_MAGIC = Buffer.from([0x48, 0x4e, 0x53, 0x57, 0x01]);
+
+ /**
+ * Serialize the index to a binary buffer.
+ *
+ * Layout (all integers big-endian):
+ * - 5 bytes: magic header "HNSW" + version (0x01)
+ * - 4 bytes: dimensions (uint32)
+ * - 4 bytes: M (uint32)
+ * - 4 bytes: efConstruction (uint32)
+ * - 4 bytes: metric length (uint32) + metric utf-8 bytes
+ * - 4 bytes: maxLevel (uint32)
+ * - 4 bytes: entryPoint length (uint32) + entryPoint utf-8 bytes (0 = null)
+ * - 4 bytes: node count (uint32)
+ * - per node:
+ * - 4 bytes id length + id utf-8 bytes
+ * - 4 bytes level
+ * - 4 bytes vector length (in floats) + vector bytes (Float32, little-endian native)
+ * - 1 byte hasNormalized (0/1)
+ * - if hasNormalized: 4 bytes normalized length + normalized bytes
+ * - 4 bytes connection-level count
+ * - per level:
+ * - 4 bytes layer index
+ * - 4 bytes neighbor count
+ * - per neighbor: 4 bytes id length + id utf-8 bytes
+ */
+ serialize(): Buffer {
+ const chunks: Buffer[] = [];
+ chunks.push(HNSWIndex.SERIALIZATION_MAGIC);
+
+ // Config header
+ const header = Buffer.alloc(16);
+ header.writeUInt32BE(this.config.dimensions, 0);
+ header.writeUInt32BE(this.config.M, 4);
+ header.writeUInt32BE(this.config.efConstruction, 8);
+ header.writeUInt32BE(this.maxLevel, 12);
+ chunks.push(header);
+
+ chunks.push(this.encodeLengthPrefixedString(this.config.metric));
+ chunks.push(this.encodeLengthPrefixedString(this.entryPoint ?? ''));
+
+ const nodeCountBuf = Buffer.alloc(4);
+ nodeCountBuf.writeUInt32BE(this.nodes.size, 0);
+ chunks.push(nodeCountBuf);
+
+ for (const [id, node] of this.nodes) {
+ chunks.push(this.encodeLengthPrefixedString(id));
+
+ const meta = Buffer.alloc(4);
+ meta.writeUInt32BE(node.level, 0);
+ chunks.push(meta);
+
+ chunks.push(this.encodeFloat32Array(node.vector));
+
+ if (node.normalizedVector) {
+ chunks.push(Buffer.from([1]));
+ chunks.push(this.encodeFloat32Array(node.normalizedVector));
+ } else {
+ chunks.push(Buffer.from([0]));
+ }
+
+ const levels = [...node.connections.entries()];
+ const lvlCountBuf = Buffer.alloc(4);
+ lvlCountBuf.writeUInt32BE(levels.length, 0);
+ chunks.push(lvlCountBuf);
+
+ for (const [layer, neighbors] of levels) {
+ const layerBuf = Buffer.alloc(8);
+ layerBuf.writeUInt32BE(layer, 0);
+ layerBuf.writeUInt32BE(neighbors.size, 4);
+ chunks.push(layerBuf);
+ for (const neighborId of neighbors) {
+ chunks.push(this.encodeLengthPrefixedString(neighborId));
+ }
+ }
+ }
+
+ return Buffer.concat(chunks);
+ }
+
+ /**
+ * Deserialize an HNSW index from a buffer produced by {@link serialize}.
+ *
+ * Throws on magic-header mismatch, version mismatch, or truncated input.
+ */
+ static deserialize(buf: Buffer): HNSWIndex {
+ if (buf.length < HNSWIndex.SERIALIZATION_MAGIC.length) {
+ throw new Error('HNSWIndex.deserialize: buffer too short for magic header');
+ }
+ for (let i = 0; i < HNSWIndex.SERIALIZATION_MAGIC.length; i++) {
+ if (buf[i] !== HNSWIndex.SERIALIZATION_MAGIC[i]) {
+ throw new Error(
+ `HNSWIndex.deserialize: magic header mismatch at byte ${i} (got 0x${buf[i].toString(16)}, expected 0x${HNSWIndex.SERIALIZATION_MAGIC[i].toString(16)})`
+ );
+ }
+ }
+
+ let offset = HNSWIndex.SERIALIZATION_MAGIC.length;
+
+ const dimensions = buf.readUInt32BE(offset); offset += 4;
+ const M = buf.readUInt32BE(offset); offset += 4;
+ const efConstruction = buf.readUInt32BE(offset); offset += 4;
+ const maxLevel = buf.readUInt32BE(offset); offset += 4;
+
+ const metricRead = readLengthPrefixedString(buf, offset);
+ offset = metricRead.offset;
+ const metric = metricRead.value as DistanceMetric;
+
+ const entryRead = readLengthPrefixedString(buf, offset);
+ offset = entryRead.offset;
+ const entryPoint = entryRead.value === '' ? null : entryRead.value;
+
+ const nodeCount = buf.readUInt32BE(offset); offset += 4;
+
+ const index = new HNSWIndex({
+ dimensions,
+ M,
+ efConstruction,
+ metric,
+ });
+ index.maxLevel = maxLevel;
+ index.entryPoint = entryPoint;
+
+ for (let n = 0; n < nodeCount; n++) {
+ const idRead = readLengthPrefixedString(buf, offset);
+ offset = idRead.offset;
+ const id = idRead.value;
+
+ const level = buf.readUInt32BE(offset); offset += 4;
+
+ const vecRead = readFloat32Array(buf, offset);
+ offset = vecRead.offset;
+ const vector = vecRead.value;
+
+ const hasNormalized = buf[offset]; offset += 1;
+ let normalizedVector: Float32Array | null = null;
+ if (hasNormalized) {
+ const normRead = readFloat32Array(buf, offset);
+ offset = normRead.offset;
+ normalizedVector = normRead.value;
+ }
+
+ const lvlCount = buf.readUInt32BE(offset); offset += 4;
+ const connections = new Map<number, Set<string>>();
+ for (let l = 0; l < lvlCount; l++) {
+ const layer = buf.readUInt32BE(offset); offset += 4;
+ const neighborCount = buf.readUInt32BE(offset); offset += 4;
+ const neighbors = new Set<string>();
+ for (let k = 0; k < neighborCount; k++) {
+ const neighborRead = readLengthPrefixedString(buf, offset);
+ offset = neighborRead.offset;
+ neighbors.add(neighborRead.value);
+ }
+ connections.set(layer, neighbors);
+ }
+
+ index.nodes.set(id, {
+ id,
+ vector,
+ normalizedVector,
+ connections,
+ level,
+ });
+ }
+
+ return index;
+ }
+
+ private encodeLengthPrefixedString(s: string): Buffer {
+ const strBuf = Buffer.from(s, 'utf-8');
+ const out = Buffer.alloc(4 + strBuf.length);
+ out.writeUInt32BE(strBuf.length, 0);
+ strBuf.copy(out, 4);
+ return out;
+ }
+
+ private encodeFloat32Array(arr: Float32Array): Buffer {
+ const lenBuf = Buffer.alloc(4);
+ lenBuf.writeUInt32BE(arr.length, 0);
+ const dataBuf = Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength);
+ return Buffer.concat([lenBuf, dataBuf]);
+ }
+
+ /**
+ * Remove a point from the index β€” alias for {@link removePoint}.
+ *
+ * Added by ADR-125 Phase 4 so {@link MemoryConsolidator} has a stable
+ * synchronous-shaped API to call. Internally delegates to {@link removePoint}.
+ */
+ async remove(id: string): Promise<boolean> {
+ return this.removePoint(id);
+ }
+
// ===== Private Methods =====
private mergeConfig(config: Partial<HNSWConfig>): HNSWConfig {
@@ -1206,4 +1405,48 @@ class Quantizer {
}
}
+// ===== Persistence Helpers (ADR-125 Phase 3) =====
+
+function readLengthPrefixedString(
+ buf: Buffer,
+ offset: number
+): { value: string; offset: number } {
+ if (offset + 4 > buf.length) {
+ throw new Error(`HNSWIndex.deserialize: truncated string length at offset ${offset}`);
+ }
+ const len = buf.readUInt32BE(offset);
+ const start = offset + 4;
+ const end = start + len;
+ if (end > buf.length) {
+ throw new Error(
+ `HNSWIndex.deserialize: truncated string payload at offset ${offset} (needed ${len} bytes, have ${buf.length - start})`
+ );
+ }
+ const value = buf.toString('utf-8', start, end);
+ return { value, offset: end };
+}
+
+function readFloat32Array(
+ buf: Buffer,
+ offset: number
+): { value: Float32Array; offset: number } {
+ if (offset + 4 > buf.length) {
+ throw new Error(`HNSWIndex.deserialize: truncated array length at offset ${offset}`);
+ }
+ const floatCount = buf.readUInt32BE(offset);
+ const start = offset + 4;
+ const byteLen = floatCount * 4;
+ const end = start + byteLen;
+ if (end > buf.length) {
+ throw new Error(
+ `HNSWIndex.deserialize: truncated array payload at offset ${offset} (needed ${byteLen} bytes, have ${buf.length - start})`
+ );
+ }
+ // Copy into a fresh ArrayBuffer so Float32Array isn't a view onto the original
+ // (de-aligned) Node Buffer pool.
+ const copy = new ArrayBuffer(byteLen);
+ new Uint8Array(copy).set(new Uint8Array(buf.buffer, buf.byteOffset + start, byteLen));
+ return { value: new Float32Array(copy), offset: end };
+}
+
export default HNSWIndex;
diff --git a/v3/@claude-flow/memory/src/hnsw-lite.ts b/v3/@claude-flow/memory/src/hnsw-lite.ts
deleted file mode 100644
index a51c817e6b..0000000000
--- a/v3/@claude-flow/memory/src/hnsw-lite.ts
+++ /dev/null
@@ -1,190 +0,0 @@
-export interface HnswSearchResult {
- id: string;
- score: number;
-}
-
-export class HnswLite {
- private vectors = new Map<string, Float32Array>();
- private neighbors = new Map<string, Set<string>>();
- private readonly dimensions: number;
- private readonly maxNeighbors: number;
- private readonly efConstruction: number;
- private readonly metric: string;
-
- constructor(dimensions: number, m: number, efConstruction: number, metric: string) {
- this.dimensions = dimensions;
- this.maxNeighbors = m;
- this.efConstruction = efConstruction;
- this.metric = metric;
- }
-
- get size(): number {
- return this.vectors.size;
- }
-
- add(id: string, vector: Float32Array): void {
- this.vectors.set(id, vector);
-
- if (this.vectors.size === 1) {
- this.neighbors.set(id, new Set());
- return;
- }
-
- const nearest = this.findNearest(vector, this.maxNeighbors);
- const neighborSet = new Set<string>();
-
- for (const n of nearest) {
- neighborSet.add(n.id);
- const nNeighbors = this.neighbors.get(n.id);
- if (nNeighbors) {
- nNeighbors.add(id);
- if (nNeighbors.size > this.maxNeighbors * 2) {
- this.pruneNeighbors(n.id);
- }
- }
- }
-
- this.neighbors.set(id, neighborSet);
- }
-
- remove(id: string): void {
- this.vectors.delete(id);
- const myNeighbors = this.neighbors.get(id);
- if (myNeighbors) {
- for (const nId of myNeighbors) {
- this.neighbors.get(nId)?.delete(id);
- }
- }
- this.neighbors.delete(id);
- }
-
- search(query: Float32Array, k: number, threshold?: number): HnswSearchResult[] {
- if (this.vectors.size === 0) return [];
- if (this.vectors.size <= k * 2) {
- return this.bruteForce(query, k, threshold);
- }
-
- const visited = new Set<string>();
- const candidates: HnswSearchResult[] = [];
-
- let entryId: string | undefined;
- let bestScore = -1;
- for (const [id] of this.vectors) {
- const score = this.similarity(query, this.vectors.get(id)!);
- if (score > bestScore) {
- bestScore = score;
- entryId = id;
- }
- if (visited.size >= Math.min(this.efConstruction, this.vectors.size)) break;
- visited.add(id);
- candidates.push({ id, score });
- }
-
- if (entryId) {
- const queue = [entryId];
- let idx = 0;
-
- while (idx < queue.length && visited.size < this.efConstruction * 2) {
- const currentId = queue[idx++];
- const currentNeighbors = this.neighbors.get(currentId);
- if (!currentNeighbors) continue;
-
- for (const nId of currentNeighbors) {
- if (visited.has(nId)) continue;
- visited.add(nId);
-
- const vec = this.vectors.get(nId);
- if (!vec) continue;
-
- const score = this.similarity(query, vec);
- candidates.push({ id: nId, score });
- queue.push(nId);
- }
- }
- }
-
- candidates.sort((a, b) => b.score - a.score);
-
- let filtered = candidates;
- if (threshold !== undefined) {
- filtered = filtered.filter(c => c.score >= threshold);
- }
-
- return filtered.slice(0, k);
- }
-
- private bruteForce(query: Float32Array, k: number, threshold?: number): HnswSearchResult[] {
- const results: HnswSearchResult[] = [];
- for (const [id, vec] of this.vectors) {
- const score = this.similarity(query, vec);
- if (threshold !== undefined && score < threshold) continue;
- results.push({ id, score });
- }
- results.sort((a, b) => b.score - a.score);
- return results.slice(0, k);
- }
-
- private findNearest(query: Float32Array, k: number): HnswSearchResult[] {
- return this.bruteForce(query, k);
- }
-
- private pruneNeighbors(id: string): void {
- const myNeighbors = this.neighbors.get(id);
- if (!myNeighbors) return;
-
- const vec = this.vectors.get(id);
- if (!vec) return;
-
- const scored: HnswSearchResult[] = [];
- for (const nId of myNeighbors) {
- const nVec = this.vectors.get(nId);
- if (!nVec) continue;
- scored.push({ id: nId, score: this.similarity(vec, nVec) });
- }
-
- scored.sort((a, b) => b.score - a.score);
- const keep = new Set(scored.slice(0, this.maxNeighbors).map(s => s.id));
-
- for (const nId of myNeighbors) {
- if (!keep.has(nId)) {
- myNeighbors.delete(nId);
- }
- }
- }
-
- private similarity(a: Float32Array, b: Float32Array): number {
- if (this.metric === 'dot') return dotProduct(a, b);
- if (this.metric === 'euclidean') return 1 / (1 + euclideanDistance(a, b));
- return cosineSimilarity(a, b);
- }
-}
-
-export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
- let dot = 0;
- let normA = 0;
- let normB = 0;
- for (let i = 0; i < a.length; i++) {
- dot += a[i] * b[i];
- normA += a[i] * a[i];
- normB += b[i] * b[i];
- }
- if (normA === 0 || normB === 0) return 0;
- return dot / (Math.sqrt(normA) * Math.sqrt(normB));
-}
-
-function dotProduct(a: Float32Array, b: Float32Array): number {
- let sum = 0;
- for (let i = 0; i < a.length; i++) {
- sum += a[i] * b[i];
- }
- return sum;
-}
-
-function euclideanDistance(a: Float32Array, b: Float32Array): number {
- let sum = 0;
- for (let i = 0; i < a.length; i++) {
- const d = a[i] - b[i];
- sum += d * d;
- }
- return Math.sqrt(sum);
-}
diff --git a/v3/@claude-flow/memory/src/hnsw-persistence.test.ts b/v3/@claude-flow/memory/src/hnsw-persistence.test.ts
new file mode 100644
index 0000000000..ebafacaebb
--- /dev/null
+++ b/v3/@claude-flow/memory/src/hnsw-persistence.test.ts
@@ -0,0 +1,236 @@
+/**
+ * Phase 3 β€” Persistent HNSW (ADR-125)
+ *
+ * Verifies that:
+ * - `HNSWIndex.serialize()` / `deserialize()` round-trip correctly.
+ * - `MemoryService` close β†’ reopen on the same dbPath recovers all entries
+ * AND the HNSW index (Acceptance Criterion #3).
+ * - A corrupt sidecar emits `health.persistence = 'corrupt'` and the service
+ * falls back to fresh state without throwing.
+ * - Snapshot-every-N triggers a sidecar mtime update once the threshold is hit.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { HNSWIndex } from './hnsw-index.js';
+import { MemoryService } from './index.js';
+import { createDefaultEntry } from './types.js';
+
+function mkTmpDir(): string {
+ return fs.mkdtempSync(path.join(os.tmpdir(), 'cf-mem-phase3-'));
+}
+
+function cleanupDir(dir: string): void {
+ try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
+}
+
+function randomVec(dim: number, seed: number): Float32Array {
+ const out = new Float32Array(dim);
+ let s = seed;
+ for (let i = 0; i < dim; i++) {
+ // xorshift32 for deterministic values
+ s ^= s << 13;
+ s ^= s >>> 17;
+ s ^= s << 5;
+ out[i] = ((s | 0) / 2 ** 31);
+ }
+ return out;
+}
+
+describe('Phase 3 β€” HNSWIndex.serialize / deserialize', () => {
+ it('round-trips an empty index', () => {
+ const idx = new HNSWIndex({ dimensions: 8, M: 4, efConstruction: 10, metric: 'cosine' });
+ const buf = idx.serialize();
+ const restored = HNSWIndex.deserialize(buf);
+ expect(restored.size).toBe(0);
+ });
+
+ it('round-trips a populated index and preserves neighbor sets', async () => {
+ const idx = new HNSWIndex({ dimensions: 16, M: 4, efConstruction: 20, metric: 'cosine' });
+ for (let i = 0; i < 32; i++) {
+ await idx.addPoint(`id-${i}`, randomVec(16, i + 1));
+ }
+ expect(idx.size).toBe(32);
+
+ const buf = idx.serialize();
+ const restored = HNSWIndex.deserialize(buf);
+ expect(restored.size).toBe(32);
+
+ // Same query β†’ same top-k ordering
+ const query = randomVec(16, 999);
+ const beforeResults = await idx.search(query, 5);
+ const afterResults = await restored.search(query, 5);
+ expect(afterResults.map(r => r.id)).toEqual(beforeResults.map(r => r.id));
+ });
+
+ it('rejects buffers without the magic header', () => {
+ const bad = Buffer.from([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]);
+ expect(() => HNSWIndex.deserialize(bad)).toThrow(/magic header/);
+ });
+
+ it('rejects truncated payloads', () => {
+ const idx = new HNSWIndex({ dimensions: 4, M: 2, efConstruction: 4, metric: 'cosine' });
+ const buf = idx.serialize();
+ // Truncate before nodeCount
+ const truncated = buf.slice(0, 6);
+ expect(() => HNSWIndex.deserialize(truncated)).toThrow();
+ });
+});
+
+describe('Phase 3 β€” MemoryService snapshot + restore', () => {
+ let dir: string;
+ let dbPath: string;
+
+ beforeEach(() => {
+ dir = mkTmpDir();
+ dbPath = path.join(dir, 'mem.db');
+ });
+
+ afterEach(() => {
+ cleanupDir(dir);
+ });
+
+ it('Acceptance Criterion #3 β€” restarts recover entries AND HNSW index', async () => {
+ // Phase A: populate
+ const dim = 16;
+ let restoredStatus: any = null;
+ const svc1 = new MemoryService({
+ dimensions: dim,
+ persistenceEnabled: true,
+ persistencePath: dbPath,
+ snapshotInterval: 0, // disable interval-based snapshots β€” only close() flushes
+ embeddingGenerator: async (text: string) => randomVec(dim, hashStr(text)),
+ });
+ await svc1.initialize();
+
+ // Capture neighbors for verification
+ const inserted: Array<{ id: string; embedding: Float32Array }> = [];
+ for (let i = 0; i < 100; i++) {
+ const entry = createDefaultEntry({
+ key: `k-${i}`,
+ content: `entry ${i}`,
+ namespace: 'phase3',
+ });
+ entry.embedding = randomVec(dim, i + 1);
+ await svc1.store(entry);
+ inserted.push({ id: entry.id, embedding: entry.embedding });
+ }
+
+ // Capture HNSW neighbors against a fixed query vector BEFORE close.
+ const queryVec = randomVec(dim, 7777);
+ const beforeNeighbors = await svc1.search(queryVec, { k: 10 });
+
+ await svc1.close();
+
+ // Phase B: reopen β€” should restore both entries and HNSW.
+ const svc2 = new MemoryService({
+ dimensions: dim,
+ persistenceEnabled: true,
+ persistencePath: dbPath,
+ snapshotInterval: 0,
+ embeddingGenerator: async (text: string) => randomVec(dim, hashStr(text)),
+ });
+ svc2.getAdapter().on('persistence:loaded', (ev: any) => { restoredStatus = ev; });
+ await svc2.initialize();
+
+ expect(restoredStatus).not.toBeNull();
+ expect(restoredStatus.status).toBe('restored');
+ expect(restoredStatus.count).toBe(100);
+
+ // Same query β†’ same neighbors after restore (no rebuild log expected).
+ const afterNeighbors = await svc2.search(queryVec, { k: 10 });
+ expect(afterNeighbors.map(r => r.entry.id)).toEqual(beforeNeighbors.map(r => r.entry.id));
+
+ await svc2.close();
+ });
+
+ it('emits health.persistence = "corrupt" on tampered sidecar; falls back to fresh', async () => {
+ // Phase A: populate + close to create sidecars
+ const dim = 8;
+ const svc1 = new MemoryService({
+ dimensions: dim,
+ persistenceEnabled: true,
+ persistencePath: dbPath,
+ snapshotInterval: 0,
+ });
+ await svc1.initialize();
+ for (let i = 0; i < 5; i++) {
+ const entry = createDefaultEntry({ key: `k-${i}`, content: `x-${i}` });
+ entry.embedding = randomVec(dim, i + 1);
+ await svc1.store(entry);
+ }
+ await svc1.close();
+
+ const hnswSidecar = `${dbPath}.hnsw`;
+ expect(fs.existsSync(hnswSidecar)).toBe(true);
+
+ // Mutate the magic header byte
+ const buf = fs.readFileSync(hnswSidecar);
+ buf[0] = 0xFF;
+ fs.writeFileSync(hnswSidecar, buf);
+
+ // Reopen β€” should detect corruption and start fresh, without throwing.
+ let loadedEvent: any = null;
+ const svc2 = new MemoryService({
+ dimensions: dim,
+ persistenceEnabled: true,
+ persistencePath: dbPath,
+ snapshotInterval: 0,
+ });
+ svc2.getAdapter().on('persistence:loaded', (ev: any) => { loadedEvent = ev; });
+ await svc2.initialize();
+
+ expect(loadedEvent).not.toBeNull();
+ expect(loadedEvent.status).toBe('corrupt');
+ // Fresh state: count should be zero
+ expect(await svc2.count()).toBe(0);
+ await svc2.close();
+ });
+
+ it('snapshot-every-N triggers a sidecar write after the threshold is hit', async () => {
+ const dim = 8;
+ const N = 3;
+ const svc = new MemoryService({
+ dimensions: dim,
+ persistenceEnabled: true,
+ persistencePath: dbPath,
+ snapshotInterval: N,
+ });
+ await svc.initialize();
+
+ const hnswSidecar = `${dbPath}.hnsw`;
+ expect(fs.existsSync(hnswSidecar)).toBe(false);
+
+ // First N-1 stores should NOT flush a snapshot.
+ for (let i = 0; i < N - 1; i++) {
+ const entry = createDefaultEntry({ key: `pre-${i}`, content: `c-${i}` });
+ entry.embedding = randomVec(dim, i + 1);
+ await svc.store(entry);
+ }
+ expect(fs.existsSync(hnswSidecar)).toBe(false);
+
+ // The Nth store should fire saveSnapshot β€” await one event-loop turn so
+ // the fire-and-forget save() promise resolves.
+ const saved = new Promise<void>((resolve) => {
+ svc.getAdapter().once('persistence:saved', () => resolve());
+ });
+ const triggerEntry = createDefaultEntry({ key: 'trigger', content: 'trigger' });
+ triggerEntry.embedding = randomVec(dim, 999);
+ await svc.store(triggerEntry);
+ await saved;
+
+ expect(fs.existsSync(hnswSidecar)).toBe(true);
+ await svc.close();
+ });
+});
+
+// Cheap deterministic string hash for the test embedder.
+function hashStr(s: string): number {
+ let h = 0;
+ for (let i = 0; i < s.length; i++) {
+ h = (h * 31 + s.charCodeAt(i)) | 0;
+ }
+ return Math.abs(h) + 1;
+}
diff --git a/v3/@claude-flow/memory/src/index.ts b/v3/@claude-flow/memory/src/index.ts
index 5d9eb045ab..7d85a4ff04 100644
--- a/v3/@claude-flow/memory/src/index.ts
+++ b/v3/@claude-flow/memory/src/index.ts
@@ -194,10 +194,10 @@ export type {
HybridQuery,
} from './hybrid-backend.js';
// `RvfBackend` and `HnswLite` are intentionally NOT re-exported from the top level
-// per ADR-125 Phase 1. They remain internal β€” RvfBackend is reachable via
-// `createDatabase({ provider: 'rvf' })`, and HnswLite is used inside RvfBackend.
-// Importers that need them directly should import the explicit module path.
-export { cosineSimilarity } from './hnsw-lite.js';
+// per ADR-125 Phase 1. `RvfBackend` remains reachable via
+// `createDatabase({ provider: 'rvf' })`. The legacy `hnsw-lite.ts` module was
+// deleted by ADR-125 Phase 3; its brute-force-degrading code is inlined into
+// `rvf-backend.ts` as a private helper.
export { HNSWIndex } from './hnsw-index.js';
export { CacheManager, TieredCacheManager } from './cache-manager.js';
export { QueryBuilder, query, QueryTemplates } from './query-builder.js';
@@ -250,6 +250,28 @@ export interface UnifiedMemoryServiceConfig extends Partial<AgentDBAdapterConfig
/** Embedding generator function */
embeddingGenerator?: EmbeddingGenerator;
+
+ /**
+ * Take an HNSW + metadata snapshot every N successful `store()` calls.
+ * Set to `0` (or `Infinity`) to disable interval-based snapshots β€” `close()`
+ * still flushes. Default: 1000.
+ *
+ * Only takes effect when `persistenceEnabled === true` and `persistencePath`
+ * is set. Added by ADR-125 Phase 3.
+ */
+ snapshotInterval?: number;
+
+ /**
+ * Configuration for the background {@link MemoryConsolidator}. Added by
+ * ADR-125 Phase 4. When `autoRun: true`, the service starts a `setInterval`
+ * timer (default 6h) that runs sweep + dedup + compact and emits the
+ * result via `'consolidation:complete'`.
+ */
+ consolidator?: {
+ autoRun?: boolean;
+ intervalMs?: number;
+ dedupStrategy?: 'keep-newest' | 'keep-oldest' | 'merge-tags';
+ };
}
/**
@@ -275,6 +297,18 @@ export class UnifiedMemoryService extends EventEmitter implements IMemoryBackend
private config: UnifiedMemoryServiceConfig;
private initialized: boolean = false;
+ /** Total successful `store()` calls since last snapshot trigger (ADR-125 Phase 3). */
+ private storeCountSinceSnapshot: number = 0;
+
+ /** Resolved snapshot interval β€” see {@link UnifiedMemoryServiceConfig.snapshotInterval}. */
+ private readonly snapshotInterval: number;
+
+ /** Background consolidator timer (ADR-125 Phase 4). */
+ private consolidatorTimer: ReturnType<typeof setInterval> | null = null;
+
+ /** Lazy-loaded consolidator instance (ADR-125 Phase 4). */
+ private consolidator: any | null = null;
+
/**
* The active memory backend. Defaults to the `AgentDBAdapter` created from
* config, but can be any `IMemoryBackend` implementation (e.g. `HybridBackend`
@@ -294,6 +328,11 @@ export class UnifiedMemoryService extends EventEmitter implements IMemoryBackend
...config,
};
+ // ADR-125 Phase 3 β€” snapshot every Nth store. 0/Infinity = disabled.
+ const raw = this.config.snapshotInterval;
+ this.snapshotInterval =
+ raw === undefined ? 1000 : raw === 0 ? Infinity : raw;
+
this.adapter = new AgentDBAdapter({
dimensions: this.config.dimensions,
cacheEnabled: this.config.cacheEnabled,
@@ -349,20 +388,126 @@ export class UnifiedMemoryService extends EventEmitter implements IMemoryBackend
// HybridBackend wired by createHybridService), the adapter may never be
// used β€” skip its initialize() in that case to avoid double-allocating.
this.initialized = true;
+
+ // ADR-125 Phase 4 β€” start background consolidator if requested.
+ this.startConsolidatorTimer();
+
this.emit('initialized');
}
async shutdown(): Promise<void> {
if (!this.initialized) return;
+
+ // ADR-125 Phase 4 β€” stop consolidator timer first to prevent a sweep
+ // from racing the snapshot below.
+ if (this.consolidatorTimer) {
+ clearInterval(this.consolidatorTimer);
+ this.consolidatorTimer = null;
+ }
+
+ // ADR-125 Phase 3 β€” flush a final HNSW + meta snapshot before tearing the
+ // backend down. Only meaningful when the AgentDBAdapter is the active
+ // backend and persistence is enabled.
+ if (this.backend === this.adapter) {
+ try {
+ await this.adapter.saveSnapshot();
+ } catch {
+ // saveToDisk already emits failure events; do not throw from shutdown.
+ }
+ }
+
await this.backend.shutdown();
this.initialized = false;
this.emit('shutdown');
}
+ /**
+ * Alias for {@link shutdown}. Matches the lifecycle name expected by callers
+ * who treat `MemoryService` like a connection β€” referenced from ADR-125
+ * Phase 3 (snapshot on close()) and Phase 4 (consolidator timer cleanup).
+ */
+ async close(): Promise<void> {
+ return this.shutdown();
+ }
+
+ /**
+ * Start the background consolidator timer if configured.
+ * @internal
+ */
+ private startConsolidatorTimer(): void {
+ const cfg = this.config.consolidator;
+ if (!cfg?.autoRun) return;
+ const intervalMs = cfg.intervalMs ?? 6 * 60 * 60 * 1000; // default 6h
+ if (intervalMs <= 0) return;
+
+ this.consolidatorTimer = setInterval(() => {
+ void this.runAutoConsolidation();
+ }, intervalMs);
+ // Don't block process exit on the timer (Node-only; no-op elsewhere).
+ if (typeof (this.consolidatorTimer as any).unref === 'function') {
+ (this.consolidatorTimer as any).unref();
+ }
+ }
+
+ /**
+ * Run a single consolidator cycle on the active adapter. Emits a
+ * `consolidation:complete` event with the {@link ConsolidationResult}.
+ * @internal
+ */
+ private async runAutoConsolidation(): Promise<void> {
+ try {
+ const consolidator = await this.getConsolidator();
+ const result = await consolidator.runAll();
+ this.emit('consolidation:complete', result);
+ } catch (err) {
+ this.emit('consolidation:failed', {
+ reason: err instanceof Error ? err.message : String(err),
+ });
+ }
+ }
+
+ /**
+ * Get (and lazily construct) the {@link MemoryConsolidator} bound to this
+ * service. Added by ADR-125 Phase 4.
+ */
+ async getConsolidator(): Promise<any> {
+ if (this.consolidator) return this.consolidator;
+ const { MemoryConsolidator } = await import('./consolidator.js');
+ // The consolidator reaches into AgentDBAdapter private state. Cast through
+ // any to bypass TS's view of those fields (the runtime structure is stable
+ // and tested by consolidator.test.ts).
+ this.consolidator = new MemoryConsolidator(this as any, {
+ dedupStrategy: this.config.consolidator?.dedupStrategy ?? 'keep-newest',
+ intervalMs: this.config.consolidator?.intervalMs,
+ });
+ return this.consolidator;
+ }
+
// ===== IMemoryBackend Implementation =====
async store(entry: MemoryEntry): Promise<void> {
- return this.backend.store(entry);
+ await this.backend.store(entry);
+ this.maybeSnapshot();
+ }
+
+ /**
+ * If a snapshot interval is configured and the threshold is hit, fire a
+ * snapshot in the background. Only meaningful when the active backend is
+ * the AgentDBAdapter with persistence enabled.
+ *
+ * @internal β€” ADR-125 Phase 3
+ */
+ private maybeSnapshot(): void {
+ if (!Number.isFinite(this.snapshotInterval)) return;
+ if (this.backend !== this.adapter) return;
+ if (!this.config.persistenceEnabled || !this.config.persistencePath) return;
+
+ this.storeCountSinceSnapshot += 1;
+ if (this.storeCountSinceSnapshot >= this.snapshotInterval) {
+ this.storeCountSinceSnapshot = 0;
+ // Fire and forget β€” saveSnapshot emits its own lifecycle events.
+ void this.adapter.saveSnapshot();
+ }
}
async get(id: string): Promise<MemoryEntry | null> {
@@ -390,7 +535,11 @@ export class UnifiedMemoryService extends EventEmitter implements IMemoryBackend
}
async bulkInsert(entries: MemoryEntry[]): Promise<void> {
- return this.backend.bulkInsert(entries);
+ await this.backend.bulkInsert(entries);
+ // Count each entry toward the snapshot threshold (ADR-125 Phase 3).
+ for (let i = 0; i < entries.length; i++) {
+ this.maybeSnapshot();
+ }
}
async bulkDelete(ids: string[]): Promise<number> {
diff --git a/v3/@claude-flow/memory/src/rvf-backend.ts b/v3/@claude-flow/memory/src/rvf-backend.ts
index 60ae7693d1..92e5327a30 100644
--- a/v3/@claude-flow/memory/src/rvf-backend.ts
+++ b/v3/@claude-flow/memory/src/rvf-backend.ts
@@ -12,7 +12,163 @@ import type {
HealthCheckResult,
MemoryType,
} from './types.js';
-import { HnswLite, cosineSimilarity } from './hnsw-lite.js';
+
+// ----------------------------------------------------------------------------
+// HnswLite β€” small brute-force-degrading HNSW used ONLY by RvfBackend.
+// Inlined here by ADR-125 Phase 3 so the canonical `src/hnsw-index.ts` can be
+// the single HNSW implementation in the public surface. Not exported.
+// ----------------------------------------------------------------------------
+
+interface HnswSearchResult {
+ id: string;
+ score: number;
+}
+
+class HnswLite {
+ private vectors = new Map<string, Float32Array>();
+ private neighbors = new Map<string, Set<string>>();
+ private readonly dimensions: number;
+ private readonly maxNeighbors: number;
+ private readonly efConstruction: number;
+ private readonly metric: string;
+
+ constructor(dimensions: number, m: number, efConstruction: number, metric: string) {
+ this.dimensions = dimensions;
+ this.maxNeighbors = m;
+ this.efConstruction = efConstruction;
+ this.metric = metric;
+ }
+
+ get size(): number { return this.vectors.size; }
+
+ add(id: string, vector: Float32Array): void {
+ this.vectors.set(id, vector);
+ if (this.vectors.size === 1) {
+ this.neighbors.set(id, new Set());
+ return;
+ }
+ const nearest = this.findNearest(vector, this.maxNeighbors);
+ const neighborSet = new Set<string>();
+ for (const n of nearest) {
+ neighborSet.add(n.id);
+ const nNeighbors = this.neighbors.get(n.id);
+ if (nNeighbors) {
+ nNeighbors.add(id);
+ if (nNeighbors.size > this.maxNeighbors * 2) this.pruneNeighbors(n.id);
+ }
+ }
+ this.neighbors.set(id, neighborSet);
+ }
+
+ remove(id: string): void {
+ this.vectors.delete(id);
+ const my = this.neighbors.get(id);
+ if (my) for (const nId of my) this.neighbors.get(nId)?.delete(id);
+ this.neighbors.delete(id);
+ }
+
+ search(query: Float32Array, k: number, threshold?: number): HnswSearchResult[] {
+ if (this.vectors.size === 0) return [];
+ if (this.vectors.size <= k * 2) return this.bruteForce(query, k, threshold);
+
+ const visited = new Set<string>();
+ const candidates: HnswSearchResult[] = [];
+
+ let entryId: string | undefined;
+ let bestScore = -1;
+ for (const [id] of this.vectors) {
+ const score = this.similarity(query, this.vectors.get(id)!);
+ if (score > bestScore) { bestScore = score; entryId = id; }
+ if (visited.size >= Math.min(this.efConstruction, this.vectors.size)) break;
+ visited.add(id);
+ candidates.push({ id, score });
+ }
+
+ if (entryId) {
+ const queue = [entryId];
+ let idx = 0;
+ while (idx < queue.length && visited.size < this.efConstruction * 2) {
+ const currentId = queue[idx++];
+ const currentNeighbors = this.neighbors.get(currentId);
+ if (!currentNeighbors) continue;
+ for (const nId of currentNeighbors) {
+ if (visited.has(nId)) continue;
+ visited.add(nId);
+ const vec = this.vectors.get(nId);
+ if (!vec) continue;
+ const score = this.similarity(query, vec);
+ candidates.push({ id: nId, score });
+ queue.push(nId);
+ }
+ }
+ }
+
+ candidates.sort((a, b) => b.score - a.score);
+ let filtered = candidates;
+ if (threshold !== undefined) filtered = filtered.filter(c => c.score >= threshold);
+ return filtered.slice(0, k);
+ }
+
+ private bruteForce(query: Float32Array, k: number, threshold?: number): HnswSearchResult[] {
+ const results: HnswSearchResult[] = [];
+ for (const [id, vec] of this.vectors) {
+ const score = this.similarity(query, vec);
+ if (threshold !== undefined && score < threshold) continue;
+ results.push({ id, score });
+ }
+ results.sort((a, b) => b.score - a.score);
+ return results.slice(0, k);
+ }
+
+ private findNearest(query: Float32Array, k: number): HnswSearchResult[] {
+ return this.bruteForce(query, k);
+ }
+
+ private pruneNeighbors(id: string): void {
+ const my = this.neighbors.get(id);
+ if (!my) return;
+ const vec = this.vectors.get(id);
+ if (!vec) return;
+ const scored: HnswSearchResult[] = [];
+ for (const nId of my) {
+ const nVec = this.vectors.get(nId);
+ if (!nVec) continue;
+ scored.push({ id: nId, score: this.similarity(vec, nVec) });
+ }
+ scored.sort((a, b) => b.score - a.score);
+ const keep = new Set(scored.slice(0, this.maxNeighbors).map(s => s.id));
+ for (const nId of my) if (!keep.has(nId)) my.delete(nId);
+ }
+
+ private similarity(a: Float32Array, b: Float32Array): number {
+ if (this.metric === 'dot') return dotProduct(a, b);
+ if (this.metric === 'euclidean') return 1 / (1 + euclideanDistance(a, b));
+ return cosineSimilarity(a, b);
+ }
+}
+
+function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+ let dot = 0, normA = 0, normB = 0;
+ for (let i = 0; i < a.length; i++) {
+ dot += a[i] * b[i];
+ normA += a[i] * a[i];
+ normB += b[i] * b[i];
+ }
+ if (normA === 0 || normB === 0) return 0;
+ return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+function dotProduct(a: Float32Array, b: Float32Array): number {
+ let sum = 0;
+ for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
+ return sum;
+}
+
+function euclideanDistance(a: Float32Array, b: Float32Array): number {
+ let sum = 0;
+ for (let i = 0; i < a.length; i++) { const d = a[i] - b[i]; sum += d * d; }
+ return Math.sqrt(sum);
+}
/** Validate a file path is safe (no null bytes, no traversal above root) */
function validatePath(p: string): void {