Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

fix(memory): migrate ADR-323 provenance_type on the bridge path, and stop the swallowed error surfacing as a WAL refusal (#2844)

Dragan Spiridonov0e957da3a435
diff --git a/v3/@claude-flow/cli/__tests__/memory-bridge-failure-diagnostics.test.ts b/v3/@claude-flow/cli/__tests__/memory-bridge-failure-diagnostics.test.ts
new file mode 100644
index 0000000000..23b8f194d7
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/memory-bridge-failure-diagnostics.test.ts
@@ -0,0 +1,105 @@
+/**
+ * Regression: a failed bridge init must be diagnosable and recoverable.
+ *
+ * `getRegistry()` latches `bridgeAvailable = false` for the life of the
+ * process, so one transient init failure routes every later write to the
+ * sql.js whole-image fallback β€” which then refuses whenever -wal/-shm
+ * sidecars are present. Observed in the wild: an MCP server silently dropped
+ * every `memory_store` for hours while CLI writes through the same database
+ * succeeded, and the only symptom was a refusal naming a cause the operator
+ * could not check.
+ *
+ * Three properties close that:
+ * 1. the failure reason is recorded rather than swallowed by a bare catch;
+ * 2. `shutdownBridge()` clears the latch even when init never produced a
+ * registry β€” previously the reset sat inside `if (registryInstance)`,
+ * which is null in exactly the case needing a reset;
+ * 3. a degradation notice is never filtered out as init noise.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// `@claude-flow/memory` is externalized in vitest.config.ts (it is a try/catch
+// dynamic import that degrades to the sql.js path), so the import inside
+// getRegistry() always throws here. That is the init failure under test β€” the
+// assertions deliberately check the recorded/cleared behaviour rather than the
+// flavour of the underlying error, which is not the property that matters.
+
+describe('bridge failure diagnostics', () => {
+ beforeEach(() => {
+ vi.resetModules();
+ });
+
+ it('should report null before the bridge has been tried', async () => {
+ const { getBridgeFailureReason } = await import('../src/memory/memory-bridge.js');
+
+ expect(getBridgeFailureReason()).toBeNull();
+ });
+
+ it('should record why init failed instead of swallowing the error', async () => {
+ const bridge = await import('../src/memory/memory-bridge.js');
+
+ await bridge.bridgeStoreEntry({ key: 'k', value: 'v' });
+
+ const reason = bridge.getBridgeFailureReason();
+ expect(reason).toBeTruthy();
+ expect(typeof reason).toBe('string');
+ });
+
+ it('should return null from bridgeStoreEntry so the caller falls back', async () => {
+ const bridge = await import('../src/memory/memory-bridge.js');
+
+ const result = await bridge.bridgeStoreEntry({ key: 'k', value: 'v' });
+
+ expect(result).toBeNull();
+ });
+
+ it('should clear the latched failure on shutdown even with no registry instance', async () => {
+ const bridge = await import('../src/memory/memory-bridge.js');
+ await bridge.bridgeStoreEntry({ key: 'k', value: 'v' });
+ expect(bridge.getBridgeFailureReason()).toBeTruthy();
+
+ await bridge.shutdownBridge();
+
+ expect(bridge.getBridgeFailureReason()).toBeNull();
+ });
+
+ it('should re-attempt init after shutdown rather than staying latched', async () => {
+ const bridge = await import('../src/memory/memory-bridge.js');
+ await bridge.bridgeStoreEntry({ key: 'k', value: 'v' });
+ await bridge.shutdownBridge();
+ expect(bridge.getBridgeFailureReason()).toBeNull();
+
+ // A latched `bridgeAvailable === false` short-circuits getRegistry() before
+ // it retries, so the reason would stay null. Repopulating it proves the
+ // retry actually happened.
+ await bridge.bridgeStoreEntry({ key: 'k2', value: 'v2' });
+
+ expect(bridge.getBridgeFailureReason()).toBeTruthy();
+ });
+});
+
+describe('init log suppression', () => {
+ it('should suppress a noisy init banner', async () => {
+ const { shouldSuppressInitLog } = await import('../src/memory/memory-bridge.js');
+
+ expect(shouldSuppressInitLog('[AgentDB] Initialized with better-sqlite3 + ruvector')).toBe(true);
+ });
+
+ it('should NOT suppress the better-sqlite3 fallback notice', async () => {
+ const { shouldSuppressInitLog } = await import('../src/memory/memory-bridge.js');
+
+ expect(shouldSuppressInitLog('[AgentDB] better-sqlite3 not available, using sql.js WASM')).toBe(false);
+ });
+
+ it('should NOT suppress a generic falling-back notice', async () => {
+ const { shouldSuppressInitLog } = await import('../src/memory/memory-bridge.js');
+
+ expect(shouldSuppressInitLog('[HNSWLibBackend] falling back to brute force')).toBe(false);
+ });
+
+ it('should leave unrelated output alone', async () => {
+ const { shouldSuppressInitLog } = await import('../src/memory/memory-bridge.js');
+
+ expect(shouldSuppressInitLog('user-facing progress line')).toBe(false);
+ });
+});
diff --git a/v3/@claude-flow/cli/__tests__/memory-bridge-provenance-migration.test.ts b/v3/@claude-flow/cli/__tests__/memory-bridge-provenance-migration.test.ts
new file mode 100644
index 0000000000..d6a89a7f77
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/memory-bridge-provenance-migration.test.ts
@@ -0,0 +1,94 @@
+/**
+ * Regression: the bridge must migrate `provenance_type` onto a pre-ADR-323
+ * `memory_entries` table.
+ *
+ * ADR-323 added `provenance_type` to bridgeStoreEntry()'s INSERT.
+ * `ensureSchemaColumns()` backfills the column on the sql.js path, but the
+ * bridge path only ran `CREATE TABLE IF NOT EXISTS` β€” a no-op on a table that
+ * already exists. So on any database created before ADR-323, every bridge
+ * write threw:
+ *
+ * SqliteError: table memory_entries has no column named provenance_type
+ *
+ * which the catch at the end of bridgeStoreEntry() swallowed, returning null.
+ * The caller then demoted to the sql.js whole-image path, whose WAL-sidecar
+ * guard reported "memory database has an active native WAL connection …
+ * restore the native better-sqlite3 bridge" β€” a cause with nothing to do with
+ * the real failure. Observed live: an MCP server dropped every memory_store
+ * for hours while better-sqlite3 was healthy the whole time.
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import initSqlJs from 'sql.js';
+import { ensureBridgeSchema } from '../src/memory/memory-bridge.js';
+
+const LEGACY_TABLE = `CREATE TABLE memory_entries (
+ id TEXT PRIMARY KEY,
+ key TEXT NOT NULL,
+ namespace TEXT DEFAULT 'default',
+ content TEXT NOT NULL,
+ type TEXT DEFAULT 'semantic',
+ status TEXT DEFAULT 'active',
+ UNIQUE(namespace, key)
+)`;
+
+let SQL: Awaited<ReturnType<typeof initSqlJs>>;
+
+const columnsOf = (db: { exec: (sql: string) => Array<{ values: unknown[][] }> }): string[] => {
+ const res = db.exec('PRAGMA table_info(memory_entries)');
+ return res.length ? res[0].values.map((row) => String(row[1])) : [];
+};
+
+beforeAll(async () => {
+ SQL = await initSqlJs();
+});
+
+describe('bridge schema migration (ADR-323 provenance_type)', () => {
+ it('should add provenance_type to a pre-ADR-323 table', () => {
+ const db = new SQL.Database();
+ db.run(LEGACY_TABLE);
+ expect(columnsOf(db)).not.toContain('provenance_type');
+
+ ensureBridgeSchema(db);
+
+ expect(columnsOf(db)).toContain('provenance_type');
+ });
+
+ it('should let the ADR-323 insert succeed after migration', () => {
+ const db = new SQL.Database();
+ db.run(LEGACY_TABLE);
+ ensureBridgeSchema(db);
+
+ const insert = () => db.run(
+ 'INSERT INTO memory_entries (id, key, namespace, content, provenance_type) VALUES (?, ?, ?, ?, ?)',
+ ['id-1', 'k', 'ns', 'v', 'agent_output'],
+ );
+
+ expect(insert).not.toThrow();
+ });
+
+ it('should include provenance_type when creating the table fresh', () => {
+ const db = new SQL.Database();
+
+ ensureBridgeSchema(db);
+
+ expect(columnsOf(db)).toContain('provenance_type');
+ });
+
+ it('should be idempotent across repeated calls', () => {
+ const db = new SQL.Database();
+ db.run(LEGACY_TABLE);
+
+ ensureBridgeSchema(db);
+ const second = () => ensureBridgeSchema(db);
+
+ expect(second).not.toThrow();
+ expect(columnsOf(db).filter((c) => c === 'provenance_type')).toHaveLength(1);
+ });
+
+ it('should report success when the schema is usable', () => {
+ const db = new SQL.Database();
+ db.run(LEGACY_TABLE);
+
+ expect(ensureBridgeSchema(db)).toBe(true);
+ });
+});
diff --git a/v3/@claude-flow/cli/src/memory/memory-bridge.ts b/v3/@claude-flow/cli/src/memory/memory-bridge.ts
index c2b0a46b3f..2ba10facde 100644
--- a/v3/@claude-flow/cli/src/memory/memory-bridge.ts
+++ b/v3/@claude-flow/cli/src/memory/memory-bridge.ts
@@ -26,6 +26,17 @@ import { createRequire } from 'node:module';
let registryPromise: Promise<any> | null = null;
let registryInstance: any = null;
let bridgeAvailable: boolean | null = null;
+/**
+ * Why the bridge is unavailable, when it is.
+ *
+ * `bridgeAvailable = false` latches for the life of the process, so a single
+ * transient init failure (a slow Xenova/ONNX fetch, a locked db) routes every
+ * later write to the sql.js whole-image fallback β€” which then refuses whenever
+ * -wal/-shm sidecars are present. Without this, that refusal is the only
+ * symptom the caller ever sees, and it names a cause ("restore the native
+ * better-sqlite3 bridge") the caller has no way to check.
+ */
+let bridgeFailureReason: string | null = null;
/**
* ADR-323: reuse memory-initializer's provenance-type allowlist rather than
@@ -112,6 +123,32 @@ function generateId(prefix: string): string {
return `${prefix}_${Date.now()}_${crypto.randomBytes(8).toString('hex')}`;
}
+/** Noisy init banners that carry no operational signal. */
+const INIT_LOG_NOISE = [
+ 'Transformers.js',
+ 'better-sqlite3',
+ '[AgentDB]',
+ '[HNSWLibBackend]',
+ 'RuVector graph',
+];
+
+/**
+ * Should this init-time log line be swallowed?
+ *
+ * A DEGRADATION notice never is. AgentDB logs "[AgentDB] better-sqlite3 not
+ * available, using sql.js WASM" when it falls back to WASM, and both the
+ * '[AgentDB]' and 'better-sqlite3' entries above match it β€” so the one line
+ * explaining why the native driver was not in use was being discarded as
+ * noise. Suppress the banners, keep the bad news.
+ */
+export function shouldSuppressInitLog(msg: string): boolean {
+ const degradation = msg.includes('not available')
+ || msg.includes('falling back')
+ || msg.includes('fallback');
+ if (degradation) return false;
+ return INIT_LOG_NOISE.some((needle) => msg.includes(needle));
+}
+
/**
* Lazily initialize the ControllerRegistry singleton.
* Returns null if @claude-flow/memory is not available.
@@ -127,15 +164,12 @@ async function getRegistry(dbPath?: string): Promise<any | null> {
const { ControllerRegistry } = await import('@claude-flow/memory');
const registry = new ControllerRegistry();
- // Suppress noisy console.log during init
+ // Suppress noisy console.log during init β€” but never suppress a
+ // DEGRADATION notice (see shouldSuppressInitLog).
const origLog = console.log;
console.log = (...args: unknown[]) => {
const msg = String(args[0] ?? '');
- if (msg.includes('Transformers.js') ||
- msg.includes('better-sqlite3') ||
- msg.includes('[AgentDB]') ||
- msg.includes('[HNSWLibBackend]') ||
- msg.includes('RuVector graph')) return;
+ if (shouldSuppressInitLog(msg)) return;
origLog.apply(console, args);
};
@@ -363,8 +397,13 @@ async function getRegistry(dbPath?: string): Promise<any | null> {
registryInstance = registry;
bridgeAvailable = true;
+ bridgeFailureReason = null;
return registry;
- } catch {
+ } catch (err) {
+ // Record WHY. This latches for the process lifetime (see the
+ // bridgeFailureReason doc comment), so discarding the error here
+ // makes the resulting sql.js-fallback refusal undiagnosable.
+ bridgeFailureReason = err instanceof Error ? err.message : String(err);
bridgeAvailable = false;
registryPromise = null;
return null;
@@ -543,6 +582,64 @@ async function logAttestation(
// every bridge call. WeakSet so handles GC without leaking.
const _schemaEnsuredDbs = new WeakSet<object>();
+/**
+ * Create/migrate the bridge's `memory_entries` table on `db`.
+ *
+ * Returns true when the schema is known-good, false when the database was not
+ * writable (caller then leaves it un-ensured so a later writable call retries).
+ *
+ * Exported so the ADR-323 column migration can be tested directly: the bug it
+ * fixes was invisible end-to-end, because the resulting error was swallowed and
+ * reported as an unrelated WAL-sidecar refusal.
+ */
+export function ensureBridgeSchema(db: { exec: (sql: string) => unknown }): boolean {
+ try {
+ db.exec(`CREATE TABLE IF NOT EXISTS memory_entries (
+ id TEXT PRIMARY KEY,
+ key TEXT NOT NULL,
+ namespace TEXT DEFAULT 'default',
+ content TEXT NOT NULL,
+ type TEXT DEFAULT 'semantic',
+ embedding TEXT,
+ embedding_model TEXT DEFAULT 'local',
+ embedding_dimensions INTEGER,
+ tags TEXT,
+ metadata TEXT,
+ owner_id TEXT,
+ created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+ updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+ expires_at INTEGER,
+ last_accessed_at INTEGER,
+ access_count INTEGER DEFAULT 0,
+ status TEXT DEFAULT 'active',
+ provenance_type TEXT DEFAULT 'unknown',
+ UNIQUE(namespace, key)
+ )`);
+ // ADR-323 added provenance_type to bridgeStoreEntry's INSERT, and
+ // ensureSchemaColumns() backfills it on the sql.js path β€” the bridge path
+ // had no equivalent. `CREATE TABLE IF NOT EXISTS` is a no-op on a table
+ // created before ADR-323, so on any pre-existing database every bridge
+ // write threw "table memory_entries has no column named provenance_type",
+ // was swallowed by the catch at the end of bridgeStoreEntry(), and demoted
+ // the caller to the sql.js whole-image path β€” which then refused with a
+ // WAL-sidecar error naming an unrelated cause. Silent, total write loss.
+ try {
+ db.exec(`ALTER TABLE memory_entries ADD COLUMN provenance_type TEXT DEFAULT 'unknown'`);
+ } catch {
+ // Already present (the common case). SQLite has no ADD COLUMN IF NOT
+ // EXISTS, so attempt-and-ignore is the idiom.
+ }
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_ns ON memory_entries(namespace)`);
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_key ON memory_entries(key)`);
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_status ON memory_entries(status)`);
+ return true;
+ } catch {
+ // Read-only database β€” leave it un-ensured so a writable call can retry.
+ return false;
+ }
+}
+
+
/**
* Get the AgentDB database handle and ensure memory_entries table exists.
* Returns null if not available.
@@ -558,36 +655,7 @@ function getDb(registry: any): any | null {
// call (store/search/get) was pure per-op overhead. Keyed by handle via a
// WeakSet so a new db instance re-ensures without a stale global flag.
if (!_schemaEnsuredDbs.has(db)) {
- try {
- db.exec(`CREATE TABLE IF NOT EXISTS memory_entries (
- id TEXT PRIMARY KEY,
- key TEXT NOT NULL,
- namespace TEXT DEFAULT 'default',
- content TEXT NOT NULL,
- type TEXT DEFAULT 'semantic',
- embedding TEXT,
- embedding_model TEXT DEFAULT 'local',
- embedding_dimensions INTEGER,
- tags TEXT,
- metadata TEXT,
- owner_id TEXT,
- created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
- updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
- expires_at INTEGER,
- last_accessed_at INTEGER,
- access_count INTEGER DEFAULT 0,
- status TEXT DEFAULT 'active',
- UNIQUE(namespace, key)
- )`);
- // Ensure indexes
- db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_ns ON memory_entries(namespace)`);
- db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_key ON memory_entries(key)`);
- db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_status ON memory_entries(status)`);
- _schemaEnsuredDbs.add(db);
- } catch {
- // Table already exists or db is read-only β€” that's fine. Don't mark
- // ensured on failure so a later writable call can retry.
- }
+ if (ensureBridgeSchema(db)) _schemaEnsuredDbs.add(db);
}
// ─── #2256-followup: rescue agentdb.embedder when its transformers.js
@@ -915,6 +983,11 @@ export async function bridgeStoreEntry(options: {
// never triggers the #2735 whole-image guard with a misleading
// "active native WAL connection" error.
const msg = err instanceof Error ? err.message : String(err);
+ // Returning null below demotes the caller to the sql.js whole-image path,
+ // whose WAL-sidecar guard then reports a cause that has nothing to do with
+ // what actually went wrong here. Record the real error so it can be
+ // surfaced alongside that guard's message.
+ bridgeFailureReason = msg;
if (/UNIQUE constraint failed/i.test(msg)) {
return {
success: false,
@@ -1802,8 +1875,30 @@ export async function getControllerRegistry(dbPath?: string): Promise<any | null
return getRegistry(dbPath);
}
+/**
+ * Why the bridge last declined a write, or null when it has not.
+ *
+ * Deliberately NOT gated on `bridgeAvailable === false`. A bridge that
+ * initialised fine can still fail every write β€” a schema mismatch throws
+ * per-operation while the registry stays healthy β€” and that case is exactly
+ * the one worth reporting, since the caller then demotes to a fallback whose
+ * error message describes something else entirely.
+ *
+ * Callers that surface a degraded-path error should include this so the
+ * operator learns the cause instead of only the symptom.
+ */
+export function getBridgeFailureReason(): string | null {
+ return bridgeFailureReason;
+}
+
/**
* Shutdown the bridge and release resources.
+ *
+ * The cached state is cleared unconditionally. Previously the reset lived
+ * inside `if (registryInstance)`, so it could not clear a FAILED init β€” the
+ * one state that actually needs clearing, since `registryInstance` is null
+ * precisely when init failed. A process that latched `bridgeAvailable = false`
+ * therefore had no recovery path short of a restart.
*/
export async function shutdownBridge(): Promise<void> {
if (registryInstance) {
@@ -1812,10 +1907,11 @@ export async function shutdownBridge(): Promise<void> {
} catch {
// Best-effort
}
- registryInstance = null;
- registryPromise = null;
- bridgeAvailable = null;
}
+ registryInstance = null;
+ registryPromise = null;
+ bridgeAvailable = null;
+ bridgeFailureReason = null;
}
// ===== Phase 3: ReasoningBank pattern operations =====
diff --git a/v3/@claude-flow/cli/src/memory/memory-initializer.ts b/v3/@claude-flow/cli/src/memory/memory-initializer.ts
index 53105f3aa7..368f7730ea 100644
--- a/v3/@claude-flow/cli/src/memory/memory-initializer.ts
+++ b/v3/@claude-flow/cli/src/memory/memory-initializer.ts
@@ -180,6 +180,31 @@ async function getBridge(): Promise<typeof import('./memory-bridge.js') | null>
}
}
+/**
+ * Build the WAL-sidecar refusal message, naming the bridge failure when one
+ * was recorded.
+ *
+ * The refusal tells the operator to "restore the native better-sqlite3
+ * bridge" but, on its own, gives them no way to discover why it is down β€”
+ * and the usual cause is a latched init failure inside the bridge, not a
+ * missing better-sqlite3. Appending the recorded reason turns an unactionable
+ * message into a diagnosis.
+ */
+async function walRefusalError(operation: 'write' | 'read/write'): Promise<string> {
+ const base = 'memory database has an active native WAL connection '
+ + '(found -wal/-shm sidecar files) β€” refusing an unsafe sql.js '
+ + `whole-image ${operation}. Retry once the native writer completes, or `
+ + 'restore the native better-sqlite3 bridge.';
+ try {
+ const bridge = await getBridge();
+ const reason = bridge?.getBridgeFailureReason?.();
+ if (reason) return `${base} Bridge unavailable: ${reason}`;
+ } catch {
+ // Diagnostics must never mask the refusal they annotate.
+ }
+ return base;
+}
+
/**
* Enhanced schema with pattern confidence, temporal decay, versioning
* Vector embeddings enabled for semantic search
@@ -2753,10 +2778,7 @@ export async function storeEntry(options: {
return {
success: false,
id: '',
- error: 'memory database has an active native WAL connection ' +
- '(found -wal/-shm sidecar files) β€” refusing an unsafe sql.js ' +
- 'whole-image write. Retry once the native writer completes, or ' +
- 'restore the native better-sqlite3 bridge.',
+ error: await walRefusalError('write'),
};
}
@@ -3402,10 +3424,7 @@ export async function getEntry(options: {
return {
success: false,
found: false,
- error: 'memory database has an active native WAL connection ' +
- '(found -wal/-shm sidecar files) β€” refusing an unsafe sql.js ' +
- 'whole-image read/write. Retry once the native writer completes, ' +
- 'or restore the native better-sqlite3 bridge.',
+ error: await walRefusalError('read/write'),
};
}
@@ -3559,10 +3578,7 @@ export async function deleteEntry(options: {
key,
namespace,
remainingEntries: 0,
- error: 'memory database has an active native WAL connection ' +
- '(found -wal/-shm sidecar files) β€” refusing an unsafe sql.js ' +
- 'whole-image write. Retry once the native writer completes, or ' +
- 'restore the native better-sqlite3 bridge.',
+ error: await walRefusalError('write'),
};
}
diff --git a/v3/@claude-flow/cli/vitest.config.ts b/v3/@claude-flow/cli/vitest.config.ts
index 67e0d5ac90..c46713cf7d 100644
--- a/v3/@claude-flow/cli/vitest.config.ts
+++ b/v3/@claude-flow/cli/vitest.config.ts
@@ -22,6 +22,9 @@ export default defineConfig({
// module resolution at transform time.
if (source.startsWith('agentic-flow')) return { id: source, external: true };
if (source.startsWith('agentdb')) return { id: source, external: true };
+ // memory-bridge dynamic-imports this inside try/catch and degrades to
+ // the sql.js path; its dist is absent unless the workspace was built.
+ if (source === '@claude-flow/memory') return { id: source, external: true };
if (source.startsWith('@ruvector/')) return { id: source, external: true };
if (source.startsWith('@huggingface/transformers')) return { id: source, external: true };
if (source.startsWith('@xenova/transformers')) return { id: source, external: true };