Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

fix(memory): distill both project memory stores (#4338)

ordinary beingfc4b364803b7
diff --git a/v3/@claude-flow/cli/__tests__/distill-both-stores-3446.test.ts b/v3/@claude-flow/cli/__tests__/distill-both-stores-3446.test.ts
new file mode 100644
index 0000000000..012bbd4d43
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/distill-both-stores-3446.test.ts
@@ -0,0 +1,218 @@
+/**
+ * #3446: distillation must cover the store the MCP bridge actually writes.
+ *
+ * `memory.db` and `agentdb-memory.db` are separate stores. Before this fix,
+ * `memory distill run` and the daemon's consolidate worker only opened
+ * memory.db. In an MCP-only project that file often has no AgentDB target
+ * tables, so every tick skipped while the rows accumulated in the sibling.
+ *
+ * The pure tests exercise the multi-store orchestration without requiring a
+ * native SQLite build; the CLI test runs the real service when better-sqlite3
+ * is available.
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import { mkdtempSync, mkdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+import type { Command, CommandContext } from '../src/types.js';
+import {
+ aggregateDistillReports,
+ projectMemoryDbPaths,
+ runDistillationForStores,
+ type DistillReport,
+} from '../src/services/memory-distillation.js';
+import { distillCommand } from '../src/commands/memory-distill.js';
+
+let Database: any;
+let haveNative = false;
+try {
+ Database = (await import('better-sqlite3')).default;
+ // The package can resolve while its native binary is missing; probe the
+ // constructor so the native-only cases skip cleanly on a source-only host.
+ const probe = new Database(':memory:');
+ probe.close();
+ haveNative = true;
+} catch { haveNative = false; }
+
+function report(extra: Partial<DistillReport> = {}): DistillReport {
+ return {
+ processed: 0, episodes: 0, episodeEmbeddings: 0, patterns: 0,
+ patternEmbeddings: 0, causalEdges: 0, promoted: 0, byProvenance: {},
+ namespaces: [], dryRun: false, spendUsd: 0, ...extra,
+ };
+}
+
+function makeCtx(flags: Record<string, unknown>, cwd: string): CommandContext {
+ return { args: [], flags: { _: [], ...flags }, cwd, interactive: false } as CommandContext;
+}
+
+function sub(name: string): Command {
+ const cmd = distillCommand.subcommands?.find((c) => c.name === name);
+ if (!cmd?.action) throw new Error(`memory distill ${name} has no action`);
+ return cmd;
+}
+
+/** Source rows plus the AgentDB target tables. */
+function seedAgentStore(dbPath: string): void {
+ const db = new Database(dbPath);
+ db.exec(`CREATE TABLE memory_entries (
+ id TEXT PRIMARY KEY, key TEXT, namespace TEXT DEFAULT 'default',
+ content TEXT, embedding TEXT, status TEXT DEFAULT 'active'
+ )`);
+ db.exec(`CREATE TABLE episodes (
+ id INTEGER PRIMARY KEY AUTOINCREMENT, ts INTEGER DEFAULT 0, session_id TEXT NOT NULL,
+ task TEXT NOT NULL, input TEXT, output TEXT, critique TEXT, reward REAL DEFAULT 0,
+ success BOOLEAN DEFAULT 0, latency_ms INTEGER, tokens_used INTEGER, tags TEXT,
+ metadata JSON, created_at INTEGER DEFAULT 0
+ )`);
+ db.exec(`CREATE TABLE reasoning_patterns (
+ id INTEGER PRIMARY KEY AUTOINCREMENT, ts INTEGER DEFAULT 0, task_type TEXT NOT NULL,
+ approach TEXT NOT NULL, success_rate REAL NOT NULL DEFAULT 0, uses INTEGER DEFAULT 0,
+ avg_reward REAL DEFAULT 0, tags TEXT, metadata TEXT
+ )`);
+ db.exec(`CREATE TABLE pattern_embeddings (
+ pattern_id INTEGER PRIMARY KEY, embedding BLOB NOT NULL
+ )`);
+ db.exec(`CREATE TABLE causal_edges (
+ id INTEGER PRIMARY KEY AUTOINCREMENT, from_memory_id INTEGER NOT NULL, from_memory_type TEXT NOT NULL,
+ to_memory_id INTEGER NOT NULL, to_memory_type TEXT NOT NULL, similarity REAL DEFAULT 0,
+ confidence REAL DEFAULT 0.5, mechanism TEXT, metadata JSON, created_at INTEGER DEFAULT 0
+ )`);
+ const ins = db.prepare('INSERT INTO memory_entries (id, key, namespace, content, embedding) VALUES (?,?,?,?,?)');
+ const vec = (seed: number) => JSON.stringify(Array.from({ length: 8 }, (_, i) => Math.sin(seed + i)));
+ ins.run('a1', 'k1', 'commands', 'refactor the auth module in src/auth.ts', vec(1));
+ ins.run('a2', 'k2', 'feedback', JSON.stringify({ taskId: 'edit-a', success: true }), vec(2));
+ db.close();
+}
+
+/** The CLI-only store: source rows but no AgentDB target tables. */
+function seedCliStore(dbPath: string): void {
+ const db = new Database(dbPath);
+ db.exec(`CREATE TABLE memory_entries (
+ id TEXT PRIMARY KEY, key TEXT, namespace TEXT DEFAULT 'default',
+ content TEXT, embedding TEXT, status TEXT DEFAULT 'active'
+ )`);
+ db.prepare('INSERT INTO memory_entries (id, key, namespace, content, embedding) VALUES (?,?,?,?,?)')
+ .run('c1', 'k3', 'commands', 'unrelated CLI-only row', JSON.stringify([1, 2, 3]));
+ db.close();
+}
+
+function patternCount(dbPath: string): number {
+ const db = new Database(dbPath, { readonly: true });
+ const c = (db.prepare('SELECT COUNT(*) AS c FROM reasoning_patterns').get() as { c: number }).c;
+ db.close();
+ return c;
+}
+
+const tempDirs: string[] = [];
+function tempDir(): string {
+ const dir = mkdtempSync(join(tmpdir(), 'distill-3446-'));
+ tempDirs.push(dir);
+ return dir;
+}
+
+afterEach(() => {
+ while (tempDirs.length) rmSync(tempDirs.pop()!, { recursive: true, force: true });
+});
+
+describe('#3446 multi-store distillation orchestration', () => {
+ it('resolves both stores under the same memory root', () => {
+ const dir = tempDir();
+ const prev = process.env.CLAUDE_FLOW_MEMORY_PATH;
+ try {
+ process.env.CLAUDE_FLOW_MEMORY_PATH = join(dir, 'state');
+ expect(projectMemoryDbPaths(dir)).toEqual([
+ join(dir, 'state', 'memory.db'),
+ join(dir, 'state', 'agentdb-memory.db'),
+ ]);
+ } finally {
+ if (prev === undefined) delete process.env.CLAUDE_FLOW_MEMORY_PATH;
+ else process.env.CLAUDE_FLOW_MEMORY_PATH = prev;
+ }
+ });
+
+ it('honors an explicit CLAUDE_FLOW_DB_PATH as a single-store pin', () => {
+ const dir = tempDir();
+ const prev = process.env.CLAUDE_FLOW_DB_PATH;
+ try {
+ process.env.CLAUDE_FLOW_DB_PATH = join('.custom', 'pinned.db');
+ expect(projectMemoryDbPaths(dir)).toEqual([join(dir, '.custom', 'pinned.db')]);
+ } finally {
+ if (prev === undefined) delete process.env.CLAUDE_FLOW_DB_PATH;
+ else process.env.CLAUDE_FLOW_DB_PATH = prev;
+ }
+ });
+
+ it('aggregates per-store counts and only reports skipped when every store skipped', () => {
+ const stores = [
+ { dbPath: 'memory.db', report: report({ processed: 2, patterns: 1, namespaces: ['commands'] }) },
+ { dbPath: 'agentdb-memory.db', report: report({ processed: 3, patterns: 2, skipped: 'target table episodes missing' }) },
+ ];
+ const merged = aggregateDistillReports(stores);
+ expect(merged.processed).toBe(5);
+ expect(merged.patterns).toBe(3);
+ expect(merged.namespaces).toEqual(['commands']);
+ expect(merged.skipped).toBeUndefined();
+
+ const allSkipped = aggregateDistillReports([
+ { dbPath: 'a', report: report({ skipped: 'no-db' }) },
+ { dbPath: 'b', report: report({ skipped: 'no memory_entries' }) },
+ ]);
+ expect(allSkipped.skipped).toContain('no-db');
+ expect(allSkipped.skipped).toContain('no memory_entries');
+ });
+
+ it('invokes the runner once per resolved store and preserves partial failure', async () => {
+ const seen: string[] = [];
+ const result = await runDistillationForStores(
+ ['/tmp/a.db', '/tmp/b.db', '/tmp/a.db'],
+ { dryRun: true },
+ async ({ dbPath }) => {
+ seen.push(dbPath);
+ if (dbPath.endsWith('b.db')) throw new Error('boom');
+ return report({ processed: 1, dryRun: true });
+ },
+ );
+
+ expect(seen).toHaveLength(2);
+ expect(result.stores).toHaveLength(2);
+ expect(result.report.processed).toBe(1);
+ expect(result.report.skipped).toBeUndefined();
+ expect(result.stores[1].report.skipped).toContain('boom');
+ });
+});
+
+describe.skipIf(!haveNative)('#3446 CLI covers both stores without --db', () => {
+ it('distills the AgentDB sibling and reports the CLI store skip', async () => {
+ const dir = tempDir();
+ mkdirSync(join(dir, '.swarm'), { recursive: true });
+ const cliStore = join(dir, '.swarm', 'memory.db');
+ const agentStore = join(dir, '.swarm', 'agentdb-memory.db');
+ seedCliStore(cliStore);
+ seedAgentStore(agentStore);
+
+ const result = await sub('run').action!(makeCtx({}, dir));
+ expect(result?.success).toBe(true);
+ const data = result?.data as any;
+
+ expect(data.dbPaths).toEqual([cliStore, agentStore]);
+ expect(data.stores).toHaveLength(2);
+ expect(data.report.patterns).toBeGreaterThan(0);
+ expect(patternCount(agentStore)).toBeGreaterThan(0);
+ expect(data.stores.find((s: any) => s.dbPath === cliStore)?.report.skipped)
+ .toContain('target table');
+ });
+
+ it('an explicit --db remains a single-store operation', async () => {
+ const dir = tempDir();
+ const dbPath = join(dir, 'only.db');
+ seedAgentStore(dbPath);
+
+ const result = await sub('run').action!(makeCtx({ db: dbPath }, dir));
+ const data = result?.data as any;
+ expect(data.dbPaths).toEqual([dbPath]);
+ expect(data.stores).toHaveLength(1);
+ expect(patternCount(dbPath)).toBeGreaterThan(0);
+ });
+});
diff --git a/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.ts b/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.ts
index 4086260793..0299a09573 100644
--- a/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.ts
+++ b/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.ts
@@ -117,6 +117,24 @@ describe.skipIf(!haveNative)('WorkerDaemon consolidate worker — ADR-174 M3 wir
expect((result as any).patternsConsolidated).toBe(metrics.patternsConsolidated);
});
+ it('distills agentdb-memory.db when memory.db is absent (#3446)', async () => {
+ const agentDbPath = join(tempDir, '.swarm', 'agentdb-memory.db');
+ seedDb(agentDbPath);
+ const daemon = new WorkerDaemon(tempDir);
+ const result = await (daemon as any).runConsolidateWorker() as any;
+
+ expect(reasoningPatternCount(agentDbPath)).toBeGreaterThan(0);
+ expect(result.patternsConsolidated).toBeGreaterThan(0);
+ expect(result.stores).toEqual(expect.arrayContaining([
+ expect.objectContaining({ dbPath: agentDbPath }),
+ ]));
+
+ const metrics = JSON.parse(readFileSync(metricsFile, 'utf-8'));
+ expect(metrics.stores).toEqual(expect.arrayContaining([
+ expect.objectContaining({ dbPath: agentDbPath }),
+ ]));
+ });
+
it('is idempotent — a second tick processes nothing new (cursor-driven)', async () => {
seedDb(dbPath);
const daemon = new WorkerDaemon(tempDir);
diff --git a/v3/@claude-flow/cli/src/commands/memory-distill.ts b/v3/@claude-flow/cli/src/commands/memory-distill.ts
index 49430ac78c..0a95fb1f33 100644
--- a/v3/@claude-flow/cli/src/commands/memory-distill.ts
+++ b/v3/@claude-flow/cli/src/commands/memory-distill.ts
@@ -30,10 +30,10 @@ import * as path from 'path';
import type { Command, CommandContext, CommandResult } from '../types.js';
import { output } from '../output.js';
import {
- runDistillation,
+ runDistillationForStores,
+ projectMemoryDbPaths,
defaultMemoryDbPath,
type DistillOptions,
- type DistillReport,
} from '../services/memory-distillation.js';
// ============================================================================
@@ -169,7 +169,15 @@ const runCommand: Command = {
return { success: false, exitCode: 1 };
}
- const dbPath = resolveDbPath(ctx);
+ // An explicit --db remains a single-store operation. Without it, cover
+ // both stores the CLI/MCP surfaces can write (#3446): memory.db and the
+ // native AgentDB sibling. An MCP-only project often has only the latter,
+ // which is why the worker used to skip with "target table ... missing".
+ const explicitDb = typeof ctx.flags.db === 'string' && ctx.flags.db.trim()
+ ? path.resolve(ctx.flags.db)
+ : undefined;
+ const dbPaths = explicitDb ? [explicitDb] : projectMemoryDbPaths(ctx.cwd || process.cwd());
+ const dbPath = explicitDb ?? dbPaths[0];
output.writeln();
output.writeln(output.bold('Memory Distillation (ADR-174)'));
@@ -179,8 +187,7 @@ const runCommand: Command = {
const spinner = output.createSpinner({ text: 'Distilling memory_entries...', spinner: 'dots' });
spinner.start();
- const distillOptions: DistillOptions = {
- dbPath,
+ const distillOptions: Omit<DistillOptions, 'dbPath'> = {
namespaces: config.namespaces,
batchSize: config.batchSize,
maxEntries: config.maxEntries,
@@ -191,24 +198,34 @@ const runCommand: Command = {
verbose: ctx.flags.verbose === true,
};
- let report: DistillReport;
+ let outcome: Awaited<ReturnType<typeof runDistillationForStores>>;
try {
- report = await runDistillation(distillOptions);
+ outcome = await runDistillationForStores(dbPaths, distillOptions);
} catch (error) {
spinner.fail('Distillation failed');
output.printError(error instanceof Error ? error.message : String(error));
return { success: false, exitCode: 1 };
}
+ const report = outcome.report;
if (report.skipped) {
spinner.succeed('Distillation skipped');
output.writeln(output.warning(`skipped: ${report.skipped}`));
- return { success: true, data: { dbPath, config, report } };
+ return { success: true, data: { dbPath, dbPaths, stores: outcome.stores, config, report } };
}
spinner.succeed(config.dryRun ? 'Dry-run complete' : 'Distillation complete');
output.writeln();
+ for (const store of outcome.stores) {
+ if (store.report.skipped) {
+ output.writeln(output.warning(`${store.dbPath}: ${store.report.skipped}`));
+ }
+ }
+ if (outcome.stores.length > 1) {
+ output.writeln(output.dim(`Stores covered: ${outcome.stores.length}`));
+ }
+
output.printTable({
columns: [
{ key: 'metric', header: 'Metric', width: 24 },
@@ -235,7 +252,7 @@ const runCommand: Command = {
output.printList(provenanceEntries.map(([tier, count]) => `${output.highlight(tier)}: ${count}`));
}
- return { success: true, data: { dbPath, config, report } };
+ return { success: true, data: { dbPath, dbPaths, stores: outcome.stores, config, report } };
},
};
diff --git a/v3/@claude-flow/cli/src/services/memory-distillation.ts b/v3/@claude-flow/cli/src/services/memory-distillation.ts
index 0dac703602..7e482b86d1 100644
--- a/v3/@claude-flow/cli/src/services/memory-distillation.ts
+++ b/v3/@claude-flow/cli/src/services/memory-distillation.ts
@@ -29,6 +29,7 @@ import * as path from 'path';
import { distillTrajectoryContent, serialiseDistilled } from '../memory/structured-distill.js';
import { loadBetterSqlite3 } from '../memory/shared-sqlite.js';
+import { resolveMemoryRoot } from '../memory/memory-root.js';
export type DistillProvenance = 'oracle:test-exec' | 'judge:fable' | 'proxy:structural';
@@ -424,7 +425,102 @@ export async function runDistillation(options: DistillOptions): Promise<DistillR
return report;
}
-/** Where the memory DB lives, for daemon/CLI callers. */
+/**
+ * The two stores the CLI/MCP surface can write:
+ * - memory.db is the sql.js/CLI store (also used with encryption at rest);
+ * - agentdb-memory.db is the native AgentDB store the MCP bridge writes to.
+ *
+ * #3446: distillation used to default to memory.db only. In an MCP-only
+ * project that file often has no AgentDB target tables, so the daemon's
+ * consolidate worker skipped every tick while the rows accumulated in the
+ * sibling store. Keep both paths here (same root resolution as backup) so
+ * callers can cover every store without guessing which surface wrote.
+ */
+export function projectMemoryDbPaths(cwd: string = process.cwd()): string[] {
+ // An explicit project DB pin stays a single-store operation, matching
+ // memory-backup's precedence.
+ const override = process.env.CLAUDE_FLOW_DB_PATH;
+ if (override && override.trim().length > 0) return [path.resolve(cwd, override)];
+ const root = resolveMemoryRoot(cwd);
+ return [path.join(root, 'memory.db'), path.join(root, 'agentdb-memory.db')];
+}
+
+/** Where the primary memory DB lives, for callers that still need one path. */
export function defaultMemoryDbPath(cwd: string = process.cwd()): string {
- return path.join(cwd, '.swarm', 'memory.db');
+ return projectMemoryDbPaths(cwd)[0];
+}
+
+export interface StoreDistillResult {
+ dbPath: string;
+ report: DistillReport;
+}
+
+export interface MultiStoreDistillResult {
+ stores: StoreDistillResult[];
+ report: DistillReport;
+}
+
+/**
+ * Merge per-store reports without hiding a store's outcome. A run over both
+ * stores is only `skipped` when every store skipped; if one store distilled
+ * and the other had no schema/DB, the successful counts remain authoritative
+ * and the per-store skip stays available in `stores`.
+ */
+export function aggregateDistillReports(
+ stores: StoreDistillResult[],
+ dryRun = false,
+): DistillReport {
+ const report = emptyReport(dryRun);
+ const skipped: string[] = [];
+ for (const { report: part } of stores) {
+ report.processed += part.processed;
+ report.episodes += part.episodes;
+ report.episodeEmbeddings += part.episodeEmbeddings;
+ report.patterns += part.patterns;
+ report.patternEmbeddings += part.patternEmbeddings;
+ report.causalEdges += part.causalEdges;
+ report.promoted += part.promoted;
+ report.spendUsd += part.spendUsd;
+ report.namespaces = [...new Set([...report.namespaces, ...part.namespaces])];
+ for (const [tier, count] of Object.entries(part.byProvenance)) {
+ report.byProvenance[tier] = (report.byProvenance[tier] ?? 0) + count;
+ }
+ if (part.corrupt) report.corrupt = true;
+ if (part.skipped) skipped.push(part.skipped);
+ }
+ if (stores.length > 0 && stores.every(store => store.report.skipped)) {
+ report.skipped = [...new Set(skipped)].join('; ');
+ }
+ return report;
+}
+
+/**
+ * Run one distillation pass against each provided store.
+ *
+ * `runner` is injectable so callers/tests can exercise the multi-store
+ * orchestration without a native SQLite build; the default is the real
+ * service. A runner failure is converted to a per-store skip so one bad store
+ * cannot prevent the other from being distilled.
+ */
+export async function runDistillationForStores(
+ dbPaths: string[],
+ options: Omit<DistillOptions, 'dbPath'> = {},
+ runner: (options: DistillOptions) => Promise<DistillReport> = runDistillation,
+): Promise<MultiStoreDistillResult> {
+ const stores: StoreDistillResult[] = [];
+ for (const dbPath of [...new Set(dbPaths.map(p => path.resolve(p)))]) {
+ let report: DistillReport;
+ try {
+ report = await runner({ ...options, dbPath });
+ } catch (error) {
+ report = emptyReport(options.dryRun === true, {
+ skipped: `error: ${error instanceof Error ? error.message : String(error)}`,
+ });
+ }
+ stores.push({ dbPath, report });
+ }
+ if (stores.length === 0) {
+ return { stores, report: emptyReport(options.dryRun === true, { skipped: 'no-db' }) };
+ }
+ return { stores, report: aggregateDistillReports(stores, options.dryRun === true) };
}
diff --git a/v3/@claude-flow/cli/src/services/worker-daemon.ts b/v3/@claude-flow/cli/src/services/worker-daemon.ts
index 75551aa2b0..300f13ef3c 100644
--- a/v3/@claude-flow/cli/src/services/worker-daemon.ts
+++ b/v3/@claude-flow/cli/src/services/worker-daemon.ts
@@ -31,7 +31,12 @@ import {
// service itself is owned/frozen elsewhere (memory-distillation.ts) — it is
// incremental (rowid cursor), non-destructive, transactional, and
// quick_check-gated, so it's safe to call unconditionally on every tick.
-import { runDistillation, defaultMemoryDbPath, type DistillReport } from './memory-distillation.js';
+import {
+ runDistillationForStores,
+ projectMemoryDbPaths,
+ type DistillReport,
+ type StoreDistillResult,
+} from './memory-distillation.js';
import { backupProjectMemoryDbs } from './memory-backup.js';
import { resolveGitWorkspaceIdentity, type GitWorkspaceIdentity } from './git-workspace-identity.js';
import { getWorkspaceLeaseRegistry } from './workspace-lease.js';
@@ -1826,13 +1831,15 @@ export class WorkerDaemon extends EventEmitter {
}
let report: DistillReport;
+ let stores: StoreDistillResult[] = [];
try {
- report = await runDistillation({
- dbPath: defaultMemoryDbPath(this.projectRoot),
+ const outcome = await runDistillationForStores(projectMemoryDbPaths(this.projectRoot), {
maxEntries: CONSOLIDATE_MAX_ENTRIES_PER_TICK,
batchSize: CONSOLIDATE_BATCH_SIZE,
dedupDistance: CONSOLIDATE_DEDUP_DISTANCE,
});
+ report = outcome.report;
+ stores = outcome.stores;
} catch (error) {
// Defensive only — runDistillation() is internally try/catch'd and
// should never reach here. A worker must never crash the daemon.
@@ -1850,6 +1857,11 @@ export class WorkerDaemon extends EventEmitter {
return errorResult;
}
+ for (const store of stores) {
+ if (store.report.skipped) {
+ this.log('info', `Consolidate worker: ${store.dbPath} skipped (${store.report.skipped})`);
+ }
+ }
if (report.corrupt) {
this.log('warn', `Consolidate worker: memory DB reports corruption — ${report.skipped ?? 'skipped'}`);
} else if (report.skipped) {
@@ -1879,6 +1891,12 @@ export class WorkerDaemon extends EventEmitter {
dryRun: report.dryRun,
corrupt: report.corrupt ?? false,
skipped: report.skipped,
+ stores: stores.map(({ dbPath, report: part }) => ({
+ dbPath,
+ processed: part.processed,
+ patterns: part.patterns,
+ skipped: part.skipped,
+ })),
};
writeFileSync(consolidateFile, JSON.stringify(result, null, 2));