🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
fix(memory): distill both project memory stores (#4338)
ordinary beingfc4b364803b7
diff --git a/v3/@claude-flow/cli/__tests__/distill-both-stores-3446.test.ts b/v3/@claude-flow/cli/__tests__/distill-both-stores-3446.test.tsnew file mode 100644index 0000000000..012bbd4d43--- /dev/null+++ b/v3/@claude-flow/cli/__tests__/distill-both-stores-3446.test.ts@@ -0,0 +1,218 @@+/**+ * #3446: distillation must cover the store the MCP bridge actually writes.+ *+ * `memory.db` and `agentdb-memory.db` are separate stores. Before this fix,+ * `memory distill run` and the daemon's consolidate worker only opened+ * memory.db. In an MCP-only project that file often has no AgentDB target+ * tables, so every tick skipped while the rows accumulated in the sibling.+ *+ * The pure tests exercise the multi-store orchestration without requiring a+ * native SQLite build; the CLI test runs the real service when better-sqlite3+ * is available.+ */+import { describe, it, expect, afterEach } from 'vitest';+import { mkdtempSync, mkdirSync, rmSync } from 'node:fs';+import { join } from 'node:path';+import { tmpdir } from 'node:os';++import type { Command, CommandContext } from '../src/types.js';+import {+ aggregateDistillReports,+ projectMemoryDbPaths,+ runDistillationForStores,+ type DistillReport,+} from '../src/services/memory-distillation.js';+import { distillCommand } from '../src/commands/memory-distill.js';++let Database: any;+let haveNative = false;+try {+ Database = (await import('better-sqlite3')).default;+ // The package can resolve while its native binary is missing; probe the+ // constructor so the native-only cases skip cleanly on a source-only host.+ const probe = new Database(':memory:');+ probe.close();+ haveNative = true;+} catch { haveNative = false; }++function report(extra: Partial<DistillReport> = {}): DistillReport {+ return {+ processed: 0, episodes: 0, episodeEmbeddings: 0, patterns: 0,+ patternEmbeddings: 0, causalEdges: 0, promoted: 0, byProvenance: {},+ namespaces: [], dryRun: false, spendUsd: 0, ...extra,+ };+}++function makeCtx(flags: Record<string, unknown>, cwd: string): CommandContext {+ return { args: [], flags: { _: [], ...flags }, cwd, interactive: false } as CommandContext;+}++function sub(name: string): Command {+ const cmd = distillCommand.subcommands?.find((c) => c.name === name);+ if (!cmd?.action) throw new Error(`memory distill ${name} has no action`);+ return cmd;+}++/** Source rows plus the AgentDB target tables. */+function seedAgentStore(dbPath: string): void {+ const db = new Database(dbPath);+ db.exec(`CREATE TABLE memory_entries (+ id TEXT PRIMARY KEY, key TEXT, namespace TEXT DEFAULT 'default',+ content TEXT, embedding TEXT, status TEXT DEFAULT 'active'+ )`);+ db.exec(`CREATE TABLE episodes (+ id INTEGER PRIMARY KEY AUTOINCREMENT, ts INTEGER DEFAULT 0, session_id TEXT NOT NULL,+ task TEXT NOT NULL, input TEXT, output TEXT, critique TEXT, reward REAL DEFAULT 0,+ success BOOLEAN DEFAULT 0, latency_ms INTEGER, tokens_used INTEGER, tags TEXT,+ metadata JSON, created_at INTEGER DEFAULT 0+ )`);+ db.exec(`CREATE TABLE reasoning_patterns (+ id INTEGER PRIMARY KEY AUTOINCREMENT, ts INTEGER DEFAULT 0, task_type TEXT NOT NULL,+ approach TEXT NOT NULL, success_rate REAL NOT NULL DEFAULT 0, uses INTEGER DEFAULT 0,+ avg_reward REAL DEFAULT 0, tags TEXT, metadata TEXT+ )`);+ db.exec(`CREATE TABLE pattern_embeddings (+ pattern_id INTEGER PRIMARY KEY, embedding BLOB NOT NULL+ )`);+ db.exec(`CREATE TABLE causal_edges (+ id INTEGER PRIMARY KEY AUTOINCREMENT, from_memory_id INTEGER NOT NULL, from_memory_type TEXT NOT NULL,+ to_memory_id INTEGER NOT NULL, to_memory_type TEXT NOT NULL, similarity REAL DEFAULT 0,+ confidence REAL DEFAULT 0.5, mechanism TEXT, metadata JSON, created_at INTEGER DEFAULT 0+ )`);+ const ins = db.prepare('INSERT INTO memory_entries (id, key, namespace, content, embedding) VALUES (?,?,?,?,?)');+ const vec = (seed: number) => JSON.stringify(Array.from({ length: 8 }, (_, i) => Math.sin(seed + i)));+ ins.run('a1', 'k1', 'commands', 'refactor the auth module in src/auth.ts', vec(1));+ ins.run('a2', 'k2', 'feedback', JSON.stringify({ taskId: 'edit-a', success: true }), vec(2));+ db.close();+}++/** The CLI-only store: source rows but no AgentDB target tables. */+function seedCliStore(dbPath: string): void {+ const db = new Database(dbPath);+ db.exec(`CREATE TABLE memory_entries (+ id TEXT PRIMARY KEY, key TEXT, namespace TEXT DEFAULT 'default',+ content TEXT, embedding TEXT, status TEXT DEFAULT 'active'+ )`);+ db.prepare('INSERT INTO memory_entries (id, key, namespace, content, embedding) VALUES (?,?,?,?,?)')+ .run('c1', 'k3', 'commands', 'unrelated CLI-only row', JSON.stringify([1, 2, 3]));+ db.close();+}++function patternCount(dbPath: string): number {+ const db = new Database(dbPath, { readonly: true });+ const c = (db.prepare('SELECT COUNT(*) AS c FROM reasoning_patterns').get() as { c: number }).c;+ db.close();+ return c;+}++const tempDirs: string[] = [];+function tempDir(): string {+ const dir = mkdtempSync(join(tmpdir(), 'distill-3446-'));+ tempDirs.push(dir);+ return dir;+}++afterEach(() => {+ while (tempDirs.length) rmSync(tempDirs.pop()!, { recursive: true, force: true });+});++describe('#3446 multi-store distillation orchestration', () => {+ it('resolves both stores under the same memory root', () => {+ const dir = tempDir();+ const prev = process.env.CLAUDE_FLOW_MEMORY_PATH;+ try {+ process.env.CLAUDE_FLOW_MEMORY_PATH = join(dir, 'state');+ expect(projectMemoryDbPaths(dir)).toEqual([+ join(dir, 'state', 'memory.db'),+ join(dir, 'state', 'agentdb-memory.db'),+ ]);+ } finally {+ if (prev === undefined) delete process.env.CLAUDE_FLOW_MEMORY_PATH;+ else process.env.CLAUDE_FLOW_MEMORY_PATH = prev;+ }+ });++ it('honors an explicit CLAUDE_FLOW_DB_PATH as a single-store pin', () => {+ const dir = tempDir();+ const prev = process.env.CLAUDE_FLOW_DB_PATH;+ try {+ process.env.CLAUDE_FLOW_DB_PATH = join('.custom', 'pinned.db');+ expect(projectMemoryDbPaths(dir)).toEqual([join(dir, '.custom', 'pinned.db')]);+ } finally {+ if (prev === undefined) delete process.env.CLAUDE_FLOW_DB_PATH;+ else process.env.CLAUDE_FLOW_DB_PATH = prev;+ }+ });++ it('aggregates per-store counts and only reports skipped when every store skipped', () => {+ const stores = [+ { dbPath: 'memory.db', report: report({ processed: 2, patterns: 1, namespaces: ['commands'] }) },+ { dbPath: 'agentdb-memory.db', report: report({ processed: 3, patterns: 2, skipped: 'target table episodes missing' }) },+ ];+ const merged = aggregateDistillReports(stores);+ expect(merged.processed).toBe(5);+ expect(merged.patterns).toBe(3);+ expect(merged.namespaces).toEqual(['commands']);+ expect(merged.skipped).toBeUndefined();++ const allSkipped = aggregateDistillReports([+ { dbPath: 'a', report: report({ skipped: 'no-db' }) },+ { dbPath: 'b', report: report({ skipped: 'no memory_entries' }) },+ ]);+ expect(allSkipped.skipped).toContain('no-db');+ expect(allSkipped.skipped).toContain('no memory_entries');+ });++ it('invokes the runner once per resolved store and preserves partial failure', async () => {+ const seen: string[] = [];+ const result = await runDistillationForStores(+ ['/tmp/a.db', '/tmp/b.db', '/tmp/a.db'],+ { dryRun: true },+ async ({ dbPath }) => {+ seen.push(dbPath);+ if (dbPath.endsWith('b.db')) throw new Error('boom');+ return report({ processed: 1, dryRun: true });+ },+ );++ expect(seen).toHaveLength(2);+ expect(result.stores).toHaveLength(2);+ expect(result.report.processed).toBe(1);+ expect(result.report.skipped).toBeUndefined();+ expect(result.stores[1].report.skipped).toContain('boom');+ });+});++describe.skipIf(!haveNative)('#3446 CLI covers both stores without --db', () => {+ it('distills the AgentDB sibling and reports the CLI store skip', async () => {+ const dir = tempDir();+ mkdirSync(join(dir, '.swarm'), { recursive: true });+ const cliStore = join(dir, '.swarm', 'memory.db');+ const agentStore = join(dir, '.swarm', 'agentdb-memory.db');+ seedCliStore(cliStore);+ seedAgentStore(agentStore);++ const result = await sub('run').action!(makeCtx({}, dir));+ expect(result?.success).toBe(true);+ const data = result?.data as any;++ expect(data.dbPaths).toEqual([cliStore, agentStore]);+ expect(data.stores).toHaveLength(2);+ expect(data.report.patterns).toBeGreaterThan(0);+ expect(patternCount(agentStore)).toBeGreaterThan(0);+ expect(data.stores.find((s: any) => s.dbPath === cliStore)?.report.skipped)+ .toContain('target table');+ });++ it('an explicit --db remains a single-store operation', async () => {+ const dir = tempDir();+ const dbPath = join(dir, 'only.db');+ seedAgentStore(dbPath);++ const result = await sub('run').action!(makeCtx({ db: dbPath }, dir));+ const data = result?.data as any;+ expect(data.dbPaths).toEqual([dbPath]);+ expect(data.stores).toHaveLength(1);+ expect(patternCount(dbPath)).toBeGreaterThan(0);+ });+});diff --git a/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.ts b/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.tsindex 4086260793..0299a09573 100644--- a/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.ts+++ b/v3/@claude-flow/cli/__tests__/services/worker-daemon-consolidate.test.ts@@ -117,6 +117,24 @@ describe.skipIf(!haveNative)('WorkerDaemon consolidate worker — ADR-174 M3 wirexpect((result as any).patternsConsolidated).toBe(metrics.patternsConsolidated);});+ it('distills agentdb-memory.db when memory.db is absent (#3446)', async () => {+ const agentDbPath = join(tempDir, '.swarm', 'agentdb-memory.db');+ seedDb(agentDbPath);+ const daemon = new WorkerDaemon(tempDir);+ const result = await (daemon as any).runConsolidateWorker() as any;++ expect(reasoningPatternCount(agentDbPath)).toBeGreaterThan(0);+ expect(result.patternsConsolidated).toBeGreaterThan(0);+ expect(result.stores).toEqual(expect.arrayContaining([+ expect.objectContaining({ dbPath: agentDbPath }),+ ]));++ const metrics = JSON.parse(readFileSync(metricsFile, 'utf-8'));+ expect(metrics.stores).toEqual(expect.arrayContaining([+ expect.objectContaining({ dbPath: agentDbPath }),+ ]));+ });+it('is idempotent — a second tick processes nothing new (cursor-driven)', async () => {seedDb(dbPath);const daemon = new WorkerDaemon(tempDir);diff --git a/v3/@claude-flow/cli/src/commands/memory-distill.ts b/v3/@claude-flow/cli/src/commands/memory-distill.tsindex 49430ac78c..0a95fb1f33 100644--- a/v3/@claude-flow/cli/src/commands/memory-distill.ts+++ b/v3/@claude-flow/cli/src/commands/memory-distill.ts@@ -30,10 +30,10 @@ import * as path from 'path';import type { Command, CommandContext, CommandResult } from '../types.js';import { output } from '../output.js';import {- runDistillation,+ runDistillationForStores,+ projectMemoryDbPaths,defaultMemoryDbPath,type DistillOptions,- type DistillReport,} from '../services/memory-distillation.js';// ============================================================================@@ -169,7 +169,15 @@ const runCommand: Command = {return { success: false, exitCode: 1 };}- const dbPath = resolveDbPath(ctx);+ // An explicit --db remains a single-store operation. Without it, cover+ // both stores the CLI/MCP surfaces can write (#3446): memory.db and the+ // native AgentDB sibling. An MCP-only project often has only the latter,+ // which is why the worker used to skip with "target table ... missing".+ const explicitDb = typeof ctx.flags.db === 'string' && ctx.flags.db.trim()+ ? path.resolve(ctx.flags.db)+ : undefined;+ const dbPaths = explicitDb ? [explicitDb] : projectMemoryDbPaths(ctx.cwd || process.cwd());+ const dbPath = explicitDb ?? dbPaths[0];output.writeln();output.writeln(output.bold('Memory Distillation (ADR-174)'));@@ -179,8 +187,7 @@ const runCommand: Command = {const spinner = output.createSpinner({ text: 'Distilling memory_entries...', spinner: 'dots' });spinner.start();- const distillOptions: DistillOptions = {- dbPath,+ const distillOptions: Omit<DistillOptions, 'dbPath'> = {namespaces: config.namespaces,batchSize: config.batchSize,maxEntries: config.maxEntries,@@ -191,24 +198,34 @@ const runCommand: Command = {verbose: ctx.flags.verbose === true,};- let report: DistillReport;+ let outcome: Awaited<ReturnType<typeof runDistillationForStores>>;try {- report = await runDistillation(distillOptions);+ outcome = await runDistillationForStores(dbPaths, distillOptions);} catch (error) {spinner.fail('Distillation failed');output.printError(error instanceof Error ? error.message : String(error));return { success: false, exitCode: 1 };}+ const report = outcome.report;if (report.skipped) {spinner.succeed('Distillation skipped');output.writeln(output.warning(`skipped: ${report.skipped}`));- return { success: true, data: { dbPath, config, report } };+ return { success: true, data: { dbPath, dbPaths, stores: outcome.stores, config, report } };}spinner.succeed(config.dryRun ? 'Dry-run complete' : 'Distillation complete');output.writeln();+ for (const store of outcome.stores) {+ if (store.report.skipped) {+ output.writeln(output.warning(`${store.dbPath}: ${store.report.skipped}`));+ }+ }+ if (outcome.stores.length > 1) {+ output.writeln(output.dim(`Stores covered: ${outcome.stores.length}`));+ }+output.printTable({columns: [{ key: 'metric', header: 'Metric', width: 24 },@@ -235,7 +252,7 @@ const runCommand: Command = {output.printList(provenanceEntries.map(([tier, count]) => `${output.highlight(tier)}: ${count}`));}- return { success: true, data: { dbPath, config, report } };+ return { success: true, data: { dbPath, dbPaths, stores: outcome.stores, config, report } };},};diff --git a/v3/@claude-flow/cli/src/services/memory-distillation.ts b/v3/@claude-flow/cli/src/services/memory-distillation.tsindex 0dac703602..7e482b86d1 100644--- a/v3/@claude-flow/cli/src/services/memory-distillation.ts+++ b/v3/@claude-flow/cli/src/services/memory-distillation.ts@@ -29,6 +29,7 @@ import * as path from 'path';import { distillTrajectoryContent, serialiseDistilled } from '../memory/structured-distill.js';import { loadBetterSqlite3 } from '../memory/shared-sqlite.js';+import { resolveMemoryRoot } from '../memory/memory-root.js';export type DistillProvenance = 'oracle:test-exec' | 'judge:fable' | 'proxy:structural';@@ -424,7 +425,102 @@ export async function runDistillation(options: DistillOptions): Promise<DistillRreturn report;}-/** Where the memory DB lives, for daemon/CLI callers. */+/**+ * The two stores the CLI/MCP surface can write:+ * - memory.db is the sql.js/CLI store (also used with encryption at rest);+ * - agentdb-memory.db is the native AgentDB store the MCP bridge writes to.+ *+ * #3446: distillation used to default to memory.db only. In an MCP-only+ * project that file often has no AgentDB target tables, so the daemon's+ * consolidate worker skipped every tick while the rows accumulated in the+ * sibling store. Keep both paths here (same root resolution as backup) so+ * callers can cover every store without guessing which surface wrote.+ */+export function projectMemoryDbPaths(cwd: string = process.cwd()): string[] {+ // An explicit project DB pin stays a single-store operation, matching+ // memory-backup's precedence.+ const override = process.env.CLAUDE_FLOW_DB_PATH;+ if (override && override.trim().length > 0) return [path.resolve(cwd, override)];+ const root = resolveMemoryRoot(cwd);+ return [path.join(root, 'memory.db'), path.join(root, 'agentdb-memory.db')];+}++/** Where the primary memory DB lives, for callers that still need one path. */export function defaultMemoryDbPath(cwd: string = process.cwd()): string {- return path.join(cwd, '.swarm', 'memory.db');+ return projectMemoryDbPaths(cwd)[0];+}++export interface StoreDistillResult {+ dbPath: string;+ report: DistillReport;+}++export interface MultiStoreDistillResult {+ stores: StoreDistillResult[];+ report: DistillReport;+}++/**+ * Merge per-store reports without hiding a store's outcome. A run over both+ * stores is only `skipped` when every store skipped; if one store distilled+ * and the other had no schema/DB, the successful counts remain authoritative+ * and the per-store skip stays available in `stores`.+ */+export function aggregateDistillReports(+ stores: StoreDistillResult[],+ dryRun = false,+): DistillReport {+ const report = emptyReport(dryRun);+ const skipped: string[] = [];+ for (const { report: part } of stores) {+ report.processed += part.processed;+ report.episodes += part.episodes;+ report.episodeEmbeddings += part.episodeEmbeddings;+ report.patterns += part.patterns;+ report.patternEmbeddings += part.patternEmbeddings;+ report.causalEdges += part.causalEdges;+ report.promoted += part.promoted;+ report.spendUsd += part.spendUsd;+ report.namespaces = [...new Set([...report.namespaces, ...part.namespaces])];+ for (const [tier, count] of Object.entries(part.byProvenance)) {+ report.byProvenance[tier] = (report.byProvenance[tier] ?? 0) + count;+ }+ if (part.corrupt) report.corrupt = true;+ if (part.skipped) skipped.push(part.skipped);+ }+ if (stores.length > 0 && stores.every(store => store.report.skipped)) {+ report.skipped = [...new Set(skipped)].join('; ');+ }+ return report;+}++/**+ * Run one distillation pass against each provided store.+ *+ * `runner` is injectable so callers/tests can exercise the multi-store+ * orchestration without a native SQLite build; the default is the real+ * service. A runner failure is converted to a per-store skip so one bad store+ * cannot prevent the other from being distilled.+ */+export async function runDistillationForStores(+ dbPaths: string[],+ options: Omit<DistillOptions, 'dbPath'> = {},+ runner: (options: DistillOptions) => Promise<DistillReport> = runDistillation,+): Promise<MultiStoreDistillResult> {+ const stores: StoreDistillResult[] = [];+ for (const dbPath of [...new Set(dbPaths.map(p => path.resolve(p)))]) {+ let report: DistillReport;+ try {+ report = await runner({ ...options, dbPath });+ } catch (error) {+ report = emptyReport(options.dryRun === true, {+ skipped: `error: ${error instanceof Error ? error.message : String(error)}`,+ });+ }+ stores.push({ dbPath, report });+ }+ if (stores.length === 0) {+ return { stores, report: emptyReport(options.dryRun === true, { skipped: 'no-db' }) };+ }+ return { stores, report: aggregateDistillReports(stores, options.dryRun === true) };}diff --git a/v3/@claude-flow/cli/src/services/worker-daemon.ts b/v3/@claude-flow/cli/src/services/worker-daemon.tsindex 75551aa2b0..300f13ef3c 100644--- a/v3/@claude-flow/cli/src/services/worker-daemon.ts+++ b/v3/@claude-flow/cli/src/services/worker-daemon.ts@@ -31,7 +31,12 @@ import {// service itself is owned/frozen elsewhere (memory-distillation.ts) — it is// incremental (rowid cursor), non-destructive, transactional, and// quick_check-gated, so it's safe to call unconditionally on every tick.-import { runDistillation, defaultMemoryDbPath, type DistillReport } from './memory-distillation.js';+import {+ runDistillationForStores,+ projectMemoryDbPaths,+ type DistillReport,+ type StoreDistillResult,+} from './memory-distillation.js';import { backupProjectMemoryDbs } from './memory-backup.js';import { resolveGitWorkspaceIdentity, type GitWorkspaceIdentity } from './git-workspace-identity.js';import { getWorkspaceLeaseRegistry } from './workspace-lease.js';@@ -1826,13 +1831,15 @@ export class WorkerDaemon extends EventEmitter {}let report: DistillReport;+ let stores: StoreDistillResult[] = [];try {- report = await runDistillation({- dbPath: defaultMemoryDbPath(this.projectRoot),+ const outcome = await runDistillationForStores(projectMemoryDbPaths(this.projectRoot), {maxEntries: CONSOLIDATE_MAX_ENTRIES_PER_TICK,batchSize: CONSOLIDATE_BATCH_SIZE,dedupDistance: CONSOLIDATE_DEDUP_DISTANCE,});+ report = outcome.report;+ stores = outcome.stores;} catch (error) {// Defensive only — runDistillation() is internally try/catch'd and// should never reach here. A worker must never crash the daemon.@@ -1850,6 +1857,11 @@ export class WorkerDaemon extends EventEmitter {return errorResult;}+ for (const store of stores) {+ if (store.report.skipped) {+ this.log('info', `Consolidate worker: ${store.dbPath} skipped (${store.report.skipped})`);+ }+ }if (report.corrupt) {this.log('warn', `Consolidate worker: memory DB reports corruption — ${report.skipped ?? 'skipped'}`);} else if (report.skipped) {@@ -1879,6 +1891,12 @@ export class WorkerDaemon extends EventEmitter {dryRun: report.dryRun,corrupt: report.corrupt ?? false,skipped: report.skipped,+ stores: stores.map(({ dbPath, report: part }) => ({+ dbPath,+ processed: part.processed,+ patterns: part.patterns,+ skipped: part.skipped,+ })),};writeFileSync(consolidateFile, JSON.stringify(result, null, 2));