π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
feat(memory): ADR-147 entity arm + signal provenance in hybridSearch (#2327)
rUvb099b705f8d0
diff --git a/v3/@claude-flow/memory/src/controller-registry.ts b/v3/@claude-flow/memory/src/controller-registry.tsindex b4044674b6..f1edac23b4 100644--- a/v3/@claude-flow/memory/src/controller-registry.ts+++ b/v3/@claude-flow/memory/src/controller-registry.ts@@ -711,16 +711,20 @@ export class ControllerRegistry extends EventEmitter {}case 'hybridSearch': {- // ADR-125 Phase 5 β real RRF + MMR hybrid search.- // Calls semanticSearch() (which degrades gracefully when embedder is- // unavailable) AND searchKeyword() independently, fuses via RRF, then- // diversifies via MMR (lambda=0.7).+ // ADR-125 Phase 5 + ADR-147 P2 β three-arm RRF + MMR hybrid search.+ // Runs semanticSearch() (dense, with graceful fallback),+ // searchKeyword() (sparse FTS5), and a per-entity keyword search+ // (entity arm, gated on extractEntities(query) returning anything)+ // independently in parallel, fuses via RRF, diversifies via MMR.+ // Results carry a `signals` field naming which arms surfaced each+ // entry (provenance for debugging + downstream rerankers).const memSvc = this.config.memoryService;if (!memSvc) return null;const adapter = typeof memSvc.getAdapter === 'function' ? memSvc.getAdapter() : null;if (!adapter) return null;const { applyRRF, applyMMR } = await import('./smart-retrieval.js');+ const { extractEntities } = await import('./entity-tagger.js');return {/**@@ -738,23 +742,6 @@ export class ControllerRegistry extends EventEmitter {const fanOutK = opts.fanOutK ?? Math.max(limit * 3, 20);const mmrLambda = opts.mmrLambda ?? 0.7;- // Dense arm β may internally fall back to keyword if embedder is- // missing, but that's still a valid signal to fuse.- let dense: any[] = [];- try {- dense = await adapter.semanticSearch(query, fanOutK);- } catch {- dense = [];- }-- // Sparse arm β FTS5 / keyword search.- let sparse: any[] = [];- try {- sparse = await adapter.searchKeyword(query, { k: fanOutK });- } catch {- sparse = [];- }-// Adapt SearchResult[] β SearchCandidate[] expected by RRFconst toCands = (results: any[]) =>results.map((r: any) => ({@@ -769,13 +756,59 @@ export class ControllerRegistry extends EventEmitter {_entry: r.entry,}));- const fused = applyRRF([toCands(dense), toCands(sparse)], 60);+ // Entity arm β only fire if the query actually contains+ // extractable entities. Empty arm is dropped from RRF input+ // so it doesn't dilute the fusion.+ const entities = extractEntities(query);+ const entityFanOut = entities.length > 0+ ? Math.max(1, Math.ceil(fanOutK / entities.length))+ : 0;++ // Run all three arms in parallel. Per-arm try/catch (via the+ // .catch(...) tails) keeps one failing backend from blanking+ // the other arms β the same defensive shape used pre-ADR-147.+ const [dense, sparse, entityHits] = await Promise.all([+ adapter.semanticSearch(query, fanOutK).catch(() => [] as any[]),+ adapter.searchKeyword(query, { k: fanOutK }).catch(() => [] as any[]),+ entities.length > 0+ ? Promise.all(+ entities.map((e: string) =>+ adapter.searchKeyword(e, { k: entityFanOut }).catch(() => [] as any[]),+ ),+ ).then((perEntity: any[][]) => perEntity.flat())+ : Promise.resolve([] as any[]),+ ]);++ const denseCands = toCands(dense);+ const sparseCands = toCands(sparse);+ const entityCands = toCands(entityHits);++ // Build signal-provenance sets keyed by candidate id BEFORE+ // RRF so we can stamp `signals` onto the fused output.+ const candKey = (c: { id?: string; key?: string; content: string }) =>+ c.id || c.key || c.content.slice(0, 128);+ const denseIds = new Set(denseCands.map(candKey));+ const sparseIds = new Set(sparseCands.map(candKey));+ const entityIds = new Set(entityCands.map(candKey));++ const arms = [denseCands, sparseCands];+ if (entityCands.length > 0) arms.push(entityCands);++ const fused = applyRRF(arms, 60);const diverse = applyMMR(fused, mmrLambda, limit);- return diverse.map((s: any) => ({- entry: s.candidate._entry,- score: s.score,- }));+ return diverse.map((s: any) => {+ const key = candKey(s.candidate);+ const signals: ('vector' | 'bm25' | 'entity')[] = [];+ if (denseIds.has(key)) signals.push('vector');+ if (sparseIds.has(key)) signals.push('bm25');+ if (entityIds.has(key)) signals.push('entity');+ return {+ entry: s.candidate._entry,+ score: s.score,+ signals,+ };+ });},source: 'hybrid-rrf-mmr' as const,};diff --git a/v3/@claude-flow/memory/src/entity-tagger.test.ts b/v3/@claude-flow/memory/src/entity-tagger.test.tsnew file mode 100644index 0000000000..f490fca79a--- /dev/null+++ b/v3/@claude-flow/memory/src/entity-tagger.test.ts@@ -0,0 +1,84 @@+/**+ * Tests for the regex entity tagger (ADR-147 P2, ruvnet/ruflo#2317).+ *+ * Goal of the tagger: a conservative third signal for hybridSearch.+ * False negatives are fine (dense + sparse cover the rest); false+ * positives would dilute the RRF score by adding noise rows. These+ * tests pin the conservatism β generic prose returns nothing, and+ * concrete structured tokens (email, URL, path, quote, proper-noun+ * bigram) round-trip cleanly.+ */++import { describe, it, expect } from 'vitest';+import { extractEntities } from './entity-tagger.js';++describe('extractEntities', () => {+ it('returns empty for empty / nullish input', () => {+ expect(extractEntities('')).toEqual([]);+ expect(extractEntities(undefined as unknown as string)).toEqual([]);+ });++ it('returns empty for plain prose with no entities', () => {+ expect(extractEntities('how do i debug this very slow query')).toEqual([]);+ });++ it('extracts email addresses', () => {+ const got = extractEntities('contact [email protected] or [email protected]');+ expect(got).toContain('[email protected]');+ expect(got).toContain('[email protected]');+ });++ it('extracts URLs', () => {+ const got = extractEntities('see https://example.com/path?x=1 and http://internal');+ expect(got).toContain('https://example.com/path?x=1');+ expect(got).toContain('http://internal');+ });++ it('extracts POSIX file paths', () => {+ const got = extractEntities('open src/foo/bar.ts and ./scripts/build.mjs');+ expect(got).toContain('src/foo/bar.ts');+ expect(got).toContain('./scripts/build.mjs');+ });++ it('extracts Windows file paths', () => {+ const got = extractEntities(String.raw`look at C:\Users\ruv\repo\file.ts`);+ expect(got.some((e) => e.includes(String.raw`C:\Users\ruv\repo\file.ts`))).toBe(true);+ });++ it('does not pick up "and/or" style false positives', () => {+ expect(extractEntities('proceed and/or stop')).not.toContain('and/or');+ });++ it('extracts quoted phrases (both quote styles)', () => {+ const got = extractEntities(`open "config alpha" then 'session beta'`);+ expect(got).toContain('config alpha');+ expect(got).toContain('session beta');+ });++ it('extracts proper-noun 2-grams', () => {+ const got = extractEntities('hand off to Alice Smith and notify Acme Corp');+ expect(got).toContain('Alice Smith');+ expect(got).toContain('Acme Corp');+ });++ it('does not extract single capitalized words (too noisy)', () => {+ expect(extractEntities('Reset the database now')).toEqual([]);+ });++ it('deduplicates repeats while preserving first-seen order', () => {+ const got = extractEntities(+ 'Alice Smith filed Alice Smith again at [email protected]. Alice Smith.'+ );+ // 'Alice Smith' should appear exactly once even though the source+ // mentions it three times.+ expect(got.filter((e) => e === 'Alice Smith').length).toBe(1);+ // Both entity types are present.+ expect(got).toContain('Alice Smith');+ expect(got).toContain('[email protected]');+ });++ it('drops trimmed-to-shorter-than-2 fragments', () => {+ // Quoted single-char phrase ("a") should be filtered by the min-length guard.+ expect(extractEntities(`prefer "a" over "b"`)).toEqual([]);+ });+});diff --git a/v3/@claude-flow/memory/src/entity-tagger.ts b/v3/@claude-flow/memory/src/entity-tagger.tsnew file mode 100644index 0000000000..4aaa5e093f--- /dev/null+++ b/v3/@claude-flow/memory/src/entity-tagger.ts@@ -0,0 +1,80 @@+/**+ * Entity tagger β regex-based proper-noun / structured-token extractor.+ *+ * Used by the `hybridSearch` controller as a third RRF arm alongside the+ * dense (vector) and sparse (BM25/FTS5) signals (ADR-147, ruvnet/ruflo#2317).+ *+ * Entity matching is a distinct signal because BM25 weights documents by+ * overall token frequency; a per-entity exact match avoids downweighting+ * for tokens that happen to be common but mention an entity by name.+ * Example: querying "Alice OAuth tokens" β BM25 may rank a doc about+ * generic OAuth above one mentioning Alice specifically; entity arm+ * surfaces the Alice doc independently.+ *+ * P1 is regex-only β no NLP model dependency. Tags:+ * - Emails: [email protected]+ * - URLs: http(s)://...+ * - File paths: ./foo/bar.ts, C:\foo\bar.ts, src/foo+ * - Quoted phrases: "..." or '...'+ * - Proper-noun 2-grams: "Alice Smith", "Acme Corp"+ *+ * Deliberately conservative β false negatives are fine (the dense + sparse+ * arms still cover the query); false positives would dilute the RRF score+ * by adding noise rows. A future P2 can swap this for a CRF/spaCy tagger+ * behind the same `extractEntities(text)` contract.+ */++/**+ * Extract entity-like tokens from free text. Returns a unique list,+ * trimmed and deduplicated, in extraction order.+ */+export function extractEntities(text: string): string[] {+ if (!text) return [];++ const seen = new Set<string>();+ const ordered: string[] = [];++ const add = (raw: string | undefined): void => {+ if (!raw) return;+ const trimmed = raw.trim();+ if (trimmed.length < 2) return;+ if (seen.has(trimmed)) return;+ seen.add(trimmed);+ ordered.push(trimmed);+ };++ // Emails β anchored on @ so we don't pick up arbitrary handles.+ for (const m of text.matchAll(/[\w.+-]+@[\w-]+(?:\.[\w-]+)+/g)) add(m[0]);++ // URLs β http(s) only. Stop at whitespace and common boundary chars.+ for (const m of text.matchAll(/https?:\/\/[^\s)>"'`]+/g)) add(m[0]);++ // File paths β two complementary patterns:+ //+ // (a) Path with a file extension. Catches `src/foo/bar.ts`, the most+ // common case. The extension (`.\w{1,5}`) is what separates a+ // real path from prose like `and/or`.+ //+ // (b) Path with an explicit leading sentinel: `./...`, `../...`, `/...`,+ // or `C:\...`. Catches extensionless paths the user clearly meant+ // (like `./scripts/build` or `/etc/hosts`).+ for (const m of text.matchAll(+ /(?:[A-Za-z]:[\\/])?(?:[\w.-]+[\\/])+[\w-]+\.\w{1,5}\b/g,+ )) {+ add(m[0]);+ }+ for (const m of text.matchAll(/(?:\.{1,2}|[A-Za-z]:)[\\/][\w./\\-]+/g)) add(m[0]);++ // Quoted phrases. Inner content only, both quote styles. The+ // (?<!\w)/(?!\w) guards keep us from pairing a *closing* quote of one+ // phrase with the *opening* quote of the next β e.g. `"a" over "b"`+ // should NOT capture ` over ` from positions 9β16.+ for (const m of text.matchAll(/(?<!\w)"([^"]{2,})"(?!\w)/g)) add(m[1]);+ for (const m of text.matchAll(/(?<!\w)'([^']{2,})'(?!\w)/g)) add(m[1]);++ // Proper-noun 2+ grams β capitalized words with capitalized neighbours.+ // Skip single capitals (too noisy: sentence-initial common nouns).+ for (const m of text.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+\b/g)) add(m[0]);++ return ordered;+}diff --git a/v3/@claude-flow/memory/src/graceful-retrieval.test.ts b/v3/@claude-flow/memory/src/graceful-retrieval.test.tsindex 2912b61e59..37062cdd42 100644--- a/v3/@claude-flow/memory/src/graceful-retrieval.test.ts+++ b/v3/@claude-flow/memory/src/graceful-retrieval.test.ts@@ -153,6 +153,61 @@ describe('Phase 5 β hybridSearch controller (RRF + MMR)', () => {// Hybrid should return up to `limit` entries (10 here, given 50 stored).expect(results.length).toBeLessThanOrEqual(10);+ // ADR-147: every result carries provenance β at least one signal name.+ for (const r of results) {+ expect(Array.isArray(r.signals)).toBe(true);+ expect(r.signals.length).toBeGreaterThan(0);+ for (const s of r.signals) {+ expect(['vector', 'bm25', 'entity']).toContain(s);+ }+ }++ await registry.shutdown();+ await svc.close();+ });++ it('ADR-147 β entity arm surfaces proper-noun matches above vector noise', async () => {+ // Store 30 generic "authentication" entries (no Alice) + a single+ // entry mentioning "Alice Smith" β the proper-noun bigram extractEntities+ // should pick up. The entity arm should boost the Alice entry above+ // the 30 generic ones for a query that names Alice explicitly.+ const dim = 8;+ const svc = new MemoryService({+ dimensions: dim,+ persistenceEnabled: false,+ snapshotInterval: 0,+ embeddingGenerator: async (text: string) => randomVec(dim, hashStr(text)),+ });+ await svc.initialize();++ for (let i = 0; i < 30; i++) {+ const entry = createDefaultEntry({+ key: `generic-${i}`,+ content: `authentication and authorization patterns ${i}`,+ });+ entry.embedding = randomVec(dim, hashStr(entry.content));+ await svc.store(entry);+ }+ const aliceEntry = createDefaultEntry({+ key: 'alice-needle',+ content: 'Alice Smith authored these authentication flows',+ });+ aliceEntry.embedding = randomVec(dim, hashStr(aliceEntry.content));+ await svc.store(aliceEntry);++ const registry = new ControllerRegistry();+ await registry.initialize({ memoryService: svc });++ const hybrid = registry.get<any>('hybridSearch');+ const results = await hybrid.search('Alice Smith authentication', { limit: 10 });++ // The Alice entry should appear in the top-10 fused results.+ const alice = results.find((r: any) => r.entry.key === 'alice-needle');+ expect(alice).toBeTruthy();+ // β¦and should carry the 'entity' provenance signal (it's the only entry+ // that matches the "Alice Smith" proper-noun bigram).+ expect(alice.signals).toContain('entity');+await registry.shutdown();await svc.close();});