Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

feat(memory): ADR-147 entity arm + signal provenance in hybridSearch (#2327)

rUvb099b705f8d0
diff --git a/v3/@claude-flow/memory/src/controller-registry.ts b/v3/@claude-flow/memory/src/controller-registry.ts
index b4044674b6..f1edac23b4 100644
--- a/v3/@claude-flow/memory/src/controller-registry.ts
+++ b/v3/@claude-flow/memory/src/controller-registry.ts
@@ -711,16 +711,20 @@ export class ControllerRegistry extends EventEmitter {
}
case 'hybridSearch': {
- // ADR-125 Phase 5 β€” real RRF + MMR hybrid search.
- // Calls semanticSearch() (which degrades gracefully when embedder is
- // unavailable) AND searchKeyword() independently, fuses via RRF, then
- // diversifies via MMR (lambda=0.7).
+ // ADR-125 Phase 5 + ADR-147 P2 β€” three-arm RRF + MMR hybrid search.
+ // Runs semanticSearch() (dense, with graceful fallback),
+ // searchKeyword() (sparse FTS5), and a per-entity keyword search
+ // (entity arm, gated on extractEntities(query) returning anything)
+ // independently in parallel, fuses via RRF, diversifies via MMR.
+ // Results carry a `signals` field naming which arms surfaced each
+ // entry (provenance for debugging + downstream rerankers).
const memSvc = this.config.memoryService;
if (!memSvc) return null;
const adapter = typeof memSvc.getAdapter === 'function' ? memSvc.getAdapter() : null;
if (!adapter) return null;
const { applyRRF, applyMMR } = await import('./smart-retrieval.js');
+ const { extractEntities } = await import('./entity-tagger.js');
return {
/**
@@ -738,23 +742,6 @@ export class ControllerRegistry extends EventEmitter {
const fanOutK = opts.fanOutK ?? Math.max(limit * 3, 20);
const mmrLambda = opts.mmrLambda ?? 0.7;
- // Dense arm β€” may internally fall back to keyword if embedder is
- // missing, but that's still a valid signal to fuse.
- let dense: any[] = [];
- try {
- dense = await adapter.semanticSearch(query, fanOutK);
- } catch {
- dense = [];
- }
-
- // Sparse arm β€” FTS5 / keyword search.
- let sparse: any[] = [];
- try {
- sparse = await adapter.searchKeyword(query, { k: fanOutK });
- } catch {
- sparse = [];
- }
-
// Adapt SearchResult[] β†’ SearchCandidate[] expected by RRF
const toCands = (results: any[]) =>
results.map((r: any) => ({
@@ -769,13 +756,59 @@ export class ControllerRegistry extends EventEmitter {
_entry: r.entry,
}));
- const fused = applyRRF([toCands(dense), toCands(sparse)], 60);
+ // Entity arm β€” only fire if the query actually contains
+ // extractable entities. Empty arm is dropped from RRF input
+ // so it doesn't dilute the fusion.
+ const entities = extractEntities(query);
+ const entityFanOut = entities.length > 0
+ ? Math.max(1, Math.ceil(fanOutK / entities.length))
+ : 0;
+
+ // Run all three arms in parallel. Per-arm try/catch (via the
+ // .catch(...) tails) keeps one failing backend from blanking
+ // the other arms β€” the same defensive shape used pre-ADR-147.
+ const [dense, sparse, entityHits] = await Promise.all([
+ adapter.semanticSearch(query, fanOutK).catch(() => [] as any[]),
+ adapter.searchKeyword(query, { k: fanOutK }).catch(() => [] as any[]),
+ entities.length > 0
+ ? Promise.all(
+ entities.map((e: string) =>
+ adapter.searchKeyword(e, { k: entityFanOut }).catch(() => [] as any[]),
+ ),
+ ).then((perEntity: any[][]) => perEntity.flat())
+ : Promise.resolve([] as any[]),
+ ]);
+
+ const denseCands = toCands(dense);
+ const sparseCands = toCands(sparse);
+ const entityCands = toCands(entityHits);
+
+ // Build signal-provenance sets keyed by candidate id BEFORE
+ // RRF so we can stamp `signals` onto the fused output.
+ const candKey = (c: { id?: string; key?: string; content: string }) =>
+ c.id || c.key || c.content.slice(0, 128);
+ const denseIds = new Set(denseCands.map(candKey));
+ const sparseIds = new Set(sparseCands.map(candKey));
+ const entityIds = new Set(entityCands.map(candKey));
+
+ const arms = [denseCands, sparseCands];
+ if (entityCands.length > 0) arms.push(entityCands);
+
+ const fused = applyRRF(arms, 60);
const diverse = applyMMR(fused, mmrLambda, limit);
- return diverse.map((s: any) => ({
- entry: s.candidate._entry,
- score: s.score,
- }));
+ return diverse.map((s: any) => {
+ const key = candKey(s.candidate);
+ const signals: ('vector' | 'bm25' | 'entity')[] = [];
+ if (denseIds.has(key)) signals.push('vector');
+ if (sparseIds.has(key)) signals.push('bm25');
+ if (entityIds.has(key)) signals.push('entity');
+ return {
+ entry: s.candidate._entry,
+ score: s.score,
+ signals,
+ };
+ });
},
source: 'hybrid-rrf-mmr' as const,
};
diff --git a/v3/@claude-flow/memory/src/entity-tagger.test.ts b/v3/@claude-flow/memory/src/entity-tagger.test.ts
new file mode 100644
index 0000000000..f490fca79a
--- /dev/null
+++ b/v3/@claude-flow/memory/src/entity-tagger.test.ts
@@ -0,0 +1,84 @@
+/**
+ * Tests for the regex entity tagger (ADR-147 P2, ruvnet/ruflo#2317).
+ *
+ * Goal of the tagger: a conservative third signal for hybridSearch.
+ * False negatives are fine (dense + sparse cover the rest); false
+ * positives would dilute the RRF score by adding noise rows. These
+ * tests pin the conservatism β€” generic prose returns nothing, and
+ * concrete structured tokens (email, URL, path, quote, proper-noun
+ * bigram) round-trip cleanly.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { extractEntities } from './entity-tagger.js';
+
+describe('extractEntities', () => {
+ it('returns empty for empty / nullish input', () => {
+ expect(extractEntities('')).toEqual([]);
+ expect(extractEntities(undefined as unknown as string)).toEqual([]);
+ });
+
+ it('returns empty for plain prose with no entities', () => {
+ expect(extractEntities('how do i debug this very slow query')).toEqual([]);
+ });
+
+ it('extracts email addresses', () => {
+ const got = extractEntities('contact [email protected] or [email protected]');
+ expect(got).toContain('[email protected]');
+ expect(got).toContain('[email protected]');
+ });
+
+ it('extracts URLs', () => {
+ const got = extractEntities('see https://example.com/path?x=1 and http://internal');
+ expect(got).toContain('https://example.com/path?x=1');
+ expect(got).toContain('http://internal');
+ });
+
+ it('extracts POSIX file paths', () => {
+ const got = extractEntities('open src/foo/bar.ts and ./scripts/build.mjs');
+ expect(got).toContain('src/foo/bar.ts');
+ expect(got).toContain('./scripts/build.mjs');
+ });
+
+ it('extracts Windows file paths', () => {
+ const got = extractEntities(String.raw`look at C:\Users\ruv\repo\file.ts`);
+ expect(got.some((e) => e.includes(String.raw`C:\Users\ruv\repo\file.ts`))).toBe(true);
+ });
+
+ it('does not pick up "and/or" style false positives', () => {
+ expect(extractEntities('proceed and/or stop')).not.toContain('and/or');
+ });
+
+ it('extracts quoted phrases (both quote styles)', () => {
+ const got = extractEntities(`open "config alpha" then 'session beta'`);
+ expect(got).toContain('config alpha');
+ expect(got).toContain('session beta');
+ });
+
+ it('extracts proper-noun 2-grams', () => {
+ const got = extractEntities('hand off to Alice Smith and notify Acme Corp');
+ expect(got).toContain('Alice Smith');
+ expect(got).toContain('Acme Corp');
+ });
+
+ it('does not extract single capitalized words (too noisy)', () => {
+ expect(extractEntities('Reset the database now')).toEqual([]);
+ });
+
+ it('deduplicates repeats while preserving first-seen order', () => {
+ const got = extractEntities(
+ 'Alice Smith filed Alice Smith again at [email protected]. Alice Smith.'
+ );
+ // 'Alice Smith' should appear exactly once even though the source
+ // mentions it three times.
+ expect(got.filter((e) => e === 'Alice Smith').length).toBe(1);
+ // Both entity types are present.
+ expect(got).toContain('Alice Smith');
+ expect(got).toContain('[email protected]');
+ });
+
+ it('drops trimmed-to-shorter-than-2 fragments', () => {
+ // Quoted single-char phrase ("a") should be filtered by the min-length guard.
+ expect(extractEntities(`prefer "a" over "b"`)).toEqual([]);
+ });
+});
diff --git a/v3/@claude-flow/memory/src/entity-tagger.ts b/v3/@claude-flow/memory/src/entity-tagger.ts
new file mode 100644
index 0000000000..4aaa5e093f
--- /dev/null
+++ b/v3/@claude-flow/memory/src/entity-tagger.ts
@@ -0,0 +1,80 @@
+/**
+ * Entity tagger β€” regex-based proper-noun / structured-token extractor.
+ *
+ * Used by the `hybridSearch` controller as a third RRF arm alongside the
+ * dense (vector) and sparse (BM25/FTS5) signals (ADR-147, ruvnet/ruflo#2317).
+ *
+ * Entity matching is a distinct signal because BM25 weights documents by
+ * overall token frequency; a per-entity exact match avoids downweighting
+ * for tokens that happen to be common but mention an entity by name.
+ * Example: querying "Alice OAuth tokens" β€” BM25 may rank a doc about
+ * generic OAuth above one mentioning Alice specifically; entity arm
+ * surfaces the Alice doc independently.
+ *
+ * P1 is regex-only β€” no NLP model dependency. Tags:
+ * - Emails: [email protected]
+ * - URLs: http(s)://...
+ * - File paths: ./foo/bar.ts, C:\foo\bar.ts, src/foo
+ * - Quoted phrases: "..." or '...'
+ * - Proper-noun 2-grams: "Alice Smith", "Acme Corp"
+ *
+ * Deliberately conservative β€” false negatives are fine (the dense + sparse
+ * arms still cover the query); false positives would dilute the RRF score
+ * by adding noise rows. A future P2 can swap this for a CRF/spaCy tagger
+ * behind the same `extractEntities(text)` contract.
+ */
+
+/**
+ * Extract entity-like tokens from free text. Returns a unique list,
+ * trimmed and deduplicated, in extraction order.
+ */
+export function extractEntities(text: string): string[] {
+ if (!text) return [];
+
+ const seen = new Set<string>();
+ const ordered: string[] = [];
+
+ const add = (raw: string | undefined): void => {
+ if (!raw) return;
+ const trimmed = raw.trim();
+ if (trimmed.length < 2) return;
+ if (seen.has(trimmed)) return;
+ seen.add(trimmed);
+ ordered.push(trimmed);
+ };
+
+ // Emails β€” anchored on @ so we don't pick up arbitrary handles.
+ for (const m of text.matchAll(/[\w.+-]+@[\w-]+(?:\.[\w-]+)+/g)) add(m[0]);
+
+ // URLs β€” http(s) only. Stop at whitespace and common boundary chars.
+ for (const m of text.matchAll(/https?:\/\/[^\s)>"'`]+/g)) add(m[0]);
+
+ // File paths β€” two complementary patterns:
+ //
+ // (a) Path with a file extension. Catches `src/foo/bar.ts`, the most
+ // common case. The extension (`.\w{1,5}`) is what separates a
+ // real path from prose like `and/or`.
+ //
+ // (b) Path with an explicit leading sentinel: `./...`, `../...`, `/...`,
+ // or `C:\...`. Catches extensionless paths the user clearly meant
+ // (like `./scripts/build` or `/etc/hosts`).
+ for (const m of text.matchAll(
+ /(?:[A-Za-z]:[\\/])?(?:[\w.-]+[\\/])+[\w-]+\.\w{1,5}\b/g,
+ )) {
+ add(m[0]);
+ }
+ for (const m of text.matchAll(/(?:\.{1,2}|[A-Za-z]:)[\\/][\w./\\-]+/g)) add(m[0]);
+
+ // Quoted phrases. Inner content only, both quote styles. The
+ // (?<!\w)/(?!\w) guards keep us from pairing a *closing* quote of one
+ // phrase with the *opening* quote of the next β€” e.g. `"a" over "b"`
+ // should NOT capture ` over ` from positions 9–16.
+ for (const m of text.matchAll(/(?<!\w)"([^"]{2,})"(?!\w)/g)) add(m[1]);
+ for (const m of text.matchAll(/(?<!\w)'([^']{2,})'(?!\w)/g)) add(m[1]);
+
+ // Proper-noun 2+ grams β€” capitalized words with capitalized neighbours.
+ // Skip single capitals (too noisy: sentence-initial common nouns).
+ for (const m of text.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+\b/g)) add(m[0]);
+
+ return ordered;
+}
diff --git a/v3/@claude-flow/memory/src/graceful-retrieval.test.ts b/v3/@claude-flow/memory/src/graceful-retrieval.test.ts
index 2912b61e59..37062cdd42 100644
--- a/v3/@claude-flow/memory/src/graceful-retrieval.test.ts
+++ b/v3/@claude-flow/memory/src/graceful-retrieval.test.ts
@@ -153,6 +153,61 @@ describe('Phase 5 β€” hybridSearch controller (RRF + MMR)', () => {
// Hybrid should return up to `limit` entries (10 here, given 50 stored).
expect(results.length).toBeLessThanOrEqual(10);
+ // ADR-147: every result carries provenance β€” at least one signal name.
+ for (const r of results) {
+ expect(Array.isArray(r.signals)).toBe(true);
+ expect(r.signals.length).toBeGreaterThan(0);
+ for (const s of r.signals) {
+ expect(['vector', 'bm25', 'entity']).toContain(s);
+ }
+ }
+
+ await registry.shutdown();
+ await svc.close();
+ });
+
+ it('ADR-147 β€” entity arm surfaces proper-noun matches above vector noise', async () => {
+ // Store 30 generic "authentication" entries (no Alice) + a single
+ // entry mentioning "Alice Smith" β€” the proper-noun bigram extractEntities
+ // should pick up. The entity arm should boost the Alice entry above
+ // the 30 generic ones for a query that names Alice explicitly.
+ const dim = 8;
+ const svc = new MemoryService({
+ dimensions: dim,
+ persistenceEnabled: false,
+ snapshotInterval: 0,
+ embeddingGenerator: async (text: string) => randomVec(dim, hashStr(text)),
+ });
+ await svc.initialize();
+
+ for (let i = 0; i < 30; i++) {
+ const entry = createDefaultEntry({
+ key: `generic-${i}`,
+ content: `authentication and authorization patterns ${i}`,
+ });
+ entry.embedding = randomVec(dim, hashStr(entry.content));
+ await svc.store(entry);
+ }
+ const aliceEntry = createDefaultEntry({
+ key: 'alice-needle',
+ content: 'Alice Smith authored these authentication flows',
+ });
+ aliceEntry.embedding = randomVec(dim, hashStr(aliceEntry.content));
+ await svc.store(aliceEntry);
+
+ const registry = new ControllerRegistry();
+ await registry.initialize({ memoryService: svc });
+
+ const hybrid = registry.get<any>('hybridSearch');
+ const results = await hybrid.search('Alice Smith authentication', { limit: 10 });
+
+ // The Alice entry should appear in the top-10 fused results.
+ const alice = results.find((r: any) => r.entry.key === 'alice-needle');
+ expect(alice).toBeTruthy();
+ // …and should carry the 'entity' provenance signal (it's the only entry
+ // that matches the "Alice Smith" proper-noun bigram).
+ expect(alice.signals).toContain('entity');
+
await registry.shutdown();
await svc.close();
});