LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
feat: hybrid retrieval — RRF fusion of token + vector search
nash_su394816997b74
diff --git a/src/lib/search-rrf.test.ts b/src/lib/search-rrf.test.tsnew file mode 100644index 00000000..ebd24c26--- /dev/null+++ b/src/lib/search-rrf.test.ts@@ -0,0 +1,253 @@+/**+ * Hybrid retrieval (RRF) tests for searchWiki.+ *+ * search.scenarios.test.ts disables embedding to test token search in+ * isolation. THIS file does the inverse: scripts both token search+ * input (real materialized wiki dir) AND vector search output (via+ * a stubbed searchByEmbedding) so we can pin the exact RRF math+ * and ranking semantics.+ *+ * Why RRF matters and what these tests guard:+ * Before this refactor, vector results contributed `vr.score * 5`+ * to the existing token score. Token-only signals (FILENAME_EXACT+ * = 200, PHRASE_IN_TITLE = 50, …) drowned out the cosine sim ≤ 1.0+ * range, so a page with NO keyword overlap but perfect semantic+ * match would max out at score 5 and never make top-10 against any+ * keyword-matching page. This file pins that this is no longer true.+ */+import { describe, it, expect, vi, beforeEach } from "vitest"+import path from "node:path"+import fs from "node:fs/promises"+import { realFs, createTempProject } from "@/test-helpers/fs-temp"++vi.mock("@/commands/fs", () => realFs)++// Stub searchByEmbedding so each test scripts the exact ranked list+// of vector results coming from LanceDB, with no real model calls.+const mockSearchByEmbedding =+ vi.fn<(...args: unknown[]) => Promise<Array<{ id: string; score: number }>>>()+vi.mock("./embedding", () => ({+ searchByEmbedding: (...args: unknown[]) => mockSearchByEmbedding(...args),+}))++import { searchWiki } from "./search"+import { useWikiStore } from "@/stores/wiki-store"++interface Ctx {+ tmp: { path: string; cleanup: () => Promise<void> }+}+let ctx: Ctx | undefined++/** Lay out a small project on disk: just the wiki/* files we need. */+async function setupProject(files: Record<string, string>): Promise<Ctx> {+ const tmp = await createTempProject("search-rrf")+ for (const [rel, content] of Object.entries(files)) {+ const full = path.join(tmp.path, rel)+ await fs.mkdir(path.dirname(full), { recursive: true })+ await fs.writeFile(full, content, "utf-8")+ }+ return { tmp }+}++beforeEach(() => {+ mockSearchByEmbedding.mockReset()+ // Default: embedding ENABLED (the whole point of this file). Tests+ // that want token-only override locally.+ useWikiStore.getState().setEmbeddingConfig({+ enabled: true,+ endpoint: "http://test/v1/embeddings",+ apiKey: "",+ model: "test-embed",+ })+})++afterEach(async () => {+ if (ctx) {+ await ctx.tmp.cleanup()+ ctx = undefined+ }+})++// Vitest's `afterEach` import — declare here to keep the import block+// near the top tidy and use the same lifecycle hooks shape as+// search.scenarios.test.ts.+import { afterEach } from "vitest"++describe("searchWiki — RRF fusion of token + vector lists", () => {+ it("a vector-only match (no keyword overlap) surfaces ahead of a token-only weak match", async () => {+ // The query mentions "memory bandwidth"; flash-attention.md doesn't+ // contain that phrase verbatim but is semantically about it. A+ // separate page literally contains the word "memory" in unrelated+ // context. Old behavior: token-match page wins. RRF behavior:+ // vector match wins because it ranks #1 in vector list while the+ // token match ranks deep in token list.+ ctx = await setupProject({+ "wiki/concepts/flash-attention.md":+ "---\ntitle: Flash Attention\n---\n\n# Flash Attention\n\nIO-aware tiled attention.",+ "wiki/concepts/memory-leak.md":+ "---\ntitle: Memory Leak Diagnosis\n---\n\n# Memory Leak\n\nRSS growing over time.",+ })++ // Token search will find "memory-leak" (literal "Memory" hit) and+ // possibly nothing for flash-attention. Vector search ranks+ // flash-attention #1.+ mockSearchByEmbedding.mockResolvedValueOnce([+ { id: "flash-attention", score: 0.85 },+ { id: "memory-leak", score: 0.4 },+ ])++ const out = await searchWiki(ctx.tmp.path, "GPU memory bandwidth optimization for attention")++ // The vector match must be in the top result. Without RRF, the+ // keyword-only "memory-leak" page would dominate.+ expect(out[0].title).toBe("Flash Attention")+ // Both pages still appear (RRF surfaces both lists' contents).+ const titles = out.map((r) => r.title)+ expect(titles).toContain("Memory Leak Diagnosis")+ })++ it("a page in BOTH lists outranks a page in only one (combined contribution)", async () => {+ // Construct three pages:+ // - A: matches token search (rank 1) AND vector search (rank 1)+ // - B: only matches token search (rank 1's neighbor at rank 2)+ // - C: only matches vector search (rank 2)+ //+ // RRF expectation:+ // A: 1/61 + 1/61 = 0.0328+ // B: 1/62 + 0 = 0.0161+ // C: 0 + 1/62 = 0.0161+ //+ // → A wins clearly; B and C tie (broken by alphabetical path).+ ctx = await setupProject({+ "wiki/concepts/aaa.md": "---\ntitle: AAA\n---\n\n# AAA\n\nrope rope rope.",+ "wiki/concepts/bbb.md": "---\ntitle: BBB\n---\n\n# BBB\n\nrope.",+ "wiki/concepts/ccc.md": "---\ntitle: CCC\n---\n\n# CCC\n\nunrelated.",+ })++ mockSearchByEmbedding.mockResolvedValueOnce([+ { id: "aaa", score: 0.95 },+ { id: "ccc", score: 0.7 },+ ])++ const out = await searchWiki(ctx.tmp.path, "rope")++ expect(out[0].title).toBe("AAA")+ // Score: aaa got contributions from BOTH; bbb and ccc each from ONE.+ expect(out[0].score).toBeCloseTo(1 / 61 + 1 / 61, 6)+ // The runner-up is bbb or ccc — both at rank 2 of their list, both+ // contributing 1/62. Tie broken by path → alphabetical. concepts/bbb+ // < concepts/ccc, so bbb is #2.+ expect(out[1].title).toBe("BBB")+ expect(out[2].title).toBe("CCC")+ expect(out[1].score).toBeCloseTo(1 / 62, 6)+ expect(out[2].score).toBeCloseTo(1 / 62, 6)+ })++ it("vector list is empty (embedding disabled) → behaves like pure token rank", async () => {+ useWikiStore.getState().setEmbeddingConfig({+ enabled: false,+ endpoint: "",+ apiKey: "",+ model: "",+ })++ ctx = await setupProject({+ "wiki/concepts/attention.md":+ "---\ntitle: Attention\n---\n\n# Attention\n\nbody about attention.",+ "wiki/concepts/random.md":+ "---\ntitle: Random\n---\n\n# Random\n\nattention is mentioned briefly.",+ })++ const out = await searchWiki(ctx.tmp.path, "attention")+ // Token search puts attention.md (filename exact match → 200 bonus)+ // at rank 1, random.md at rank 2.+ expect(out[0].title).toBe("Attention")+ expect(out[0].score).toBeCloseTo(1 / 61, 6)+ expect(out[1].title).toBe("Random")+ expect(out[1].score).toBeCloseTo(1 / 62, 6)+ // searchByEmbedding must NOT have been called at all.+ expect(mockSearchByEmbedding).not.toHaveBeenCalled()+ })++ it("token list empty (no keyword match anywhere) → vector ranks alone determine order", async () => {+ ctx = await setupProject({+ "wiki/concepts/foo.md": "---\ntitle: Foo\n---\n\n# Foo\n\nfoo body.",+ "wiki/concepts/bar.md": "---\ntitle: Bar\n---\n\n# Bar\n\nbar body.",+ })++ // Query has no token overlap with either page; vector finds them.+ mockSearchByEmbedding.mockResolvedValueOnce([+ { id: "bar", score: 0.7 },+ { id: "foo", score: 0.3 },+ ])++ const out = await searchWiki(ctx.tmp.path, "completely unrelated query terms")+ // Order matches vector ranks: bar (rank 1) > foo (rank 2).+ expect(out.map((r) => r.title)).toEqual(["Bar", "Foo"])+ expect(out[0].score).toBeCloseTo(1 / 61, 6)+ expect(out[1].score).toBeCloseTo(1 / 62, 6)+ })++ it("vector contribution is rank-based, NOT score-magnitude — fixes the score-dwarfing bug", async () => {+ // This is the precise regression we're guarding. Old code:+ // existing.score += vr.score * 5+ // → vector cos_sim 0.95 boosts by only 4.75+ // → buried under any token PHRASE_IN_TITLE_BONUS (50)+ //+ // Build: page A is the ONLY token match (low-quality match,+ // rank 1 of token but at the absolute floor of token scoring);+ // page B is vector rank 1 (no token hits at all).+ // Expected: B (in top-1 of vector) MUST appear at the top+ // alongside A — under the old behavior, B never made it in.+ ctx = await setupProject({+ "wiki/concepts/quick-mention.md":+ "---\ntitle: Quick Mention\n---\n\n# Quick Mention\n\nrope is mentioned once.",+ "wiki/concepts/positional-encoding.md":+ "---\ntitle: Positional Encoding\n---\n\n# Positional Encoding\n\nstrong semantic match.",+ })++ // Token search will hit "rope" only in quick-mention.md.+ // Vector search puts positional-encoding at rank 1.+ mockSearchByEmbedding.mockResolvedValueOnce([+ { id: "positional-encoding", score: 0.92 },+ ])++ const out = await searchWiki(ctx.tmp.path, "rope")++ // Both must be present.+ const ids = out.map((r) => r.path.split("/").pop())+ expect(ids).toContain("quick-mention.md")+ expect(ids).toContain("positional-encoding.md")+ // positional-encoding contributes 1/61 (vector rank 1) and+ // quick-mention contributes 1/61 (token rank 1). They tie on+ // score; alphabetical tie-break gives positional-encoding (p < q).+ expect(out[0].title).toBe("Positional Encoding")+ expect(out[1].title).toBe("Quick Mention")+ // Both scores should be 1/61 — proving vector rank 1 carries+ // the same weight as token rank 1, NOT 5/200th of it.+ expect(out[0].score).toBeCloseTo(1 / 61, 6)+ expect(out[1].score).toBeCloseTo(1 / 61, 6)+ })++ it("vector results referencing nonexistent page ids are silently dropped (post-source-delete safety)", async () => {+ // After a source-delete cascade, LanceDB *should* have its chunks+ // gone too (we just fixed that bug), but be defensive: if vector+ // search returns an id whose .md file doesn't exist, the search+ // pipeline must not crash and must not surface a phantom result.+ ctx = await setupProject({+ "wiki/concepts/exists.md":+ "---\ntitle: Exists\n---\n\n# Exists\n\nbody.",+ })++ mockSearchByEmbedding.mockResolvedValueOnce([+ { id: "exists", score: 0.9 },+ { id: "ghost", score: 0.85 }, // no file with this slug+ ])++ const out = await searchWiki(ctx.tmp.path, "anything")+ // Only the real page surfaces. Ghost is dropped during+ // materialization (every `tryPath` readFile throws ENOENT).+ expect(out).toHaveLength(1)+ expect(out[0].title).toBe("Exists")+ })+})diff --git a/src/lib/search.ts b/src/lib/search.tsindex 64d7414b..947453e4 100644--- a/src/lib/search.ts+++ b/src/lib/search.ts@@ -13,6 +13,23 @@ export interface SearchResult {const MAX_RESULTS = 20const SNIPPET_CONTEXT = 80+// ── Reciprocal Rank Fusion ─────────────────────────────────────────────────+// Token search and vector search produce two independently-ranked lists.+// Their absolute scores are incommensurable (token score: 1-400, vector+// cosine: 0-1), so summing them privileges whichever list happens to use+// the larger numbers. RRF sidesteps that by fusing on RANK only:+//+// fused(p) = sum over lists L of 1 / (K + rank_L(p))+//+// A page that ranks #1 in BOTH lists wins handily. A page that's only in+// one list still surfaces if it ranks high there, but a page in BOTH a+// little lower can outrank it — exactly what we want for hybrid retrieval.+//+// K=60 is the canonical constant from Cormack et al. (SIGIR 2009), large+// enough that small rank differences near the top don't dominate but+// small enough that being deep in either list still falls off quickly.+const RRF_K = 60+// ── Scoring weights ────────────────────────────────────────────────────────// Exact lexical matches dominate everything else. The rationale: when a// user types "attention", the page literally named `attention.md` MUST@@ -176,7 +193,20 @@ export async function searchWiki(// no raw sources}- // Vector search: merge semantic results if embedding enabled+ // ── Build the token-side ranking (still based on the score field+ // populated by searchFiles above). Snapshot it BEFORE the vector+ // step, so adding vector-only pages doesn't shift token ranks.+ const tokenSorted = [...results].sort((a, b) => b.score - a.score)+ const tokenRank = new Map<string, number>()+ tokenSorted.forEach((r, i) => {+ tokenRank.set(normalizePath(r.path), i + 1) // 1-indexed+ })++ // ── Vector search: collect ranked list of page-ids and materialize+ // pages that token search missed. We do NOT add to results' score+ // here — that's done in the RRF step below.+ let vectorRank = new Map<string, number>()+ let vectorCount = 0try {const { useWikiStore } = await import("@/stores/wiki-store")const embCfg = useWikiStore.getState().embeddingConfig@@ -186,6 +216,7 @@ export async function searchWiki(const { searchByEmbedding } = await import("@/lib/embedding")const vectorResults = await searchByEmbedding(pp, query, embCfg, 10)const vectorMs = Math.round(performance.now() - t0)+ vectorCount = vectorResults.lengthconsole.log(`[Vector Search] query="${query}" | ${vectorResults.length} results in ${vectorMs}ms | model=${embCfg.model}` +@@ -194,61 +225,74 @@ export async function searchWiki(: ""))- let boosted = 0- let added = 0- // Normalize existing paths to forward slashes so Windows backslash- // paths from the filesystem match the forward-slash paths we- // construct for tryPath below.- const existingPaths = new Set(results.map((r) => normalizePath(r.path)))+ // Build vectorRank by page_id (slug); searchByEmbedding returns+ // results pre-sorted by descending similarity.+ vectorResults.forEach((vr, i) => vectorRank.set(vr.id, i + 1))+ // Materialize any vector-result page that token search didn't+ // already include — without this, `results` has no entry for+ // them and they can't surface even with a top vector rank.+ const knownIds = new Set(results.map((r) => getFileStem(r.path)))+ let added = 0for (const vr of vectorResults) {- // Check if already in results. getFileStem handles both / and \- // separators so this works on Windows.- const existing = results.find((r) => getFileStem(r.path) === vr.id)-- if (existing) {- // Boost score of existing result- existing.score += vr.score * 5- boosted++- } else {- // Try to find the file and add it- const dirs = ["entities", "concepts", "sources", "synthesis", "comparison", "queries"]- for (const dir of dirs) {- const tryPath = `${pp}/wiki/${dir}/${vr.id}.md`- if (existingPaths.has(tryPath)) break- try {- const content = await readFile(tryPath)- const title = extractTitle(content, `${vr.id}.md`)- results.push({- path: tryPath,- title,- snippet: buildSnippet(content, query),- titleMatch: false,- score: vr.score * 5,- })- existingPaths.add(tryPath)- added++- break- } catch {- // not in this directory- }+ if (knownIds.has(vr.id)) continue+ const dirs = ["entities", "concepts", "sources", "synthesis", "comparison", "queries"]+ for (const dir of dirs) {+ const tryPath = `${pp}/wiki/${dir}/${vr.id}.md`+ try {+ const content = await readFile(tryPath)+ const title = extractTitle(content, `${vr.id}.md`)+ results.push({+ path: tryPath,+ title,+ snippet: buildSnippet(content, query),+ titleMatch: false,+ score: 0, // overwritten by RRF below+ })+ knownIds.add(vr.id)+ added+++ break+ } catch {+ // not in this directory}}}-- if (boosted > 0 || added > 0) {- console.log(`[Vector Search] Merged: ${boosted} boosted, ${added} new pages added`)+ if (added > 0) {+ console.log(`[Vector Search] Added ${added} vector-only pages to candidate set`)}}} catch (err) {console.log(`[Vector Search] Skipped: ${err instanceof Error ? err.message : "not available"}`)+ vectorRank = new Map()}- // Sort by score descending- results.sort((a, b) => b.score - a.score)+ // ── RRF fusion: replace each result's score with+ // 1/(K + token_rank) + 1/(K + vector_rank)+ //+ // Pages absent from either list contribute 0 from that side.+ // Pages absent from BOTH never make it here (we only iterate the+ // results array, which already contains every candidate).+ for (const r of results) {+ const tRank = tokenRank.get(normalizePath(r.path))+ const vRank = vectorRank.get(getFileStem(r.path))+ let rrf = 0+ if (tRank !== undefined) rrf += 1 / (RRF_K + tRank)+ if (vRank !== undefined) rrf += 1 / (RRF_K + vRank)+ r.score = rrf+ }- const tokenCount = results.filter((r) => r.score > 0).length- console.log(`[Search] query="${query}" | ${tokenCount} token matches | ${results.length} total results`)+ // Sort by RRF score descending. Ties (e.g. two pages both at vector+ // rank 1 but neither in token list) are broken by alphabetical path+ // order so output is deterministic for tests.+ results.sort((a, b) => {+ if (b.score !== a.score) return b.score - a.score+ return a.path.localeCompare(b.path)+ })++ const tokenHits = tokenRank.size+ console.log(+ `[Search] query="${query}" | RRF fused: ${tokenHits} token + ${vectorCount} vector → ${results.length} unique`,+ )return results.slice(0, MAX_RESULTS)}