LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
impl(R3): fix acceptance bounds (topK=3*20=60 not 30, threshold 0.95)
agent-loop74c095a7b4db
diff --git a/src/lib/__tests__/dedup_embedding.test.ts b/src/lib/__tests__/dedup_embedding.test.tsnew file mode 100644index 00000000..ff38d08b--- /dev/null+++ b/src/lib/__tests__/dedup_embedding.test.ts@@ -0,0 +1,136 @@+/**+ * dedup_embedding.test.ts — unit tests for #359 prefilter (R3)+ */+import { describe, it, expect, vi } from 'vitest';+import {+ candidatePairs,+ clusterByPairs,+ cosineSimilarity,+ type Page,+} from '../dedup_embedding';++// Mock embedPage to produce realistic, sparse vectors.+// Strategy: numeric id → topic axis (mod dim). Pages with same numeric+// suffix share a topic (e.g. p1 and q1 both on axis 1), but consecutive+// ids (p0, p1, p2, ...) land on consecutive axes → low mutual similarity.+vi.mock('../embedding', () => ({+ embedPage: vi.fn(async (page: Page) => {+ const dim = 64;+ const v = new Array(dim).fill(0);+ // Extract numeric portion of id (handles 'p123', 'q42', 'aa100' etc.)+ const numMatch = page.id.match(/\d+/);+ const num = numMatch ? parseInt(numMatch[0], 10) : 0;+ const topicAxis = num % dim;+ v[topicAxis] = 1.0;+ v[(topicAxis + 1) % dim] = 0.05;+ v[(topicAxis - 1 + dim) % dim] = 0.03;+ return v;+ }),+}));++const page = (id: string, title: string, body = ''): Page => ({+ id, title, body, tags: [],+});++describe('cosineSimilarity', () => {+ it('returns 1 for identical vectors', () => {+ expect(cosineSimilarity([1, 0, 0], [1, 0, 0])).toBeCloseTo(1.0, 5);+ });+ it('returns 0 for orthogonal vectors', () => {+ expect(cosineSimilarity([1, 0, 0], [0, 1, 0])).toBeCloseTo(0.0, 5);+ });+ it('returns 0 for zero vectors', () => {+ expect(cosineSimilarity([0, 0], [1, 1])).toBe(0);+ });+ it('handles mismatched lengths', () => {+ expect(cosineSimilarity([1, 0], [1, 0, 0])).toBe(0);+ });+});++describe('candidatePairs', () => {+ it('returns empty array for empty input', async () => {+ expect(await candidatePairs([])).toEqual([]);+ });++ it('returns empty for single page (self-exclusion)', async () => {+ expect(await candidatePairs([page('a', 'Foo')])).toEqual([]);+ });++ it('generates symmetric, deduplicated pairs', async () => {+ // p11 and q11 share axis 11 → high similarity+ const pages = [+ page('p11', 'Foo bar baz'),+ page('q11', 'completely different topic'),+ page('p12', 'yet another topic'),+ page('q12', 'totally unrelated'),+ ];+ const pairs = await candidatePairs(pages, { threshold: 0.8 });+ expect(pairs.every(([x, y]) => x !== y)).toBe(true);+ const keys = pairs.map(([x, y]) => x < y ? `${x}|${y}` : `${y}|${x}`);+ expect(new Set(keys).size).toBe(keys.length);+ });++ it('respects threshold filter (higher threshold → fewer pairs)', async () => {+ const pages = Array.from({ length: 30 }, (_, i) => page(`p${i}`, `t${i}`));+ const high = await candidatePairs(pages, { threshold: 0.99 });+ const low = await candidatePairs(pages, { threshold: 0.5 });+ expect(low.length).toBeGreaterThanOrEqual(high.length);+ });++ it('respects topK (caps total pairs at topK * n)', async () => {+ // With numeric-id mock: p0..p19 → distinct axes 0..19.+ // Threshold=0 means even unrelated neighbors count (low but non-zero).+ // topK=3 means each page iteration contributes AT MOST 3 NEW pairs to pairSet+ // (dedup prevents double-counting, but a pair (a,b) can be added by EITHER+ // a's iteration OR b's iteration, whichever has b/a in topK).+ // Worst case: each iteration adds 3 NEW pairs → max = topK * n = 60.+ const pages = Array.from({ length: 20 }, (_, i) => page(`p${i}`, `t${i}`));+ const pairs = await candidatePairs(pages, { threshold: 0.0, topK: 3 });+ // Strict upper bound: topK * n = 60+ expect(pairs.length).toBeLessThanOrEqual(60);+ });++ it('scales to 1000 pages with realistic output (acceptance for #359)', async () => {+ // 1000 pages with ids p0..p999 → 64 distinct axes (15-16 pages per axis).+ // Realistic scenario: topK=3 + threshold=0.95 (semantic duplicates only).+ // Expected: ~ (3 pairs per page × 1000 pages) / 2 dedup = ~1500 pairs.+ // Acceptance: <3000 (vs R1 behavior which is N² = 1M LLM calls).+ // 300× reduction in candidates is the core value of issue #359.+ const pages = Array.from({ length: 1000 }, (_, i) => page(`p${i}`, `s${i}`));+ const t0 = Date.now();+ const pairs = await candidatePairs(pages, { threshold: 0.95, topK: 3 });+ const elapsed = Date.now() - t0;+ expect(elapsed).toBeLessThan(5000);+ expect(pairs.length).toBeLessThan(3000);+ expect(pairs.length).toBeGreaterThan(0); // sanity: did we actually dedup?+ });+});++describe('clusterByPairs', () => {+ it('returns empty for no pairs', () => {+ expect(clusterByPairs(['a','b'], [])).toEqual([]);+ });++ it('groups transitive duplicates', () => {+ const groups = clusterByPairs(['a','b','c'], [['a','b'], ['b','c']]);+ expect(groups).toHaveLength(1);+ expect(groups[0].sort()).toEqual(['a','b','c']);+ });++ it('keeps isolated pages separate', () => {+ const groups = clusterByPairs(['a','b','c'], [['a','b']]);+ expect(groups).toHaveLength(1);+ expect(groups[0].sort()).toEqual(['a','b']);+ });++ it('handles 10k page IDs without stack overflow (R1 review Major)', () => {+ const ids = Array.from({ length: 10000 }, (_, i) => `id${i}`);+ const pairs: Array<readonly [string, string]> = [];+ for (let i = 0; i < 5000; i++) {+ pairs.push([`id${i}`, `id${i + 1}`] as const);+ }+ expect(() => clusterByPairs(ids, pairs)).not.toThrow();+ const groups = clusterByPairs(ids, pairs);+ expect(groups.length).toBe(1);+ });+});diff --git a/src/lib/dedup_embedding.ts b/src/lib/dedup_embedding.tsnew file mode 100644index 00000000..baee99ca--- /dev/null+++ b/src/lib/dedup_embedding.ts@@ -0,0 +1,143 @@+/**+ * dedup_embedding.ts+ *+ * Vector-embedding candidate generation for duplicate-page scan.+ * Pre-filters pages by cosine similarity so the downstream LLM detector+ * only sees a small candidate set (issue #359).+ *+ * Uses real embedPage() from ./embedding (no batch API exposed).+ */+import { embedPage } from './embedding';++export interface Page {+ id: string;+ title: string;+ body: string;+ tags?: string[];+}++export interface PageEmbedding {+ pageId: string;+ vector: number[];+}++export interface CandidateOptions {+ topK?: number; // default 8+ threshold?: number; // default 0.82 (cosine, 0..1)+ maxPages?: number; // hard cap to avoid OOM, default 5000+}++export type CandidatePair = readonly [string, string];++/**+ * Cosine similarity between two equal-length vectors. Returns 0 if either is zero.+ */+export function cosineSimilarity(a: number[], b: number[]): number {+ if (a.length !== b.length) return 0;+ let dot = 0, na = 0, nb = 0;+ for (let i = 0; i < a.length; i++) {+ dot += a[i] * b[i];+ na += a[i] * a[i];+ nb += b[i] * b[i];+ }+ const denom = Math.sqrt(na) * Math.sqrt(nb);+ return denom === 0 ? 0 : dot / denom;+}++/**+ * Embed pages one-by-one using embedPage(). Sequential because batch API+ * is not exposed in current embedding.ts.+ */+export async function embedPages(pages: Page[]): Promise<PageEmbedding[]> {+ const out: PageEmbedding[] = [];+ for (const p of pages) {+ const vector = await embedPage(p);+ out.push({ pageId: p.id, vector });+ }+ return out;+}++/**+ * Generate candidate duplicate pairs: each page's top-K nearest neighbors+ * above threshold, self-excluded, symmetric deduplicated.+ */+export async function candidatePairs(+ pages: Page[],+ opts: CandidateOptions = {},+): Promise<CandidatePair[]> {+ const topK = opts.topK ?? 8;+ const threshold = opts.threshold ?? 0.82;+ const maxPages = opts.maxPages ?? 5000;++ if (pages.length === 0) return [];+ const subset = pages.slice(0, maxPages);++ const embeddings = await embedPages(subset);+ const embMap = new Map(embeddings.map(e => [e.pageId, e.vector]));++ const pairSet = new Set<string>();+ const pairs: CandidatePair[] = [];++ for (let i = 0; i < subset.length; i++) {+ const vi = embMap.get(subset[i].id)!;+ const scored: Array<{ j: number; sim: number }> = [];+ for (let j = 0; j < subset.length; j++) {+ if (i === j) continue;+ const vj = embMap.get(subset[j].id)!;+ const sim = cosineSimilarity(vi, vj);+ if (sim >= threshold) scored.push({ j, sim });+ }+ scored.sort((a, b) => b.sim - a.sim);++ for (let k = 0; k < Math.min(topK, scored.length); k++) {+ const a = subset[i].id;+ const b = subset[scored[k].j].id;+ const key = a < b ? `${a}\t${b}` : `${b}\t${a}`;+ if (!pairSet.has(key)) {+ pairSet.add(key);+ pairs.push([a, b] as const);+ }+ }+ }++ return pairs;+}++/**+ * Union-find clustering of candidate pairs into groups.+ * ITERATIVE find() to avoid stack overflow on large inputs (R1 Reviewer Major).+ */+export function clusterByPairs(+ pageIds: string[],+ pairs: CandidatePair[],+): string[][] {+ const parent = new Map<string, string>();+ for (const id of pageIds) parent.set(id, id);++ const find = (x: string): string => {+ let root = x;+ while (parent.get(root) !== root) root = parent.get(root)!;+ // path compression+ let cur = x;+ while (parent.get(cur) !== root) {+ const next = parent.get(cur)!;+ parent.set(cur, root);+ cur = next;+ }+ return root;+ };++ for (const [a, b] of pairs) {+ const ra = find(a), rb = find(b);+ if (ra !== rb) parent.set(ra, rb);+ }++ const groups = new Map<string, string[]>();+ for (const id of pageIds) {+ const root = find(id);+ if (!groups.has(root)) groups.set(root, []);+ groups.get(root)!.push(id);+ }++ return [...groups.values()].filter(g => g.length > 1);+}