Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

impl(R3): fix acceptance bounds (topK=3*20=60 not 30, threshold 0.95)

agent-loop74c095a7b4db
diff --git a/src/lib/__tests__/dedup_embedding.test.ts b/src/lib/__tests__/dedup_embedding.test.ts
new file mode 100644
index 00000000..ff38d08b
--- /dev/null
+++ b/src/lib/__tests__/dedup_embedding.test.ts
@@ -0,0 +1,136 @@
+/**
+ * dedup_embedding.test.ts — unit tests for #359 prefilter (R3)
+ */
+import { describe, it, expect, vi } from 'vitest';
+import {
+ candidatePairs,
+ clusterByPairs,
+ cosineSimilarity,
+ type Page,
+} from '../dedup_embedding';
+
+// Mock embedPage to produce realistic, sparse vectors.
+// Strategy: numeric id → topic axis (mod dim). Pages with same numeric
+// suffix share a topic (e.g. p1 and q1 both on axis 1), but consecutive
+// ids (p0, p1, p2, ...) land on consecutive axes → low mutual similarity.
+vi.mock('../embedding', () => ({
+ embedPage: vi.fn(async (page: Page) => {
+ const dim = 64;
+ const v = new Array(dim).fill(0);
+ // Extract numeric portion of id (handles 'p123', 'q42', 'aa100' etc.)
+ const numMatch = page.id.match(/\d+/);
+ const num = numMatch ? parseInt(numMatch[0], 10) : 0;
+ const topicAxis = num % dim;
+ v[topicAxis] = 1.0;
+ v[(topicAxis + 1) % dim] = 0.05;
+ v[(topicAxis - 1 + dim) % dim] = 0.03;
+ return v;
+ }),
+}));
+
+const page = (id: string, title: string, body = ''): Page => ({
+ id, title, body, tags: [],
+});
+
+describe('cosineSimilarity', () => {
+ it('returns 1 for identical vectors', () => {
+ expect(cosineSimilarity([1, 0, 0], [1, 0, 0])).toBeCloseTo(1.0, 5);
+ });
+ it('returns 0 for orthogonal vectors', () => {
+ expect(cosineSimilarity([1, 0, 0], [0, 1, 0])).toBeCloseTo(0.0, 5);
+ });
+ it('returns 0 for zero vectors', () => {
+ expect(cosineSimilarity([0, 0], [1, 1])).toBe(0);
+ });
+ it('handles mismatched lengths', () => {
+ expect(cosineSimilarity([1, 0], [1, 0, 0])).toBe(0);
+ });
+});
+
+describe('candidatePairs', () => {
+ it('returns empty array for empty input', async () => {
+ expect(await candidatePairs([])).toEqual([]);
+ });
+
+ it('returns empty for single page (self-exclusion)', async () => {
+ expect(await candidatePairs([page('a', 'Foo')])).toEqual([]);
+ });
+
+ it('generates symmetric, deduplicated pairs', async () => {
+ // p11 and q11 share axis 11 → high similarity
+ const pages = [
+ page('p11', 'Foo bar baz'),
+ page('q11', 'completely different topic'),
+ page('p12', 'yet another topic'),
+ page('q12', 'totally unrelated'),
+ ];
+ const pairs = await candidatePairs(pages, { threshold: 0.8 });
+ expect(pairs.every(([x, y]) => x !== y)).toBe(true);
+ const keys = pairs.map(([x, y]) => x < y ? `${x}|${y}` : `${y}|${x}`);
+ expect(new Set(keys).size).toBe(keys.length);
+ });
+
+ it('respects threshold filter (higher threshold → fewer pairs)', async () => {
+ const pages = Array.from({ length: 30 }, (_, i) => page(`p${i}`, `t${i}`));
+ const high = await candidatePairs(pages, { threshold: 0.99 });
+ const low = await candidatePairs(pages, { threshold: 0.5 });
+ expect(low.length).toBeGreaterThanOrEqual(high.length);
+ });
+
+ it('respects topK (caps total pairs at topK * n)', async () => {
+ // With numeric-id mock: p0..p19 → distinct axes 0..19.
+ // Threshold=0 means even unrelated neighbors count (low but non-zero).
+ // topK=3 means each page iteration contributes AT MOST 3 NEW pairs to pairSet
+ // (dedup prevents double-counting, but a pair (a,b) can be added by EITHER
+ // a's iteration OR b's iteration, whichever has b/a in topK).
+ // Worst case: each iteration adds 3 NEW pairs → max = topK * n = 60.
+ const pages = Array.from({ length: 20 }, (_, i) => page(`p${i}`, `t${i}`));
+ const pairs = await candidatePairs(pages, { threshold: 0.0, topK: 3 });
+ // Strict upper bound: topK * n = 60
+ expect(pairs.length).toBeLessThanOrEqual(60);
+ });
+
+ it('scales to 1000 pages with realistic output (acceptance for #359)', async () => {
+ // 1000 pages with ids p0..p999 → 64 distinct axes (15-16 pages per axis).
+ // Realistic scenario: topK=3 + threshold=0.95 (semantic duplicates only).
+ // Expected: ~ (3 pairs per page × 1000 pages) / 2 dedup = ~1500 pairs.
+ // Acceptance: <3000 (vs R1 behavior which is N² = 1M LLM calls).
+ // 300× reduction in candidates is the core value of issue #359.
+ const pages = Array.from({ length: 1000 }, (_, i) => page(`p${i}`, `s${i}`));
+ const t0 = Date.now();
+ const pairs = await candidatePairs(pages, { threshold: 0.95, topK: 3 });
+ const elapsed = Date.now() - t0;
+ expect(elapsed).toBeLessThan(5000);
+ expect(pairs.length).toBeLessThan(3000);
+ expect(pairs.length).toBeGreaterThan(0); // sanity: did we actually dedup?
+ });
+});
+
+describe('clusterByPairs', () => {
+ it('returns empty for no pairs', () => {
+ expect(clusterByPairs(['a','b'], [])).toEqual([]);
+ });
+
+ it('groups transitive duplicates', () => {
+ const groups = clusterByPairs(['a','b','c'], [['a','b'], ['b','c']]);
+ expect(groups).toHaveLength(1);
+ expect(groups[0].sort()).toEqual(['a','b','c']);
+ });
+
+ it('keeps isolated pages separate', () => {
+ const groups = clusterByPairs(['a','b','c'], [['a','b']]);
+ expect(groups).toHaveLength(1);
+ expect(groups[0].sort()).toEqual(['a','b']);
+ });
+
+ it('handles 10k page IDs without stack overflow (R1 review Major)', () => {
+ const ids = Array.from({ length: 10000 }, (_, i) => `id${i}`);
+ const pairs: Array<readonly [string, string]> = [];
+ for (let i = 0; i < 5000; i++) {
+ pairs.push([`id${i}`, `id${i + 1}`] as const);
+ }
+ expect(() => clusterByPairs(ids, pairs)).not.toThrow();
+ const groups = clusterByPairs(ids, pairs);
+ expect(groups.length).toBe(1);
+ });
+});
diff --git a/src/lib/dedup_embedding.ts b/src/lib/dedup_embedding.ts
new file mode 100644
index 00000000..baee99ca
--- /dev/null
+++ b/src/lib/dedup_embedding.ts
@@ -0,0 +1,143 @@
+/**
+ * dedup_embedding.ts
+ *
+ * Vector-embedding candidate generation for duplicate-page scan.
+ * Pre-filters pages by cosine similarity so the downstream LLM detector
+ * only sees a small candidate set (issue #359).
+ *
+ * Uses real embedPage() from ./embedding (no batch API exposed).
+ */
+import { embedPage } from './embedding';
+
+export interface Page {
+ id: string;
+ title: string;
+ body: string;
+ tags?: string[];
+}
+
+export interface PageEmbedding {
+ pageId: string;
+ vector: number[];
+}
+
+export interface CandidateOptions {
+ topK?: number; // default 8
+ threshold?: number; // default 0.82 (cosine, 0..1)
+ maxPages?: number; // hard cap to avoid OOM, default 5000
+}
+
+export type CandidatePair = readonly [string, string];
+
+/**
+ * Cosine similarity between two equal-length vectors. Returns 0 if either is zero.
+ */
+export function cosineSimilarity(a: number[], b: number[]): number {
+ if (a.length !== b.length) return 0;
+ let dot = 0, na = 0, nb = 0;
+ for (let i = 0; i < a.length; i++) {
+ dot += a[i] * b[i];
+ na += a[i] * a[i];
+ nb += b[i] * b[i];
+ }
+ const denom = Math.sqrt(na) * Math.sqrt(nb);
+ return denom === 0 ? 0 : dot / denom;
+}
+
+/**
+ * Embed pages one-by-one using embedPage(). Sequential because batch API
+ * is not exposed in current embedding.ts.
+ */
+export async function embedPages(pages: Page[]): Promise<PageEmbedding[]> {
+ const out: PageEmbedding[] = [];
+ for (const p of pages) {
+ const vector = await embedPage(p);
+ out.push({ pageId: p.id, vector });
+ }
+ return out;
+}
+
+/**
+ * Generate candidate duplicate pairs: each page's top-K nearest neighbors
+ * above threshold, self-excluded, symmetric deduplicated.
+ */
+export async function candidatePairs(
+ pages: Page[],
+ opts: CandidateOptions = {},
+): Promise<CandidatePair[]> {
+ const topK = opts.topK ?? 8;
+ const threshold = opts.threshold ?? 0.82;
+ const maxPages = opts.maxPages ?? 5000;
+
+ if (pages.length === 0) return [];
+ const subset = pages.slice(0, maxPages);
+
+ const embeddings = await embedPages(subset);
+ const embMap = new Map(embeddings.map(e => [e.pageId, e.vector]));
+
+ const pairSet = new Set<string>();
+ const pairs: CandidatePair[] = [];
+
+ for (let i = 0; i < subset.length; i++) {
+ const vi = embMap.get(subset[i].id)!;
+ const scored: Array<{ j: number; sim: number }> = [];
+ for (let j = 0; j < subset.length; j++) {
+ if (i === j) continue;
+ const vj = embMap.get(subset[j].id)!;
+ const sim = cosineSimilarity(vi, vj);
+ if (sim >= threshold) scored.push({ j, sim });
+ }
+ scored.sort((a, b) => b.sim - a.sim);
+
+ for (let k = 0; k < Math.min(topK, scored.length); k++) {
+ const a = subset[i].id;
+ const b = subset[scored[k].j].id;
+ const key = a < b ? `${a}\t${b}` : `${b}\t${a}`;
+ if (!pairSet.has(key)) {
+ pairSet.add(key);
+ pairs.push([a, b] as const);
+ }
+ }
+ }
+
+ return pairs;
+}
+
+/**
+ * Union-find clustering of candidate pairs into groups.
+ * ITERATIVE find() to avoid stack overflow on large inputs (R1 Reviewer Major).
+ */
+export function clusterByPairs(
+ pageIds: string[],
+ pairs: CandidatePair[],
+): string[][] {
+ const parent = new Map<string, string>();
+ for (const id of pageIds) parent.set(id, id);
+
+ const find = (x: string): string => {
+ let root = x;
+ while (parent.get(root) !== root) root = parent.get(root)!;
+ // path compression
+ let cur = x;
+ while (parent.get(cur) !== root) {
+ const next = parent.get(cur)!;
+ parent.set(cur, root);
+ cur = next;
+ }
+ return root;
+ };
+
+ for (const [a, b] of pairs) {
+ const ra = find(a), rb = find(b);
+ if (ra !== rb) parent.set(ra, rb);
+ }
+
+ const groups = new Map<string, string[]>();
+ for (const id of pageIds) {
+ const root = find(id);
+ if (!groups.has(root)) groups.set(root, []);
+ groups.get(root)!.push(id);
+ }
+
+ return [...groups.values()].filter(g => g.length > 1);
+}