LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
feat(embedding/rag Phase 3+4): chunk-level embedding + auto-halve retry
nash_sufc0c2d8e6cad
diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.tsnew file mode 100644index 00000000..557eae65--- /dev/null+++ b/src/lib/embedding.test.ts@@ -0,0 +1,313 @@+/**+ * Unit-level tests for embedding.ts, focused on the pure / mockable+ * pieces: auto-halve retry heuristics and the chunk→page aggregation+ * contract inside `searchByEmbedding`.+ *+ * The actual HTTP layer and Tauri LanceDB commands are mocked — we're+ * NOT testing Rust vectorstore here (that has its own 15 Rust tests)+ * nor the webview fetch (that's tauri-fetch.ts). The boundary we pin+ * down here is "given these chunk-level results, do we aggregate to+ * page-level scores the way the design says?".+ */+import { describe, it, expect, vi, beforeEach } from "vitest"++// Module-level mock for the Tauri invoke boundary so we can script the+// chunk-search response without touching real LanceDB.+const mockInvoke = vi.fn<(cmd: string, args?: unknown) => Promise<unknown>>()+vi.mock("@tauri-apps/api/core", () => ({+ invoke: (cmd: string, args?: unknown) => mockInvoke(cmd, args),+}))++// Stub getHttpFetch so fetchEmbedding calls hit our in-test responder.+const mockHttpFetch = vi.fn<(url: string, opts?: RequestInit) => Promise<Response>>()+vi.mock("@/lib/tauri-fetch", () => ({+ getHttpFetch: () => Promise.resolve(mockHttpFetch),+ isFetchNetworkError: (err: unknown) =>+ err instanceof TypeError ||+ (err instanceof Error &&+ (err.message === "Load failed" || err.message === "Failed to fetch")),+}))++// readFile / listDirectory aren't exercised in this file's cases; stub.+vi.mock("@/commands/fs", () => ({+ readFile: vi.fn(),+ listDirectory: vi.fn(),+}))++import {+ searchByEmbedding,+ embedPage,+ getLastEmbeddingError,+ type PageSearchResult,+} from "./embedding"++const cfg = {+ enabled: true,+ endpoint: "http://localhost:1234/v1/embeddings",+ apiKey: "",+ model: "test-embed",+}++/** Build an embedding-shaped JSON Response. */+function okResponse(embedding: number[]): Response {+ return new Response(JSON.stringify({ data: [{ embedding }] }), {+ status: 200,+ headers: { "Content-Type": "application/json" },+ })+}++/** Build an HTTP-error Response that looks like an oversize rejection. */+function oversizeErrorResponse(status = 400): Response {+ return new Response(+ JSON.stringify({ error: "input length 3200 exceeds maximum context 512" }),+ { status, statusText: "Bad Request" },+ )+}++function genericErrorResponse(status: number, body: string): Response {+ return new Response(body, { status, statusText: "Error" })+}++beforeEach(() => {+ mockInvoke.mockReset()+ mockHttpFetch.mockReset()+})++// ── searchByEmbedding — chunk→page aggregation ─────────────────────++describe("searchByEmbedding — aggregation", () => {+ it("returns [] when embedding call fails (null vector)", async () => {+ mockHttpFetch.mockResolvedValue(genericErrorResponse(401, "Unauthorized"))+ const out = await searchByEmbedding("/tmp/p", "hello", cfg, 5)+ expect(out).toEqual([])+ })++ it("returns [] when no chunks come back from LanceDB", async () => {+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))+ mockInvoke.mockResolvedValueOnce([] /* vector_search_chunks result */)+ const out = await searchByEmbedding("/tmp/p", "hello", cfg, 5)+ expect(out).toEqual([])+ })++ it("groups chunks by page and ranks by max-pool score", async () => {+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))+ // Three pages: A has two chunks (strong + weak), B has one strong,+ // C has one weak. Expected order: A (top 0.9 + blended tail) > B+ // (0.88) > C (0.2).+ mockInvoke.mockResolvedValueOnce([+ { chunk_id: "A#0", page_id: "A", chunk_index: 0, chunk_text: "a0", heading_path: "## X", score: 0.9 },+ { chunk_id: "A#1", page_id: "A", chunk_index: 1, chunk_text: "a1", heading_path: "## Y", score: 0.5 },+ { chunk_id: "B#0", page_id: "B", chunk_index: 0, chunk_text: "b0", heading_path: "", score: 0.88 },+ { chunk_id: "C#0", page_id: "C", chunk_index: 0, chunk_text: "c0", heading_path: "", score: 0.2 },+ ])+ const out = await searchByEmbedding("/tmp/p", "q", cfg, 10)+ expect(out.map((p) => p.id)).toEqual(["A", "B", "C"])+ // A's score: 0.9 + min(0.5 * 0.3, 1 - 0.9) = 0.9 + min(0.15, 0.1) = 1.0+ expect(out[0].score).toBeCloseTo(1.0, 5)+ // B's score: 0.88 + 0 = 0.88+ expect(out[1].score).toBeCloseTo(0.88, 5)+ // C's score: 0.2 + 0 = 0.2+ expect(out[2].score).toBeCloseTo(0.2, 5)+ })++ it("caps tail contribution so a page with many weak chunks can't overtake a strong single-chunk page", async () => {+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))+ // Page X: one chunk at 0.6, four chunks at 0.4 each (tail sum = 1.6).+ // top=0.6, tail_weighted = 0.3 * 1.6 = 0.48, capped at 1 - 0.6 = 0.4+ // blended = 0.6 + 0.4 = 1.0+ // Page Y: single 0.95 chunk.+ // blended = 0.95+ // X would unfairly win if we didn't cap the tail.+ mockInvoke.mockResolvedValueOnce([+ { chunk_id: "X#0", page_id: "X", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.6 },+ { chunk_id: "X#1", page_id: "X", chunk_index: 1, chunk_text: "", heading_path: "", score: 0.4 },+ { chunk_id: "X#2", page_id: "X", chunk_index: 2, chunk_text: "", heading_path: "", score: 0.4 },+ { chunk_id: "X#3", page_id: "X", chunk_index: 3, chunk_text: "", heading_path: "", score: 0.4 },+ { chunk_id: "X#4", page_id: "X", chunk_index: 4, chunk_text: "", heading_path: "", score: 0.4 },+ { chunk_id: "Y#0", page_id: "Y", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.95 },+ ])+ const out: PageSearchResult[] = await searchByEmbedding("/tmp/p", "q", cfg, 10)+ // X can reach 1.0 but Y is 0.95 — both are valid cap outcomes; the+ // key assertion is that X does NOT score 0.6 + 0.48 = 1.08 (uncapped+ // math), which the cap is designed to prevent.+ const x = out.find((p) => p.id === "X")!+ expect(x.score).toBeLessThanOrEqual(1.0 + 1e-6)+ })++ it("attaches up to 3 matched chunks with metadata", async () => {+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))+ mockInvoke.mockResolvedValueOnce([+ { chunk_id: "A#0", page_id: "A", chunk_index: 0, chunk_text: "first", heading_path: "## Intro", score: 0.9 },+ { chunk_id: "A#1", page_id: "A", chunk_index: 1, chunk_text: "second", heading_path: "## Body", score: 0.6 },+ { chunk_id: "A#2", page_id: "A", chunk_index: 2, chunk_text: "third", heading_path: "## Body", score: 0.4 },+ { chunk_id: "A#3", page_id: "A", chunk_index: 3, chunk_text: "fourth", heading_path: "## Body", score: 0.3 },+ ])+ const out = await searchByEmbedding("/tmp/p", "q", cfg, 10)+ const a = out[0]+ expect(a.matchedChunks).toHaveLength(3)+ expect(a.matchedChunks![0].text).toBe("first")+ expect(a.matchedChunks![0].headingPath).toBe("## Intro")+ expect(a.matchedChunks![0].score).toBeCloseTo(0.9, 5)+ })++ it("respects the topK cutoff", async () => {+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))+ mockInvoke.mockResolvedValueOnce(+ Array.from({ length: 10 }, (_, i) => ({+ chunk_id: `${i}#0`,+ page_id: `page-${i}`,+ chunk_index: 0,+ chunk_text: "",+ heading_path: "",+ score: 1 - i * 0.05,+ })),+ )+ const out = await searchByEmbedding("/tmp/p", "q", cfg, 3)+ expect(out).toHaveLength(3)+ // Should be the three highest-scoring page ids.+ expect(out.map((p) => p.id)).toEqual(["page-0", "page-1", "page-2"])+ })+})++// ── fetchEmbedding auto-halve — via embedPage/searchByEmbedding ─────++describe("fetchEmbedding (via searchByEmbedding) — auto-halve", () => {+ it("retries after an oversize 400 with halved text and succeeds", async () => {+ const responses = [oversizeErrorResponse(400), okResponse([0.1, 0.2])]+ let call = 0+ mockHttpFetch.mockImplementation(async () => responses[call++] ?? okResponse([0]))+ mockInvoke.mockResolvedValueOnce([+ { chunk_id: "P#0", page_id: "P", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.5 },+ ])++ const out = await searchByEmbedding("/tmp/p", "a".repeat(2000), cfg, 5)+ expect(out.map((p) => p.id)).toEqual(["P"])+ // First call with 2000 chars, second with 1000 chars.+ expect(mockHttpFetch).toHaveBeenCalledTimes(2)+ const firstBody = JSON.parse((mockHttpFetch.mock.calls[0][1] as RequestInit).body as string)+ const secondBody = JSON.parse((mockHttpFetch.mock.calls[1][1] as RequestInit).body as string)+ expect(firstBody.input.length).toBe(2000)+ expect(secondBody.input.length).toBe(1000)+ })++ it("recognises HTTP 413 as oversize and halves", async () => {+ mockHttpFetch+ .mockResolvedValueOnce(new Response("", { status: 413, statusText: "Payload Too Large" }))+ .mockResolvedValueOnce(okResponse([0.1]))+ mockInvoke.mockResolvedValueOnce([+ { chunk_id: "P#0", page_id: "P", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.5 },+ ])+ await searchByEmbedding("/tmp/p", "a".repeat(500), cfg, 5)+ expect(mockHttpFetch).toHaveBeenCalledTimes(2)+ })++ it("does NOT halve on auth errors (401) — retries there would be pointless", async () => {+ mockHttpFetch.mockResolvedValueOnce(+ new Response(JSON.stringify({ error: "Invalid API key" }), { status: 401, statusText: "Unauthorized" }),+ )+ const out = await searchByEmbedding("/tmp/p", "hello", cfg, 5)+ expect(out).toEqual([])+ expect(mockHttpFetch).toHaveBeenCalledTimes(1)+ expect(getLastEmbeddingError()).toMatch(/401/)+ })++ it("gives up after 3 halvings and surfaces a specific error", async () => {+ // A fresh Response instance per call — `mockResolvedValue` returns+ // the same object repeatedly and Response bodies can only be+ // consumed once, which would mask the real retry count.+ mockHttpFetch.mockImplementation(async () => oversizeErrorResponse(400))+ const out = await searchByEmbedding("/tmp/p", "a".repeat(2000), cfg, 5)+ expect(out).toEqual([])+ // Initial call + 3 retries at halved sizes = 4 attempts.+ expect(mockHttpFetch).toHaveBeenCalledTimes(4)+ // The final error message should name a concrete smallest size+ // attempted so the user knows how low the server's context is.+ expect(getLastEmbeddingError()).toMatch(/chars/)+ })++ it("stops halving once text is short enough that retry is pointless", async () => {+ // With a 100-char input and the 64-char floor, only one retry at+ // 50 chars is possible before hitting the floor.+ mockHttpFetch.mockImplementation(async () => oversizeErrorResponse(400))+ await searchByEmbedding("/tmp/p", "a".repeat(100), cfg, 5)+ // Attempts: 100 → 50 (still > 64? no, 50 < 64 so no retry) → done.+ // Actually 100 is > 64, halved to 50, then 50 is not > 64, so no+ // further halving. But the current attempt still issues the+ // 50-char request, which also errors. So 2 calls total.+ expect(mockHttpFetch).toHaveBeenCalledTimes(2)+ })+})++// ── embedPage — replaces page's chunks in LanceDB ──────────────────++describe("embedPage", () => {+ it("chunks the page, embeds each, and upserts", async () => {+ // Fresh Response per call — Response body streams can't be+ // double-consumed.+ mockHttpFetch.mockImplementation(async () => okResponse([0.1, 0.2, 0.3]))++ // Short page → likely one chunk under default opts.+ await embedPage(+ "/tmp/p",+ "rope",+ "RoPE",+ "# RoPE\n\nRotary positional embeddings are a positional-encoding scheme.",+ cfg,+ )++ expect(mockInvoke).toHaveBeenCalledTimes(1)+ const [cmd, args] = mockInvoke.mock.calls[0]+ expect(cmd).toBe("vector_upsert_chunks")+ // TS→Rust conversion happens at the Tauri boundary — from the TS+ // side the arg object keeps its camelCase keys.+ const payload = args as { pageId: string; chunks: Array<{ chunk_index: number; embedding: number[] }> }+ expect(payload.pageId).toBe("rope")+ expect(payload.chunks.length).toBeGreaterThan(0)+ // embedding.ts rounds every float to f32 via Math.fround before+ // sending to Rust, which drifts the IEEE-754 representation+ // slightly — compare element-wise with a tolerance instead of+ // strict equality.+ const emb = payload.chunks[0].embedding+ expect(emb).toHaveLength(3)+ expect(emb[0]).toBeCloseTo(0.1, 5)+ expect(emb[1]).toBeCloseTo(0.2, 5)+ expect(emb[2]).toBeCloseTo(0.3, 5)+ })++ it("skips LanceDB call when all chunks fail to embed", async () => {+ mockHttpFetch.mockImplementation(async () => genericErrorResponse(500, "internal"))+ await embedPage("/tmp/p", "rope", "RoPE", "# RoPE\n\nbody text", cfg)+ expect(mockInvoke).not.toHaveBeenCalled()+ })++ it("no-ops when embedding is disabled in config", async () => {+ const disabled = { ...cfg, enabled: false }+ await embedPage("/tmp/p", "rope", "RoPE", "body", disabled)+ expect(mockHttpFetch).not.toHaveBeenCalled()+ expect(mockInvoke).not.toHaveBeenCalled()+ })++ it("no-ops when the content produces zero chunks (empty page)", async () => {+ await embedPage("/tmp/p", "rope", "RoPE", " ", cfg)+ expect(mockHttpFetch).not.toHaveBeenCalled()+ expect(mockInvoke).not.toHaveBeenCalled()+ })++ it("passes page title + heading path + chunk text to the embed request", async () => {+ mockHttpFetch.mockResolvedValue(okResponse([0.5]))++ await embedPage(+ "/tmp/p",+ "attention",+ "Attention Mechanism",+ "## Intro\n\nCore concept of Transformers.",+ cfg,+ )++ expect(mockHttpFetch).toHaveBeenCalled()+ const body = JSON.parse((mockHttpFetch.mock.calls[0][1] as RequestInit).body as string)+ expect(body.input).toContain("Attention Mechanism")+ expect(body.input).toContain("## Intro")+ expect(body.input).toContain("Core concept")+ })+})diff --git a/src/lib/embedding.ts b/src/lib/embedding.tsindex 96e88edd..393562f2 100644--- a/src/lib/embedding.ts+++ b/src/lib/embedding.ts@@ -1,17 +1,40 @@+/**+ * Embedding pipeline — standard RAG flow.+ *+ * 1. chunkMarkdown(content) (src/lib/text-chunker.ts)+ * 2. for each chunk:+ * fetchEmbedding(title + heading_path + chunk_text)+ * with auto-halve retry on "input too long" errors+ * 3. vector_upsert_chunks(page_id, [{chunk_index, chunk_text,+ * heading_path, embedding}, …])+ *+ * Search:+ * 1. fetchEmbedding(query)+ * 2. vector_search_chunks(query_emb, topK × 3)+ * 3. group by page_id, max-pool primary score + weighted tail sum+ * 4. return top-K pages, outer API-compatible with the old per-page+ * `{id, score}[]` shape; matched chunks available on the+ * optional `matchedChunks` field for future UI surfacing.+ *+ * HTTP goes through the Tauri plugin (`src/lib/tauri-fetch.ts`) so+ * CORS-unfriendly endpoints work the same as the LLM path.+ */+import { readFile, listDirectory } from "@/commands/fs"import { invoke } from "@tauri-apps/api/core"import type { EmbeddingConfig } from "@/stores/wiki-store"import type { FileNode } from "@/types/wiki"import { normalizePath } from "@/lib/path-utils"import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"+import { chunkMarkdown, type Chunk } from "@/lib/text-chunker"-// ── Embedding API ─────────────────────────────────────────────────────────+// ── Error surfacing ──────────────────────────────────────────────────────/**- * Track the most recent embedding failure so the UI / callers can- * surface *why* embedding isn't working instead of silently falling- * back to BM25-only retrieval. Set on every failure, cleared on every- * success. Read-only from outside via `getLastEmbeddingError`.+ * Most recent embedding failure description, so Settings → Embedding+ * can show the user WHY vector search fell back to BM25 instead of+ * silently dropping to keyword match. Cleared on any successful+ * embed.*/let lastEmbeddingError: string | null = null@@ -19,134 +42,288 @@ export function getLastEmbeddingError(): string | null {return lastEmbeddingError}+// ── fetchEmbedding with auto-halve retry ────────────────────────────────++/**+ * Heuristic: does this error response look like an "input too long /+ * exceeds model context / payload too large" rejection? True for all+ * the phrasings we've seen from OpenAI, LM Studio, llama.cpp,+ * Ollama, and Azure. Safer to over-match than under-match — a false+ * positive just means a retry at half size, which will still succeed+ * on a real auth/model-id error (it won't) or just log the same error.+ */+function looksLikeOversizeError(httpStatus: number, body: string): boolean {+ if (httpStatus === 413) return true+ const lower = body.toLowerCase()+ return (+ lower.includes("too long") ||+ lower.includes("maximum context") ||+ lower.includes("max_tokens") ||+ lower.includes("max tokens") ||+ lower.includes("context length") ||+ lower.includes("token limit") ||+ lower.includes("exceeds") ||+ lower.includes("input length")+ )+}++/**+ * POST one embedding request; on an oversize rejection, halve the text+ * and retry up to `maxRetries` times. Returns null on definitive+ * failure (auth, network, dim mismatch, retries exhausted) with a+ * human-readable reason left in `lastEmbeddingError`.+ *+ * The returned vector represents the (possibly truncated) text that+ * actually got through. Chunker config should be tuned to minimise+ * truncation — this is a safety net, not the main line of defence.+ */async function fetchEmbedding(text: string,- embeddingConfig: EmbeddingConfig,+ cfg: EmbeddingConfig,+ maxRetries = 3,): Promise<number[] | null> {- if (!embeddingConfig.endpoint) return null+ if (!cfg.endpoint) return nullconst headers: Record<string, string> = { "Content-Type": "application/json" }- if (embeddingConfig.apiKey) {- headers["Authorization"] = `Bearer ${embeddingConfig.apiKey}`- }+ if (cfg.apiKey) headers.Authorization = `Bearer ${cfg.apiKey}`- try {- // Route through the Tauri HTTP plugin so user-configured embedding- // endpoints — local LM Studio / llama.cpp with `--embedding`, remote- // proxies, enterprise gateways — aren't blocked by the same CORS- // preflight issues the LLM path hit before we migrated it. See- // `src/lib/tauri-fetch.ts` for why the plugin beats browser fetch- // for third-party URLs.- const httpFetch = await getHttpFetch()- const resp = await httpFetch(embeddingConfig.endpoint, {- method: "POST",- headers,- body: JSON.stringify({- model: embeddingConfig.model,- input: text.slice(0, 2000),- }),- })- if (!resp.ok) {- let detail = `${resp.status} ${resp.statusText}`+ let current = text+ let attempts = 0+ while (attempts <= maxRetries) {+ attempts+++ try {+ const httpFetch = await getHttpFetch()+ const resp = await httpFetch(cfg.endpoint, {+ method: "POST",+ headers,+ body: JSON.stringify({ model: cfg.model, input: current }),+ })++ if (resp.ok) {+ const data = await resp.json()+ const embedding = data?.data?.[0]?.embedding ?? null+ if (embedding) {+ lastEmbeddingError = null+ return embedding+ }+ lastEmbeddingError = `Embedding response missing data[0].embedding (got ${JSON.stringify(data).slice(0, 200)})`+ console.warn(`[Embedding] ${lastEmbeddingError}`)+ return null+ }++ // Non-OK: try to read the body for an oversize hint.+ let bodyText = ""try {- const body = await resp.text()- if (body) detail += ` — ${body.slice(0, 200)}`+ bodyText = await resp.text()} catch {- // ignore body read errors+ // ignore — some servers return empty bodies on error+ }++ if (looksLikeOversizeError(resp.status, bodyText)) {+ // Can we still halve-and-retry? Need room on both axes:+ // text not yet at the 64-char floor, and retry budget left.+ if (current.length > 64 && attempts <= maxRetries) {+ const prev = current.length+ current = current.slice(0, Math.floor(current.length / 2))+ console.warn(+ `[Embedding] auto-halving after HTTP ${resp.status} at ${prev} chars → retrying at ${current.length} chars (attempt ${attempts}/${maxRetries + 1})`,+ )+ continue+ }+ // Out of retries on a SERVER-oversize error — give the user a+ // message that names the smallest size that still failed so+ // they can tune Settings → Embedding accordingly.+ lastEmbeddingError = `Endpoint rejected input even at ${current.length} chars — server context smaller than expected. Lower Settings → Embedding → Max Chunk Chars (${bodyText.slice(0, 160)}).`+ console.warn(`[Embedding] ${lastEmbeddingError}`)+ return null}- lastEmbeddingError = `API ${detail} at ${embeddingConfig.endpoint}`++ // Non-oversize definitive failure (auth, rate limit, server down, …).+ lastEmbeddingError = `API ${resp.status} ${resp.statusText}${bodyText ? ` — ${bodyText.slice(0, 200)}` : ""} at ${cfg.endpoint}`console.warn(`[Embedding] ${lastEmbeddingError}`)return null- }- const data = await resp.json()- const embedding = data?.data?.[0]?.embedding ?? null- if (embedding) {- lastEmbeddingError = null- } else {- lastEmbeddingError = `Response did not contain data[0].embedding (got ${JSON.stringify(data).slice(0, 200)})`+ } catch (err) {+ if (isFetchNetworkError(err)) {+ lastEmbeddingError = `Network error reaching ${cfg.endpoint}. Check endpoint URL, API key, and connectivity.`+ } else {+ lastEmbeddingError = err instanceof Error ? err.message : String(err)+ }console.warn(`[Embedding] ${lastEmbeddingError}`)+ return null}- return embedding- } catch (err) {- // Translate cross-webview fetch failures into a single actionable- // message that names the URL, mirroring the LLM-client treatment.- if (isFetchNetworkError(err)) {- lastEmbeddingError = `Network error reaching ${embeddingConfig.endpoint}. Check endpoint URL, API key, and connectivity.`- } else {- lastEmbeddingError = err instanceof Error ? err.message : String(err)- }- console.warn(`[Embedding] ${lastEmbeddingError}`)- return null}++ // Exhausted retries (only reachable if every halving round triggered+ // the retry branch and then the loop condition ended).+ lastEmbeddingError = `Embedding endpoint rejected every size down to ${current.length} chars — the server's context is smaller than ${current.length * 2}. Lower Settings → Embedding → Max Chunk Chars.`+ console.warn(`[Embedding] ${lastEmbeddingError}`)+ return null}-// ── LanceDB operations via Tauri commands ─────────────────────────────────+// ── LanceDB v2 operations (via Rust Tauri commands) ──────────────────────++interface ChunkUpsertInput {+ chunkIndex: number+ chunkText: string+ headingPath: string+ embedding: number[]+}-async function vectorUpsert(projectPath: string, pageId: string, embedding: number[]): Promise<void> {- await invoke("vector_upsert", {+async function vectorUpsertChunks(+ projectPath: string,+ pageId: string,+ chunks: ChunkUpsertInput[],+): Promise<void> {+ await invoke("vector_upsert_chunks", {projectPath: normalizePath(projectPath),pageId,- embedding: embedding.map((v) => Math.fround(v)), // ensure f32+ chunks: chunks.map((c) => ({+ chunk_index: c.chunkIndex,+ chunk_text: c.chunkText,+ heading_path: c.headingPath,+ embedding: c.embedding.map((v) => Math.fround(v)),+ })),})}-async function vectorSearchLance(projectPath: string, queryEmbedding: number[], topK: number): Promise<Array<{ page_id: string; score: number }>> {- return await invoke("vector_search", {+interface ChunkSearchResult {+ chunk_id: string+ page_id: string+ chunk_index: number+ chunk_text: string+ heading_path: string+ score: number+}++async function vectorSearchChunks(+ projectPath: string,+ queryEmbedding: number[],+ topK: number,+): Promise<ChunkSearchResult[]> {+ return await invoke("vector_search_chunks", {projectPath: normalizePath(projectPath),queryEmbedding: queryEmbedding.map((v) => Math.fround(v)),topK,})}-async function vectorDelete(projectPath: string, pageId: string): Promise<void> {- await invoke("vector_delete", {+async function vectorDeletePage(projectPath: string, pageId: string): Promise<void> {+ await invoke("vector_delete_page", {projectPath: normalizePath(projectPath),pageId,})}-async function vectorCount(projectPath: string): Promise<number> {- return await invoke("vector_count", {+async function vectorCountChunks(projectPath: string): Promise<number> {+ return await invoke("vector_count_chunks", {projectPath: normalizePath(projectPath),})}-// ── Public API ────────────────────────────────────────────────────────────+export async function legacyVectorRowCount(projectPath: string): Promise<number> {+ try {+ return await invoke("vector_legacy_row_count", {+ projectPath: normalizePath(projectPath),+ })+ } catch {+ return 0+ }+}++export async function dropLegacyVectorTable(projectPath: string): Promise<void> {+ await invoke("vector_drop_legacy", {+ projectPath: normalizePath(projectPath),+ })+}++// ── Chunk enrichment ─────────────────────────────────────────────────────/**- * Embed a single page and store in LanceDB.- * Called after ingest to keep embeddings up to date.+ * Build the text we actually embed for a chunk: page title + heading+ * breadcrumb + chunk content. The breadcrumb is the most important+ * context for a short chunk — a 300-char excerpt about "Mixture of+ * Experts" is far more findable when the embedded text explicitly+ * names its containing sections.+ */+function enrichChunkForEmbedding(+ pageTitle: string,+ chunk: Chunk,+): string {+ const parts: string[] = []+ if (pageTitle.trim().length > 0) parts.push(pageTitle.trim())+ if (chunk.headingPath.trim().length > 0) parts.push(chunk.headingPath.trim())+ parts.push(chunk.text.trim())+ return parts.join("\n\n")+}++// ── Public API: embedPage / embedAllPages / searchByEmbedding ────────────++/**+ * Embed a wiki page: chunk → per-chunk embed → replace the page's+ * vectors in LanceDB in one batch. Every transient failure leaves the+ * existing v2 rows intact (empty upsert is a no-op Rust-side).+ *+ * Called by ingest.ts after writing a page to disk.*/export async function embedPage(projectPath: string,pageId: string,title: string,content: string,- embeddingConfig: EmbeddingConfig,+ cfg: EmbeddingConfig,): Promise<void> {- if (!embeddingConfig.enabled || !embeddingConfig.model) return+ if (!cfg.enabled || !cfg.model) returnconst t0 = performance.now()- const text = `${title}\n${content.slice(0, 1500)}`- const emb = await fetchEmbedding(text, embeddingConfig)- if (emb) {- await vectorUpsert(projectPath, pageId, emb)- console.log(`[Embedding] Indexed "${pageId}" (${emb.length}d) in ${Math.round(performance.now() - t0)}ms`)- } else {- console.log(`[Embedding] Failed to embed "${pageId}"`)+ const chunks = chunkMarkdown(content, {+ targetChars: cfg.maxChunkChars ?? 1000,+ overlapChars: cfg.overlapChunkChars ?? 200,+ })+ if (chunks.length === 0) return++ const rows: ChunkUpsertInput[] = []+ let failedChunks = 0+ for (const chunk of chunks) {+ const embedText = enrichChunkForEmbedding(title, chunk)+ const vec = await fetchEmbedding(embedText, cfg)+ if (vec) {+ rows.push({+ chunkIndex: chunk.index,+ chunkText: chunk.text,+ headingPath: chunk.headingPath,+ embedding: vec,+ })+ } else {+ failedChunks+++ }+ }++ if (rows.length === 0) {+ console.log(+ `[Embedding] Indexed nothing for "${pageId}" — all ${chunks.length} chunks failed. See getLastEmbeddingError().`,+ )+ return}++ await vectorUpsertChunks(projectPath, pageId, rows)+ const elapsed = Math.round(performance.now() - t0)+ console.log(+ `[Embedding] Indexed "${pageId}": ${rows.length}/${chunks.length} chunks (${failedChunks} skipped) in ${elapsed}ms`,+ )}/**- * Embed all wiki pages that are not yet indexed.- * Called on first enable or when model changes.+ * Embed every wiki content page that isn't already indexed (or re-embed+ * all when `force === true`). Driven from Settings → Embedding or on+ * first enable. Skips structural pages (index / log / overview /+ * purpose / schema) — they're aggregate views, not retrieval targets.*/export async function embedAllPages(projectPath: string,- embeddingConfig: EmbeddingConfig,+ cfg: EmbeddingConfig,onProgress?: (done: number, total: number) => void,): Promise<number> {- if (!embeddingConfig.enabled || !embeddingConfig.model) return 0+ if (!cfg.enabled || !cfg.model) return 0const pp = normalizePath(projectPath)@@ -178,16 +355,10 @@ export async function embedAllPages(const content = await readFile(file.path)const titleMatch = content.match(/^---\n[\s\S]*?^title:\s*["']?(.+?)["']?\s*$/m)const title = titleMatch ? titleMatch[1].trim() : file.id-- const text = `${title}\n${content.slice(0, 1500)}`- const emb = await fetchEmbedding(text, embeddingConfig)- if (emb) {- await vectorUpsert(pp, file.id, emb)- }+ await embedPage(pp, file.id, title, content, cfg)} catch {- // skip+ // skip — individual file failure doesn't halt the batch}-done++if (onProgress) onProgress(done, mdFiles.length)}@@ -196,53 +367,107 @@ export async function embedAllPages(}/**- * Search wiki pages by semantic similarity via LanceDB.- * Returns page IDs sorted by similarity score.+ * Vector search over the v2 chunk store, shaped to stay API-compatible+ * with the pre-0.3.11 per-page interface. Under the hood:+ * 1. Embed the query.+ * 2. Over-fetch top-K × 3 chunks.+ * 3. Group by page_id; score each page as max(chunk_scores) plus+ * 0.3 × sum of the other chunks' scores (bounded — capped at+ * 1.0 - max_score), so a page with two good chunks outranks a+ * page with one equally-good chunk and a weaker one.+ * 4. Sort pages by score, return top-K.+ *+ * The optional `matchedChunks` field gives callers the raw chunk+ * context when they want to surface "matched in this section" in+ * the UI. Existing callers can ignore it.*/+export interface PageSearchResult {+ id: string+ score: number+ matchedChunks?: Array<{ text: string; headingPath: string; score: number }>+}+export async function searchByEmbedding(projectPath: string,query: string,- embeddingConfig: EmbeddingConfig,+ cfg: EmbeddingConfig,topK: number = 10,-): Promise<Array<{ id: string; score: number }>> {- if (!embeddingConfig.enabled || !embeddingConfig.model) return []+): Promise<PageSearchResult[]> {+ if (!cfg.enabled || !cfg.model) return []- const queryEmb = await fetchEmbedding(query, embeddingConfig)+ const queryEmb = await fetchEmbedding(query, cfg)if (!queryEmb) return []+ const t0 = performance.now()+ let rawChunks: ChunkSearchResult[] = []try {- const t0 = performance.now()- const results = await vectorSearchLance(projectPath, queryEmb, topK)- console.log(`[Embedding] LanceDB search: ${results.length} results in ${Math.round(performance.now() - t0)}ms`)- return results.map((r) => ({ id: r.page_id, score: r.score }))+ rawChunks = await vectorSearchChunks(projectPath, queryEmb, Math.max(topK * 3, 30))} catch (err) {- console.log(`[Embedding] LanceDB search failed: ${err instanceof Error ? err.message : err}`)+ console.log(`[Embedding] LanceDB chunk search failed: ${err instanceof Error ? err.message : err}`)return []}+ if (rawChunks.length === 0) return []++ // Group by page; keep every matched chunk's score so we can compute+ // a blended per-page score.+ const byPage = new Map<string, ChunkSearchResult[]>()+ for (const c of rawChunks) {+ const bucket = byPage.get(c.page_id)+ if (bucket) bucket.push(c)+ else byPage.set(c.page_id, [c])+ }++ const ranked: PageSearchResult[] = []+ for (const [pageId, chunks] of byPage.entries()) {+ chunks.sort((a, b) => b.score - a.score)+ const top = chunks[0].score+ const tail = chunks.slice(1).reduce((sum, c) => sum + c.score, 0)+ // Cap the tail contribution so many-weak-chunks can't drown a+ // single-strong-chunk page. 0.3 weight is empirical; adjust later+ // with real data.+ const blended = top + Math.min(tail * 0.3, Math.max(0, 1 - top))+ ranked.push({+ id: pageId,+ score: blended,+ matchedChunks: chunks.slice(0, 3).map((c) => ({+ text: c.chunk_text,+ headingPath: c.heading_path,+ score: c.score,+ })),+ })+ }+ ranked.sort((a, b) => b.score - a.score)++ const elapsed = Math.round(performance.now() - t0)+ console.log(+ `[Embedding] LanceDB chunk search: ${rawChunks.length} chunks → ${ranked.length} pages in ${elapsed}ms`,+ )++ return ranked.slice(0, topK)}/**- * Remove a page from the vector index.+ * Remove a page's embeddings from the v2 index. Called from the+ * source-delete flow so orphaned chunks don't pollute future searches.*/export async function removePageEmbedding(projectPath: string,pageId: string,): Promise<void> {try {- await vectorDelete(projectPath, pageId)+ await vectorDeletePage(projectPath, pageId)} catch {// non-critical}}/**- * Get the number of indexed vectors.+ * Total chunks in the v2 index. Surfaces "N chunks indexed" status+ * in Settings.*/-export async function getEmbeddingCount(- projectPath: string,-): Promise<number> {+export async function getEmbeddingCount(projectPath: string): Promise<number> {try {- return await vectorCount(projectPath)+ return await vectorCountChunks(projectPath)} catch {return 0}diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.tsindex a5da187d..257a3f44 100644--- a/src/stores/wiki-store.ts+++ b/src/stores/wiki-store.ts@@ -29,6 +29,21 @@ interface EmbeddingConfig {endpoint: string // e.g. "http://127.0.0.1:1234/v1/embeddings"apiKey: stringmodel: string // e.g. "text-embedding-qwen3-embedding-0.6b"+ /**+ * Chunking knobs (Phase 1 RAG). Undefined values fall back to the+ * chunker's built-in defaults in `src/lib/text-chunker.ts`:+ * targetChars 1000+ * maxChars 1500+ * minChars 200+ * overlapChars 200+ *+ * Users on small-context endpoints (e.g. llama.cpp with n_ctx=512,+ * Ollama `mxbai-embed-large`) should lower `maxChunkChars` to avoid+ * per-request rejections; fetchEmbedding also auto-halves on+ * "too long" server errors as a second line of defence.+ */+ maxChunkChars?: number+ overlapChunkChars?: number}/**