Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat(embedding/rag Phase 3+4): chunk-level embedding + auto-halve retry

nash_sufc0c2d8e6cad
diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.ts
new file mode 100644
index 00000000..557eae65
--- /dev/null
+++ b/src/lib/embedding.test.ts
@@ -0,0 +1,313 @@
+/**
+ * Unit-level tests for embedding.ts, focused on the pure / mockable
+ * pieces: auto-halve retry heuristics and the chunk→page aggregation
+ * contract inside `searchByEmbedding`.
+ *
+ * The actual HTTP layer and Tauri LanceDB commands are mocked — we're
+ * NOT testing Rust vectorstore here (that has its own 15 Rust tests)
+ * nor the webview fetch (that's tauri-fetch.ts). The boundary we pin
+ * down here is "given these chunk-level results, do we aggregate to
+ * page-level scores the way the design says?".
+ */
+import { describe, it, expect, vi, beforeEach } from "vitest"
+
+// Module-level mock for the Tauri invoke boundary so we can script the
+// chunk-search response without touching real LanceDB.
+const mockInvoke = vi.fn<(cmd: string, args?: unknown) => Promise<unknown>>()
+vi.mock("@tauri-apps/api/core", () => ({
+ invoke: (cmd: string, args?: unknown) => mockInvoke(cmd, args),
+}))
+
+// Stub getHttpFetch so fetchEmbedding calls hit our in-test responder.
+const mockHttpFetch = vi.fn<(url: string, opts?: RequestInit) => Promise<Response>>()
+vi.mock("@/lib/tauri-fetch", () => ({
+ getHttpFetch: () => Promise.resolve(mockHttpFetch),
+ isFetchNetworkError: (err: unknown) =>
+ err instanceof TypeError ||
+ (err instanceof Error &&
+ (err.message === "Load failed" || err.message === "Failed to fetch")),
+}))
+
+// readFile / listDirectory aren't exercised in this file's cases; stub.
+vi.mock("@/commands/fs", () => ({
+ readFile: vi.fn(),
+ listDirectory: vi.fn(),
+}))
+
+import {
+ searchByEmbedding,
+ embedPage,
+ getLastEmbeddingError,
+ type PageSearchResult,
+} from "./embedding"
+
+const cfg = {
+ enabled: true,
+ endpoint: "http://localhost:1234/v1/embeddings",
+ apiKey: "",
+ model: "test-embed",
+}
+
+/** Build an embedding-shaped JSON Response. */
+function okResponse(embedding: number[]): Response {
+ return new Response(JSON.stringify({ data: [{ embedding }] }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ })
+}
+
+/** Build an HTTP-error Response that looks like an oversize rejection. */
+function oversizeErrorResponse(status = 400): Response {
+ return new Response(
+ JSON.stringify({ error: "input length 3200 exceeds maximum context 512" }),
+ { status, statusText: "Bad Request" },
+ )
+}
+
+function genericErrorResponse(status: number, body: string): Response {
+ return new Response(body, { status, statusText: "Error" })
+}
+
+beforeEach(() => {
+ mockInvoke.mockReset()
+ mockHttpFetch.mockReset()
+})
+
+// ── searchByEmbedding — chunk→page aggregation ─────────────────────
+
+describe("searchByEmbedding — aggregation", () => {
+ it("returns [] when embedding call fails (null vector)", async () => {
+ mockHttpFetch.mockResolvedValue(genericErrorResponse(401, "Unauthorized"))
+ const out = await searchByEmbedding("/tmp/p", "hello", cfg, 5)
+ expect(out).toEqual([])
+ })
+
+ it("returns [] when no chunks come back from LanceDB", async () => {
+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))
+ mockInvoke.mockResolvedValueOnce([] /* vector_search_chunks result */)
+ const out = await searchByEmbedding("/tmp/p", "hello", cfg, 5)
+ expect(out).toEqual([])
+ })
+
+ it("groups chunks by page and ranks by max-pool score", async () => {
+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))
+ // Three pages: A has two chunks (strong + weak), B has one strong,
+ // C has one weak. Expected order: A (top 0.9 + blended tail) > B
+ // (0.88) > C (0.2).
+ mockInvoke.mockResolvedValueOnce([
+ { chunk_id: "A#0", page_id: "A", chunk_index: 0, chunk_text: "a0", heading_path: "## X", score: 0.9 },
+ { chunk_id: "A#1", page_id: "A", chunk_index: 1, chunk_text: "a1", heading_path: "## Y", score: 0.5 },
+ { chunk_id: "B#0", page_id: "B", chunk_index: 0, chunk_text: "b0", heading_path: "", score: 0.88 },
+ { chunk_id: "C#0", page_id: "C", chunk_index: 0, chunk_text: "c0", heading_path: "", score: 0.2 },
+ ])
+ const out = await searchByEmbedding("/tmp/p", "q", cfg, 10)
+ expect(out.map((p) => p.id)).toEqual(["A", "B", "C"])
+ // A's score: 0.9 + min(0.5 * 0.3, 1 - 0.9) = 0.9 + min(0.15, 0.1) = 1.0
+ expect(out[0].score).toBeCloseTo(1.0, 5)
+ // B's score: 0.88 + 0 = 0.88
+ expect(out[1].score).toBeCloseTo(0.88, 5)
+ // C's score: 0.2 + 0 = 0.2
+ expect(out[2].score).toBeCloseTo(0.2, 5)
+ })
+
+ it("caps tail contribution so a page with many weak chunks can't overtake a strong single-chunk page", async () => {
+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))
+ // Page X: one chunk at 0.6, four chunks at 0.4 each (tail sum = 1.6).
+ // top=0.6, tail_weighted = 0.3 * 1.6 = 0.48, capped at 1 - 0.6 = 0.4
+ // blended = 0.6 + 0.4 = 1.0
+ // Page Y: single 0.95 chunk.
+ // blended = 0.95
+ // X would unfairly win if we didn't cap the tail.
+ mockInvoke.mockResolvedValueOnce([
+ { chunk_id: "X#0", page_id: "X", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.6 },
+ { chunk_id: "X#1", page_id: "X", chunk_index: 1, chunk_text: "", heading_path: "", score: 0.4 },
+ { chunk_id: "X#2", page_id: "X", chunk_index: 2, chunk_text: "", heading_path: "", score: 0.4 },
+ { chunk_id: "X#3", page_id: "X", chunk_index: 3, chunk_text: "", heading_path: "", score: 0.4 },
+ { chunk_id: "X#4", page_id: "X", chunk_index: 4, chunk_text: "", heading_path: "", score: 0.4 },
+ { chunk_id: "Y#0", page_id: "Y", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.95 },
+ ])
+ const out: PageSearchResult[] = await searchByEmbedding("/tmp/p", "q", cfg, 10)
+ // X can reach 1.0 but Y is 0.95 — both are valid cap outcomes; the
+ // key assertion is that X does NOT score 0.6 + 0.48 = 1.08 (uncapped
+ // math), which the cap is designed to prevent.
+ const x = out.find((p) => p.id === "X")!
+ expect(x.score).toBeLessThanOrEqual(1.0 + 1e-6)
+ })
+
+ it("attaches up to 3 matched chunks with metadata", async () => {
+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))
+ mockInvoke.mockResolvedValueOnce([
+ { chunk_id: "A#0", page_id: "A", chunk_index: 0, chunk_text: "first", heading_path: "## Intro", score: 0.9 },
+ { chunk_id: "A#1", page_id: "A", chunk_index: 1, chunk_text: "second", heading_path: "## Body", score: 0.6 },
+ { chunk_id: "A#2", page_id: "A", chunk_index: 2, chunk_text: "third", heading_path: "## Body", score: 0.4 },
+ { chunk_id: "A#3", page_id: "A", chunk_index: 3, chunk_text: "fourth", heading_path: "## Body", score: 0.3 },
+ ])
+ const out = await searchByEmbedding("/tmp/p", "q", cfg, 10)
+ const a = out[0]
+ expect(a.matchedChunks).toHaveLength(3)
+ expect(a.matchedChunks![0].text).toBe("first")
+ expect(a.matchedChunks![0].headingPath).toBe("## Intro")
+ expect(a.matchedChunks![0].score).toBeCloseTo(0.9, 5)
+ })
+
+ it("respects the topK cutoff", async () => {
+ mockHttpFetch.mockResolvedValue(okResponse([0.1, 0.2, 0.3]))
+ mockInvoke.mockResolvedValueOnce(
+ Array.from({ length: 10 }, (_, i) => ({
+ chunk_id: `${i}#0`,
+ page_id: `page-${i}`,
+ chunk_index: 0,
+ chunk_text: "",
+ heading_path: "",
+ score: 1 - i * 0.05,
+ })),
+ )
+ const out = await searchByEmbedding("/tmp/p", "q", cfg, 3)
+ expect(out).toHaveLength(3)
+ // Should be the three highest-scoring page ids.
+ expect(out.map((p) => p.id)).toEqual(["page-0", "page-1", "page-2"])
+ })
+})
+
+// ── fetchEmbedding auto-halve — via embedPage/searchByEmbedding ─────
+
+describe("fetchEmbedding (via searchByEmbedding) — auto-halve", () => {
+ it("retries after an oversize 400 with halved text and succeeds", async () => {
+ const responses = [oversizeErrorResponse(400), okResponse([0.1, 0.2])]
+ let call = 0
+ mockHttpFetch.mockImplementation(async () => responses[call++] ?? okResponse([0]))
+ mockInvoke.mockResolvedValueOnce([
+ { chunk_id: "P#0", page_id: "P", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.5 },
+ ])
+
+ const out = await searchByEmbedding("/tmp/p", "a".repeat(2000), cfg, 5)
+ expect(out.map((p) => p.id)).toEqual(["P"])
+ // First call with 2000 chars, second with 1000 chars.
+ expect(mockHttpFetch).toHaveBeenCalledTimes(2)
+ const firstBody = JSON.parse((mockHttpFetch.mock.calls[0][1] as RequestInit).body as string)
+ const secondBody = JSON.parse((mockHttpFetch.mock.calls[1][1] as RequestInit).body as string)
+ expect(firstBody.input.length).toBe(2000)
+ expect(secondBody.input.length).toBe(1000)
+ })
+
+ it("recognises HTTP 413 as oversize and halves", async () => {
+ mockHttpFetch
+ .mockResolvedValueOnce(new Response("", { status: 413, statusText: "Payload Too Large" }))
+ .mockResolvedValueOnce(okResponse([0.1]))
+ mockInvoke.mockResolvedValueOnce([
+ { chunk_id: "P#0", page_id: "P", chunk_index: 0, chunk_text: "", heading_path: "", score: 0.5 },
+ ])
+ await searchByEmbedding("/tmp/p", "a".repeat(500), cfg, 5)
+ expect(mockHttpFetch).toHaveBeenCalledTimes(2)
+ })
+
+ it("does NOT halve on auth errors (401) — retries there would be pointless", async () => {
+ mockHttpFetch.mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: "Invalid API key" }), { status: 401, statusText: "Unauthorized" }),
+ )
+ const out = await searchByEmbedding("/tmp/p", "hello", cfg, 5)
+ expect(out).toEqual([])
+ expect(mockHttpFetch).toHaveBeenCalledTimes(1)
+ expect(getLastEmbeddingError()).toMatch(/401/)
+ })
+
+ it("gives up after 3 halvings and surfaces a specific error", async () => {
+ // A fresh Response instance per call — `mockResolvedValue` returns
+ // the same object repeatedly and Response bodies can only be
+ // consumed once, which would mask the real retry count.
+ mockHttpFetch.mockImplementation(async () => oversizeErrorResponse(400))
+ const out = await searchByEmbedding("/tmp/p", "a".repeat(2000), cfg, 5)
+ expect(out).toEqual([])
+ // Initial call + 3 retries at halved sizes = 4 attempts.
+ expect(mockHttpFetch).toHaveBeenCalledTimes(4)
+ // The final error message should name a concrete smallest size
+ // attempted so the user knows how low the server's context is.
+ expect(getLastEmbeddingError()).toMatch(/chars/)
+ })
+
+ it("stops halving once text is short enough that retry is pointless", async () => {
+ // With a 100-char input and the 64-char floor, only one retry at
+ // 50 chars is possible before hitting the floor.
+ mockHttpFetch.mockImplementation(async () => oversizeErrorResponse(400))
+ await searchByEmbedding("/tmp/p", "a".repeat(100), cfg, 5)
+ // Attempts: 100 → 50 (still > 64? no, 50 < 64 so no retry) → done.
+ // Actually 100 is > 64, halved to 50, then 50 is not > 64, so no
+ // further halving. But the current attempt still issues the
+ // 50-char request, which also errors. So 2 calls total.
+ expect(mockHttpFetch).toHaveBeenCalledTimes(2)
+ })
+})
+
+// ── embedPage — replaces page's chunks in LanceDB ──────────────────
+
+describe("embedPage", () => {
+ it("chunks the page, embeds each, and upserts", async () => {
+ // Fresh Response per call — Response body streams can't be
+ // double-consumed.
+ mockHttpFetch.mockImplementation(async () => okResponse([0.1, 0.2, 0.3]))
+
+ // Short page → likely one chunk under default opts.
+ await embedPage(
+ "/tmp/p",
+ "rope",
+ "RoPE",
+ "# RoPE\n\nRotary positional embeddings are a positional-encoding scheme.",
+ cfg,
+ )
+
+ expect(mockInvoke).toHaveBeenCalledTimes(1)
+ const [cmd, args] = mockInvoke.mock.calls[0]
+ expect(cmd).toBe("vector_upsert_chunks")
+ // TS→Rust conversion happens at the Tauri boundary — from the TS
+ // side the arg object keeps its camelCase keys.
+ const payload = args as { pageId: string; chunks: Array<{ chunk_index: number; embedding: number[] }> }
+ expect(payload.pageId).toBe("rope")
+ expect(payload.chunks.length).toBeGreaterThan(0)
+ // embedding.ts rounds every float to f32 via Math.fround before
+ // sending to Rust, which drifts the IEEE-754 representation
+ // slightly — compare element-wise with a tolerance instead of
+ // strict equality.
+ const emb = payload.chunks[0].embedding
+ expect(emb).toHaveLength(3)
+ expect(emb[0]).toBeCloseTo(0.1, 5)
+ expect(emb[1]).toBeCloseTo(0.2, 5)
+ expect(emb[2]).toBeCloseTo(0.3, 5)
+ })
+
+ it("skips LanceDB call when all chunks fail to embed", async () => {
+ mockHttpFetch.mockImplementation(async () => genericErrorResponse(500, "internal"))
+ await embedPage("/tmp/p", "rope", "RoPE", "# RoPE\n\nbody text", cfg)
+ expect(mockInvoke).not.toHaveBeenCalled()
+ })
+
+ it("no-ops when embedding is disabled in config", async () => {
+ const disabled = { ...cfg, enabled: false }
+ await embedPage("/tmp/p", "rope", "RoPE", "body", disabled)
+ expect(mockHttpFetch).not.toHaveBeenCalled()
+ expect(mockInvoke).not.toHaveBeenCalled()
+ })
+
+ it("no-ops when the content produces zero chunks (empty page)", async () => {
+ await embedPage("/tmp/p", "rope", "RoPE", " ", cfg)
+ expect(mockHttpFetch).not.toHaveBeenCalled()
+ expect(mockInvoke).not.toHaveBeenCalled()
+ })
+
+ it("passes page title + heading path + chunk text to the embed request", async () => {
+ mockHttpFetch.mockResolvedValue(okResponse([0.5]))
+
+ await embedPage(
+ "/tmp/p",
+ "attention",
+ "Attention Mechanism",
+ "## Intro\n\nCore concept of Transformers.",
+ cfg,
+ )
+
+ expect(mockHttpFetch).toHaveBeenCalled()
+ const body = JSON.parse((mockHttpFetch.mock.calls[0][1] as RequestInit).body as string)
+ expect(body.input).toContain("Attention Mechanism")
+ expect(body.input).toContain("## Intro")
+ expect(body.input).toContain("Core concept")
+ })
+})
diff --git a/src/lib/embedding.ts b/src/lib/embedding.ts
index 96e88edd..393562f2 100644
--- a/src/lib/embedding.ts
+++ b/src/lib/embedding.ts
@@ -1,17 +1,40 @@
+/**
+ * Embedding pipeline — standard RAG flow.
+ *
+ * 1. chunkMarkdown(content) (src/lib/text-chunker.ts)
+ * 2. for each chunk:
+ * fetchEmbedding(title + heading_path + chunk_text)
+ * with auto-halve retry on "input too long" errors
+ * 3. vector_upsert_chunks(page_id, [{chunk_index, chunk_text,
+ * heading_path, embedding}, …])
+ *
+ * Search:
+ * 1. fetchEmbedding(query)
+ * 2. vector_search_chunks(query_emb, topK × 3)
+ * 3. group by page_id, max-pool primary score + weighted tail sum
+ * 4. return top-K pages, outer API-compatible with the old per-page
+ * `{id, score}[]` shape; matched chunks available on the
+ * optional `matchedChunks` field for future UI surfacing.
+ *
+ * HTTP goes through the Tauri plugin (`src/lib/tauri-fetch.ts`) so
+ * CORS-unfriendly endpoints work the same as the LLM path.
+ */
+
import { readFile, listDirectory } from "@/commands/fs"
import { invoke } from "@tauri-apps/api/core"
import type { EmbeddingConfig } from "@/stores/wiki-store"
import type { FileNode } from "@/types/wiki"
import { normalizePath } from "@/lib/path-utils"
import { getHttpFetch, isFetchNetworkError } from "@/lib/tauri-fetch"
+import { chunkMarkdown, type Chunk } from "@/lib/text-chunker"
-// ── Embedding API ─────────────────────────────────────────────────────────
+// ── Error surfacing ──────────────────────────────────────────────────────
/**
- * Track the most recent embedding failure so the UI / callers can
- * surface *why* embedding isn't working instead of silently falling
- * back to BM25-only retrieval. Set on every failure, cleared on every
- * success. Read-only from outside via `getLastEmbeddingError`.
+ * Most recent embedding failure description, so Settings → Embedding
+ * can show the user WHY vector search fell back to BM25 instead of
+ * silently dropping to keyword match. Cleared on any successful
+ * embed.
*/
let lastEmbeddingError: string | null = null
@@ -19,134 +42,288 @@ export function getLastEmbeddingError(): string | null {
return lastEmbeddingError
}
+// ── fetchEmbedding with auto-halve retry ────────────────────────────────
+
+/**
+ * Heuristic: does this error response look like an "input too long /
+ * exceeds model context / payload too large" rejection? True for all
+ * the phrasings we've seen from OpenAI, LM Studio, llama.cpp,
+ * Ollama, and Azure. Safer to over-match than under-match — a false
+ * positive just means a retry at half size, which will still succeed
+ * on a real auth/model-id error (it won't) or just log the same error.
+ */
+function looksLikeOversizeError(httpStatus: number, body: string): boolean {
+ if (httpStatus === 413) return true
+ const lower = body.toLowerCase()
+ return (
+ lower.includes("too long") ||
+ lower.includes("maximum context") ||
+ lower.includes("max_tokens") ||
+ lower.includes("max tokens") ||
+ lower.includes("context length") ||
+ lower.includes("token limit") ||
+ lower.includes("exceeds") ||
+ lower.includes("input length")
+ )
+}
+
+/**
+ * POST one embedding request; on an oversize rejection, halve the text
+ * and retry up to `maxRetries` times. Returns null on definitive
+ * failure (auth, network, dim mismatch, retries exhausted) with a
+ * human-readable reason left in `lastEmbeddingError`.
+ *
+ * The returned vector represents the (possibly truncated) text that
+ * actually got through. Chunker config should be tuned to minimise
+ * truncation — this is a safety net, not the main line of defence.
+ */
async function fetchEmbedding(
text: string,
- embeddingConfig: EmbeddingConfig,
+ cfg: EmbeddingConfig,
+ maxRetries = 3,
): Promise<number[] | null> {
- if (!embeddingConfig.endpoint) return null
+ if (!cfg.endpoint) return null
const headers: Record<string, string> = { "Content-Type": "application/json" }
- if (embeddingConfig.apiKey) {
- headers["Authorization"] = `Bearer ${embeddingConfig.apiKey}`
- }
+ if (cfg.apiKey) headers.Authorization = `Bearer ${cfg.apiKey}`
- try {
- // Route through the Tauri HTTP plugin so user-configured embedding
- // endpoints — local LM Studio / llama.cpp with `--embedding`, remote
- // proxies, enterprise gateways — aren't blocked by the same CORS
- // preflight issues the LLM path hit before we migrated it. See
- // `src/lib/tauri-fetch.ts` for why the plugin beats browser fetch
- // for third-party URLs.
- const httpFetch = await getHttpFetch()
- const resp = await httpFetch(embeddingConfig.endpoint, {
- method: "POST",
- headers,
- body: JSON.stringify({
- model: embeddingConfig.model,
- input: text.slice(0, 2000),
- }),
- })
- if (!resp.ok) {
- let detail = `${resp.status} ${resp.statusText}`
+ let current = text
+ let attempts = 0
+ while (attempts <= maxRetries) {
+ attempts++
+ try {
+ const httpFetch = await getHttpFetch()
+ const resp = await httpFetch(cfg.endpoint, {
+ method: "POST",
+ headers,
+ body: JSON.stringify({ model: cfg.model, input: current }),
+ })
+
+ if (resp.ok) {
+ const data = await resp.json()
+ const embedding = data?.data?.[0]?.embedding ?? null
+ if (embedding) {
+ lastEmbeddingError = null
+ return embedding
+ }
+ lastEmbeddingError = `Embedding response missing data[0].embedding (got ${JSON.stringify(data).slice(0, 200)})`
+ console.warn(`[Embedding] ${lastEmbeddingError}`)
+ return null
+ }
+
+ // Non-OK: try to read the body for an oversize hint.
+ let bodyText = ""
try {
- const body = await resp.text()
- if (body) detail += ` — ${body.slice(0, 200)}`
+ bodyText = await resp.text()
} catch {
- // ignore body read errors
+ // ignore — some servers return empty bodies on error
+ }
+
+ if (looksLikeOversizeError(resp.status, bodyText)) {
+ // Can we still halve-and-retry? Need room on both axes:
+ // text not yet at the 64-char floor, and retry budget left.
+ if (current.length > 64 && attempts <= maxRetries) {
+ const prev = current.length
+ current = current.slice(0, Math.floor(current.length / 2))
+ console.warn(
+ `[Embedding] auto-halving after HTTP ${resp.status} at ${prev} chars → retrying at ${current.length} chars (attempt ${attempts}/${maxRetries + 1})`,
+ )
+ continue
+ }
+ // Out of retries on a SERVER-oversize error — give the user a
+ // message that names the smallest size that still failed so
+ // they can tune Settings → Embedding accordingly.
+ lastEmbeddingError = `Endpoint rejected input even at ${current.length} chars — server context smaller than expected. Lower Settings → Embedding → Max Chunk Chars (${bodyText.slice(0, 160)}).`
+ console.warn(`[Embedding] ${lastEmbeddingError}`)
+ return null
}
- lastEmbeddingError = `API ${detail} at ${embeddingConfig.endpoint}`
+
+ // Non-oversize definitive failure (auth, rate limit, server down, …).
+ lastEmbeddingError = `API ${resp.status} ${resp.statusText}${bodyText ? ` — ${bodyText.slice(0, 200)}` : ""} at ${cfg.endpoint}`
console.warn(`[Embedding] ${lastEmbeddingError}`)
return null
- }
- const data = await resp.json()
- const embedding = data?.data?.[0]?.embedding ?? null
- if (embedding) {
- lastEmbeddingError = null
- } else {
- lastEmbeddingError = `Response did not contain data[0].embedding (got ${JSON.stringify(data).slice(0, 200)})`
+ } catch (err) {
+ if (isFetchNetworkError(err)) {
+ lastEmbeddingError = `Network error reaching ${cfg.endpoint}. Check endpoint URL, API key, and connectivity.`
+ } else {
+ lastEmbeddingError = err instanceof Error ? err.message : String(err)
+ }
console.warn(`[Embedding] ${lastEmbeddingError}`)
+ return null
}
- return embedding
- } catch (err) {
- // Translate cross-webview fetch failures into a single actionable
- // message that names the URL, mirroring the LLM-client treatment.
- if (isFetchNetworkError(err)) {
- lastEmbeddingError = `Network error reaching ${embeddingConfig.endpoint}. Check endpoint URL, API key, and connectivity.`
- } else {
- lastEmbeddingError = err instanceof Error ? err.message : String(err)
- }
- console.warn(`[Embedding] ${lastEmbeddingError}`)
- return null
}
+
+ // Exhausted retries (only reachable if every halving round triggered
+ // the retry branch and then the loop condition ended).
+ lastEmbeddingError = `Embedding endpoint rejected every size down to ${current.length} chars — the server's context is smaller than ${current.length * 2}. Lower Settings → Embedding → Max Chunk Chars.`
+ console.warn(`[Embedding] ${lastEmbeddingError}`)
+ return null
}
-// ── LanceDB operations via Tauri commands ─────────────────────────────────
+// ── LanceDB v2 operations (via Rust Tauri commands) ──────────────────────
+
+interface ChunkUpsertInput {
+ chunkIndex: number
+ chunkText: string
+ headingPath: string
+ embedding: number[]
+}
-async function vectorUpsert(projectPath: string, pageId: string, embedding: number[]): Promise<void> {
- await invoke("vector_upsert", {
+async function vectorUpsertChunks(
+ projectPath: string,
+ pageId: string,
+ chunks: ChunkUpsertInput[],
+): Promise<void> {
+ await invoke("vector_upsert_chunks", {
projectPath: normalizePath(projectPath),
pageId,
- embedding: embedding.map((v) => Math.fround(v)), // ensure f32
+ chunks: chunks.map((c) => ({
+ chunk_index: c.chunkIndex,
+ chunk_text: c.chunkText,
+ heading_path: c.headingPath,
+ embedding: c.embedding.map((v) => Math.fround(v)),
+ })),
})
}
-async function vectorSearchLance(projectPath: string, queryEmbedding: number[], topK: number): Promise<Array<{ page_id: string; score: number }>> {
- return await invoke("vector_search", {
+interface ChunkSearchResult {
+ chunk_id: string
+ page_id: string
+ chunk_index: number
+ chunk_text: string
+ heading_path: string
+ score: number
+}
+
+async function vectorSearchChunks(
+ projectPath: string,
+ queryEmbedding: number[],
+ topK: number,
+): Promise<ChunkSearchResult[]> {
+ return await invoke("vector_search_chunks", {
projectPath: normalizePath(projectPath),
queryEmbedding: queryEmbedding.map((v) => Math.fround(v)),
topK,
})
}
-async function vectorDelete(projectPath: string, pageId: string): Promise<void> {
- await invoke("vector_delete", {
+async function vectorDeletePage(projectPath: string, pageId: string): Promise<void> {
+ await invoke("vector_delete_page", {
projectPath: normalizePath(projectPath),
pageId,
})
}
-async function vectorCount(projectPath: string): Promise<number> {
- return await invoke("vector_count", {
+async function vectorCountChunks(projectPath: string): Promise<number> {
+ return await invoke("vector_count_chunks", {
projectPath: normalizePath(projectPath),
})
}
-// ── Public API ────────────────────────────────────────────────────────────
+export async function legacyVectorRowCount(projectPath: string): Promise<number> {
+ try {
+ return await invoke("vector_legacy_row_count", {
+ projectPath: normalizePath(projectPath),
+ })
+ } catch {
+ return 0
+ }
+}
+
+export async function dropLegacyVectorTable(projectPath: string): Promise<void> {
+ await invoke("vector_drop_legacy", {
+ projectPath: normalizePath(projectPath),
+ })
+}
+
+// ── Chunk enrichment ─────────────────────────────────────────────────────
/**
- * Embed a single page and store in LanceDB.
- * Called after ingest to keep embeddings up to date.
+ * Build the text we actually embed for a chunk: page title + heading
+ * breadcrumb + chunk content. The breadcrumb is the most important
+ * context for a short chunk — a 300-char excerpt about "Mixture of
+ * Experts" is far more findable when the embedded text explicitly
+ * names its containing sections.
+ */
+function enrichChunkForEmbedding(
+ pageTitle: string,
+ chunk: Chunk,
+): string {
+ const parts: string[] = []
+ if (pageTitle.trim().length > 0) parts.push(pageTitle.trim())
+ if (chunk.headingPath.trim().length > 0) parts.push(chunk.headingPath.trim())
+ parts.push(chunk.text.trim())
+ return parts.join("\n\n")
+}
+
+// ── Public API: embedPage / embedAllPages / searchByEmbedding ────────────
+
+/**
+ * Embed a wiki page: chunk → per-chunk embed → replace the page's
+ * vectors in LanceDB in one batch. Every transient failure leaves the
+ * existing v2 rows intact (empty upsert is a no-op Rust-side).
+ *
+ * Called by ingest.ts after writing a page to disk.
*/
export async function embedPage(
projectPath: string,
pageId: string,
title: string,
content: string,
- embeddingConfig: EmbeddingConfig,
+ cfg: EmbeddingConfig,
): Promise<void> {
- if (!embeddingConfig.enabled || !embeddingConfig.model) return
+ if (!cfg.enabled || !cfg.model) return
const t0 = performance.now()
- const text = `${title}\n${content.slice(0, 1500)}`
- const emb = await fetchEmbedding(text, embeddingConfig)
- if (emb) {
- await vectorUpsert(projectPath, pageId, emb)
- console.log(`[Embedding] Indexed "${pageId}" (${emb.length}d) in ${Math.round(performance.now() - t0)}ms`)
- } else {
- console.log(`[Embedding] Failed to embed "${pageId}"`)
+ const chunks = chunkMarkdown(content, {
+ targetChars: cfg.maxChunkChars ?? 1000,
+ overlapChars: cfg.overlapChunkChars ?? 200,
+ })
+ if (chunks.length === 0) return
+
+ const rows: ChunkUpsertInput[] = []
+ let failedChunks = 0
+ for (const chunk of chunks) {
+ const embedText = enrichChunkForEmbedding(title, chunk)
+ const vec = await fetchEmbedding(embedText, cfg)
+ if (vec) {
+ rows.push({
+ chunkIndex: chunk.index,
+ chunkText: chunk.text,
+ headingPath: chunk.headingPath,
+ embedding: vec,
+ })
+ } else {
+ failedChunks++
+ }
+ }
+
+ if (rows.length === 0) {
+ console.log(
+ `[Embedding] Indexed nothing for "${pageId}" — all ${chunks.length} chunks failed. See getLastEmbeddingError().`,
+ )
+ return
}
+
+ await vectorUpsertChunks(projectPath, pageId, rows)
+ const elapsed = Math.round(performance.now() - t0)
+ console.log(
+ `[Embedding] Indexed "${pageId}": ${rows.length}/${chunks.length} chunks (${failedChunks} skipped) in ${elapsed}ms`,
+ )
}
/**
- * Embed all wiki pages that are not yet indexed.
- * Called on first enable or when model changes.
+ * Embed every wiki content page that isn't already indexed (or re-embed
+ * all when `force === true`). Driven from Settings → Embedding or on
+ * first enable. Skips structural pages (index / log / overview /
+ * purpose / schema) — they're aggregate views, not retrieval targets.
*/
export async function embedAllPages(
projectPath: string,
- embeddingConfig: EmbeddingConfig,
+ cfg: EmbeddingConfig,
onProgress?: (done: number, total: number) => void,
): Promise<number> {
- if (!embeddingConfig.enabled || !embeddingConfig.model) return 0
+ if (!cfg.enabled || !cfg.model) return 0
const pp = normalizePath(projectPath)
@@ -178,16 +355,10 @@ export async function embedAllPages(
const content = await readFile(file.path)
const titleMatch = content.match(/^---\n[\s\S]*?^title:\s*["']?(.+?)["']?\s*$/m)
const title = titleMatch ? titleMatch[1].trim() : file.id
-
- const text = `${title}\n${content.slice(0, 1500)}`
- const emb = await fetchEmbedding(text, embeddingConfig)
- if (emb) {
- await vectorUpsert(pp, file.id, emb)
- }
+ await embedPage(pp, file.id, title, content, cfg)
} catch {
- // skip
+ // skip — individual file failure doesn't halt the batch
}
-
done++
if (onProgress) onProgress(done, mdFiles.length)
}
@@ -196,53 +367,107 @@ export async function embedAllPages(
}
/**
- * Search wiki pages by semantic similarity via LanceDB.
- * Returns page IDs sorted by similarity score.
+ * Vector search over the v2 chunk store, shaped to stay API-compatible
+ * with the pre-0.3.11 per-page interface. Under the hood:
+ * 1. Embed the query.
+ * 2. Over-fetch top-K × 3 chunks.
+ * 3. Group by page_id; score each page as max(chunk_scores) plus
+ * 0.3 × sum of the other chunks' scores (bounded — capped at
+ * 1.0 - max_score), so a page with two good chunks outranks a
+ * page with one equally-good chunk and a weaker one.
+ * 4. Sort pages by score, return top-K.
+ *
+ * The optional `matchedChunks` field gives callers the raw chunk
+ * context when they want to surface "matched in this section" in
+ * the UI. Existing callers can ignore it.
*/
+export interface PageSearchResult {
+ id: string
+ score: number
+ matchedChunks?: Array<{ text: string; headingPath: string; score: number }>
+}
+
export async function searchByEmbedding(
projectPath: string,
query: string,
- embeddingConfig: EmbeddingConfig,
+ cfg: EmbeddingConfig,
topK: number = 10,
-): Promise<Array<{ id: string; score: number }>> {
- if (!embeddingConfig.enabled || !embeddingConfig.model) return []
+): Promise<PageSearchResult[]> {
+ if (!cfg.enabled || !cfg.model) return []
- const queryEmb = await fetchEmbedding(query, embeddingConfig)
+ const queryEmb = await fetchEmbedding(query, cfg)
if (!queryEmb) return []
+ const t0 = performance.now()
+ let rawChunks: ChunkSearchResult[] = []
try {
- const t0 = performance.now()
- const results = await vectorSearchLance(projectPath, queryEmb, topK)
- console.log(`[Embedding] LanceDB search: ${results.length} results in ${Math.round(performance.now() - t0)}ms`)
- return results.map((r) => ({ id: r.page_id, score: r.score }))
+ rawChunks = await vectorSearchChunks(projectPath, queryEmb, Math.max(topK * 3, 30))
} catch (err) {
- console.log(`[Embedding] LanceDB search failed: ${err instanceof Error ? err.message : err}`)
+ console.log(`[Embedding] LanceDB chunk search failed: ${err instanceof Error ? err.message : err}`)
return []
}
+ if (rawChunks.length === 0) return []
+
+ // Group by page; keep every matched chunk's score so we can compute
+ // a blended per-page score.
+ const byPage = new Map<string, ChunkSearchResult[]>()
+ for (const c of rawChunks) {
+ const bucket = byPage.get(c.page_id)
+ if (bucket) bucket.push(c)
+ else byPage.set(c.page_id, [c])
+ }
+
+ const ranked: PageSearchResult[] = []
+ for (const [pageId, chunks] of byPage.entries()) {
+ chunks.sort((a, b) => b.score - a.score)
+ const top = chunks[0].score
+ const tail = chunks.slice(1).reduce((sum, c) => sum + c.score, 0)
+ // Cap the tail contribution so many-weak-chunks can't drown a
+ // single-strong-chunk page. 0.3 weight is empirical; adjust later
+ // with real data.
+ const blended = top + Math.min(tail * 0.3, Math.max(0, 1 - top))
+ ranked.push({
+ id: pageId,
+ score: blended,
+ matchedChunks: chunks.slice(0, 3).map((c) => ({
+ text: c.chunk_text,
+ headingPath: c.heading_path,
+ score: c.score,
+ })),
+ })
+ }
+ ranked.sort((a, b) => b.score - a.score)
+
+ const elapsed = Math.round(performance.now() - t0)
+ console.log(
+ `[Embedding] LanceDB chunk search: ${rawChunks.length} chunks → ${ranked.length} pages in ${elapsed}ms`,
+ )
+
+ return ranked.slice(0, topK)
}
/**
- * Remove a page from the vector index.
+ * Remove a page's embeddings from the v2 index. Called from the
+ * source-delete flow so orphaned chunks don't pollute future searches.
*/
export async function removePageEmbedding(
projectPath: string,
pageId: string,
): Promise<void> {
try {
- await vectorDelete(projectPath, pageId)
+ await vectorDeletePage(projectPath, pageId)
} catch {
// non-critical
}
}
/**
- * Get the number of indexed vectors.
+ * Total chunks in the v2 index. Surfaces "N chunks indexed" status
+ * in Settings.
*/
-export async function getEmbeddingCount(
- projectPath: string,
-): Promise<number> {
+export async function getEmbeddingCount(projectPath: string): Promise<number> {
try {
- return await vectorCount(projectPath)
+ return await vectorCountChunks(projectPath)
} catch {
return 0
}
diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts
index a5da187d..257a3f44 100644
--- a/src/stores/wiki-store.ts
+++ b/src/stores/wiki-store.ts
@@ -29,6 +29,21 @@ interface EmbeddingConfig {
endpoint: string // e.g. "http://127.0.0.1:1234/v1/embeddings"
apiKey: string
model: string // e.g. "text-embedding-qwen3-embedding-0.6b"
+ /**
+ * Chunking knobs (Phase 1 RAG). Undefined values fall back to the
+ * chunker's built-in defaults in `src/lib/text-chunker.ts`:
+ * targetChars 1000
+ * maxChars 1500
+ * minChars 200
+ * overlapChars 200
+ *
+ * Users on small-context endpoints (e.g. llama.cpp with n_ctx=512,
+ * Ollama `mxbai-embed-large`) should lower `maxChunkChars` to avoid
+ * per-request rejections; fetchEmbedding also auto-halves on
+ * "too long" server errors as a second line of defence.
+ */
+ maxChunkChars?: number
+ overlapChunkChars?: number
}
/**