Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

fix: rebuild embedding chunks safely

nash_su76104aba344a
diff --git a/src-tauri/src/commands/vectorstore.rs b/src-tauri/src/commands/vectorstore.rs
index ded7fee0..8c0d7796 100644
--- a/src-tauri/src/commands/vectorstore.rs
+++ b/src-tauri/src/commands/vectorstore.rs
@@ -460,12 +460,15 @@ pub async fn vector_upsert_chunks(
.await
.map_err(|e| format!("Open table error: {e}"))?;
- if let Err(e) = table.delete(&format!("page_id = '{}'", page_id)).await {
- eprintln!(
- "[vectorstore v2] Warning: delete before upsert failed for page '{}': {}",
- page_id, e
- );
- }
+ table
+ .delete(&format!("page_id = '{}'", page_id))
+ .await
+ .map_err(|e| {
+ format!(
+ "Delete existing chunks before upsert failed for page '{}': {}",
+ page_id, e
+ )
+ })?;
table
.add(data)
@@ -648,6 +651,36 @@ pub async fn vector_count_chunks(project_path: String) -> Result<usize, String>
.await
}
+/// Drop the v2 chunk table entirely. Used by Settings → Embedding
+/// "Re-index all pages" so a rebuild reflects the current wiki tree
+/// exactly and removes chunks for deleted/renamed pages.
+#[tauri::command]
+pub async fn vector_clear_chunks(project_path: String) -> Result<(), String> {
+ run_guarded_async("vector_clear_chunks", async move {
+ let db = connect(&db_path(&project_path))
+ .execute()
+ .await
+ .map_err(|e| format!("DB connect error: {e}"))?;
+
+ let tables = db
+ .table_names()
+ .execute()
+ .await
+ .map_err(|e| format!("List tables error: {e}"))?;
+
+ if !tables.contains(&TABLE_V2.to_string()) {
+ return Ok(());
+ }
+
+ db.drop_table(TABLE_V2, &[])
+ .await
+ .map_err(|e| format!("Drop chunk table error: {e}"))?;
+
+ Ok(())
+ })
+ .await
+}
+
/// Report whether the legacy per-page v1 table exists with any rows —
/// the TS layer uses this to show a one-time "re-index to v2" prompt in
/// Settings → Embedding after upgrading. Returns 0 when v1 is absent or
@@ -921,6 +954,26 @@ mod tests_v2 {
assert_eq!(vector_count_chunks(pp).await.unwrap(), 0);
}
+ #[tokio::test]
+ async fn v2_clear_chunks_drops_entire_chunk_table_and_is_idempotent() {
+ let p = tmp_project();
+ let pp = p.to_string_lossy().to_string();
+
+ vector_upsert_chunks(pp.clone(), "page-a".into(), make_chunks("page-a", 3, 16))
+ .await
+ .unwrap();
+ vector_upsert_chunks(pp.clone(), "page-b".into(), make_chunks("page-b", 4, 16))
+ .await
+ .unwrap();
+ assert_eq!(vector_count_chunks(pp.clone()).await.unwrap(), 7);
+
+ vector_clear_chunks(pp.clone()).await.unwrap();
+ assert_eq!(vector_count_chunks(pp.clone()).await.unwrap(), 0);
+
+ vector_clear_chunks(pp.clone()).await.unwrap();
+ assert_eq!(vector_count_chunks(pp.clone()).await.unwrap(), 0);
+ }
+
#[tokio::test]
async fn v2_rejects_mismatched_embedding_dimensions() {
let p = tmp_project();
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index afaf4505..3f98b157 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -225,6 +225,7 @@ pub fn run() {
commands::vectorstore::vector_search_chunks,
commands::vectorstore::vector_delete_page,
commands::vectorstore::vector_count_chunks,
+ commands::vectorstore::vector_clear_chunks,
commands::vectorstore::vector_legacy_row_count,
commands::vectorstore::vector_drop_legacy,
commands::claude_cli::claude_cli_detect,
diff --git a/src/components/settings/sections/embedding-section.tsx b/src/components/settings/sections/embedding-section.tsx
index 77927474..4b05d175 100644
--- a/src/components/settings/sections/embedding-section.tsx
+++ b/src/components/settings/sections/embedding-section.tsx
@@ -24,6 +24,7 @@ type ReindexState =
| { kind: "idle" }
| { kind: "running"; done: number; total: number }
| { kind: "done"; count: number }
+ | { kind: "error"; message: string }
type TestState =
| { kind: "idle" }
@@ -113,11 +114,22 @@ export function EmbeddingSection({ draft, setDraft }: Props) {
const handleReindex = useCallback(async () => {
if (!project) return
setReindex({ kind: "running", done: 0, total: 0 })
- const count = await embedAllPages(project.path, embeddingConfig, (done, total) => {
- setReindex({ kind: "running", done, total })
- })
- setReindex({ kind: "done", count })
- await refreshStats()
+ try {
+ const count = await embedAllPages(
+ project.path,
+ embeddingConfig,
+ (done, total) => {
+ setReindex({ kind: "running", done, total })
+ },
+ { clearExisting: true },
+ )
+ setReindex({ kind: "done", count })
+ await refreshStats()
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err)
+ setReindex({ kind: "error", message })
+ await refreshStats()
+ }
}, [project, embeddingConfig, refreshStats])
const handleDropLegacy = useCallback(async () => {
@@ -406,6 +418,12 @@ export function EmbeddingSection({ draft, setDraft }: Props) {
</p>
)}
+ {reindex.kind === "error" && (
+ <p className="text-xs text-destructive">
+ {t("settings.sections.embedding.reindexError", { message: reindex.message })}
+ </p>
+ )}
+
{legacyDropped && (
<p className="text-xs text-muted-foreground">
{t("settings.sections.embedding.dropLegacyDone")}
diff --git a/src/i18n/en.json b/src/i18n/en.json
index c8ee9e48..7ddb5f41 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -332,6 +332,7 @@
"reindexAll": "Re-index all pages",
"reindexing": "Indexing {{done}}/{{total}}…",
"reindexDone": "Indexed {{count}} pages.",
+ "reindexError": "Re-index failed: {{message}}",
"legacyPromptTitle": "Legacy per-page index detected",
"legacyPromptBody": "Your project has {{count}} entries in the old per-page vector store. The new per-chunk index gives better retrieval but needs a one-time re-embed. Run \"Re-index all pages\" to upgrade.",
"dropLegacy": "Drop legacy index",
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index a595e95a..f84df0cd 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -332,6 +332,7 @@
"reindexAll": "重新索引全部页面",
"reindexing": "索引中 {{done}}/{{total}}…",
"reindexDone": "已索引 {{count}} 个页面。",
+ "reindexError": "重新索引失败:{{message}}",
"legacyPromptTitle": "检测到旧版(按页)索引",
"legacyPromptBody": "当前项目在旧版按页向量存储里有 {{count}} 条记录。新版按 chunk 索引检索质量更高,但需要一次性重新 embed。点击\"重新索引全部页面\"完成升级。",
"dropLegacy": "删除旧版索引",
diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.ts
index cd23ab67..fd8abbae 100644
--- a/src/lib/embedding.test.ts
+++ b/src/lib/embedding.test.ts
@@ -1090,6 +1090,224 @@ describe("embedAllPages", () => {
expect(pageIds).toEqual(["attention", "rope"])
})
+ it("clears the chunk table before a forced rebuild", async () => {
+ listDirectoryMock.mockResolvedValueOnce(makeTree())
+ readFileMock.mockResolvedValue("# Title\n\nBody.")
+ mockHttpFetch.mockImplementation(async () => okResponse([0.5]))
+
+ const count = await embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )
+
+ expect(count).toBe(2)
+ const commands = mockInvoke.mock.calls.map((call) => call[0])
+ expect(commands[0]).toBe("vector_clear_chunks")
+ expect(commands.filter((cmd) => cmd === "vector_upsert_chunks")).toHaveLength(2)
+ })
+
+ it("does not clear the chunk table during ordinary embedAllPages runs", async () => {
+ listDirectoryMock.mockResolvedValueOnce(makeTree())
+ readFileMock.mockResolvedValue("# Title\n\nBody.")
+ mockHttpFetch.mockImplementation(async () => okResponse([0.5]))
+
+ await embedAllPages("/proj", cfg)
+
+ expect(mockInvoke.mock.calls.map((call) => call[0])).not.toContain("vector_clear_chunks")
+ })
+
+ it("does not clear existing chunks when forced rebuild cannot embed every page", async () => {
+ listDirectoryMock.mockResolvedValueOnce([
+ { name: "a.md", path: "/proj/wiki/a.md", is_dir: false },
+ { name: "b.md", path: "/proj/wiki/b.md", is_dir: false },
+ ])
+ readFileMock.mockResolvedValue("# Title\n\nBody.")
+ mockHttpFetch
+ .mockResolvedValueOnce(okResponse([0.5]))
+ .mockResolvedValueOnce(genericErrorResponse(500, "embedding server down"))
+
+ let message = ""
+ try {
+ await embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )
+ } catch (err) {
+ message = err instanceof Error ? err.message : String(err)
+ }
+ expect(message).toContain("1 of 2 pages could not be embedded")
+ expect(message).not.toContain("Re-index failed: Re-index failed")
+
+ expect(mockInvoke.mock.calls.map((call) => call[0])).not.toContain("vector_clear_chunks")
+ expect(mockInvoke.mock.calls.map((call) => call[0])).not.toContain("vector_upsert_chunks")
+ })
+
+ it("does not clear existing chunks when forced rebuild cannot embed any page", async () => {
+ listDirectoryMock.mockResolvedValueOnce(makeTree())
+ readFileMock.mockResolvedValue("# Title\n\nBody.")
+ mockHttpFetch.mockResolvedValue(genericErrorResponse(500, "embedding server down"))
+
+ await expect(embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )).rejects.toThrow("2 of 2 pages could not be embedded")
+
+ expect(mockInvoke.mock.calls.map((call) => call[0])).not.toContain("vector_clear_chunks")
+ expect(mockInvoke.mock.calls.map((call) => call[0])).not.toContain("vector_upsert_chunks")
+ })
+
+ it("skips empty content pages during forced rebuild without treating them as failures", async () => {
+ listDirectoryMock.mockResolvedValueOnce([
+ { name: "empty.md", path: "/proj/wiki/empty.md", is_dir: false },
+ { name: "body.md", path: "/proj/wiki/body.md", is_dir: false },
+ ])
+ readFileMock
+ .mockResolvedValueOnce("---\ntitle: Empty Stub\n---\n")
+ .mockResolvedValueOnce("# Body\n\nThis page has content.")
+ mockHttpFetch.mockImplementation(async () => okResponse([0.5]))
+
+ const count = await embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )
+
+ expect(count).toBe(1)
+ const commands = mockInvoke.mock.calls.map((call) => call[0])
+ expect(commands[0]).toBe("vector_clear_chunks")
+ const upserts = mockInvoke.mock.calls.filter((call) => call[0] === "vector_upsert_chunks")
+ expect(upserts).toHaveLength(1)
+ expect((upserts[0][1] as { pageId: string }).pageId).toBe("body")
+ })
+
+ it("does not clear existing chunks when a forced rebuild page only embeds partially", async () => {
+ listDirectoryMock.mockResolvedValueOnce([
+ { name: "partial.md", path: "/proj/wiki/partial.md", is_dir: false },
+ ])
+ readFileMock.mockResolvedValueOnce(`${"First chunk body. ".repeat(20)}\n\n${"Second chunk body. ".repeat(20)}`)
+ const smallChunks = { ...cfg, maxChunkChars: 220, overlapChunkChars: 0 }
+ mockHttpFetch
+ .mockResolvedValueOnce(okResponse([0.5]))
+ .mockResolvedValueOnce(genericErrorResponse(500, "embedding server down"))
+
+ let message = ""
+ try {
+ await embedAllPages(
+ "/proj",
+ smallChunks,
+ undefined,
+ { clearExisting: true },
+ )
+ } catch (err) {
+ message = err instanceof Error ? err.message : String(err)
+ }
+ expect(message).toContain("chunks failed to embed")
+ expect(message).toContain("Check endpoint URL")
+
+ const commands = mockInvoke.mock.calls.map((call) => call[0])
+ expect(commands).not.toContain("vector_clear_chunks")
+ expect(commands).not.toContain("vector_upsert_chunks")
+ })
+
+ it("surfaces an incomplete-index warning when forced rebuild write fails after clearing", async () => {
+ listDirectoryMock.mockResolvedValueOnce([
+ { name: "a.md", path: "/proj/wiki/a.md", is_dir: false },
+ { name: "b.md", path: "/proj/wiki/b.md", is_dir: false },
+ ])
+ readFileMock.mockResolvedValue("# Title\n\nBody.")
+ mockHttpFetch.mockImplementation(async () => okResponse([0.5]))
+ let upsertCount = 0
+ mockInvoke.mockImplementation(async (command) => {
+ if (command === "vector_upsert_chunks") {
+ upsertCount++
+ if (upsertCount === 2) throw new Error("lancedb write failed")
+ }
+ return undefined
+ })
+
+ await expect(embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )).rejects.toThrow("rebuilt index may be incomplete")
+
+ const commands = mockInvoke.mock.calls.map((call) => call[0])
+ expect(commands[0]).toBe("vector_clear_chunks")
+ expect(commands.filter((command) => command === "vector_upsert_chunks")).toHaveLength(2)
+ })
+
+ it("clears stale chunks when forced rebuild has no content pages in a readable wiki tree", async () => {
+ listDirectoryMock.mockResolvedValueOnce([
+ { name: "index.md", path: "/proj/wiki/index.md", is_dir: false },
+ { name: "log.md", path: "/proj/wiki/log.md", is_dir: false },
+ ])
+ mockInvoke.mockImplementation(async (command) => {
+ if (command === "vector_count_chunks") return 0
+ return undefined
+ })
+
+ const out = await embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )
+
+ expect(out).toBe(0)
+ expect(mockInvoke.mock.calls.map((call) => call[0])).toContain("vector_clear_chunks")
+ expect(mockHttpFetch).not.toHaveBeenCalled()
+ })
+
+ it("does not clear existing chunks when all content pages are empty", async () => {
+ listDirectoryMock.mockResolvedValueOnce([
+ { name: "empty.md", path: "/proj/wiki/empty.md", is_dir: false },
+ ])
+ readFileMock.mockResolvedValueOnce("---\ntitle: Empty Stub\n---\n")
+ mockInvoke.mockImplementation(async (command) => {
+ if (command === "vector_count_chunks") return 5
+ return undefined
+ })
+
+ await expect(embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )).rejects.toThrow("Existing index was left unchanged")
+
+ const commands = mockInvoke.mock.calls.map((call) => call[0])
+ expect(commands).toContain("vector_count_chunks")
+ expect(commands).not.toContain("vector_clear_chunks")
+ expect(mockHttpFetch).not.toHaveBeenCalled()
+ })
+
+ it("does not clear existing chunks when a readable wiki tree unexpectedly has no content pages", async () => {
+ listDirectoryMock.mockResolvedValueOnce([])
+ mockInvoke.mockImplementation(async (command) => {
+ if (command === "vector_count_chunks") return 7
+ return undefined
+ })
+
+ await expect(embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )).rejects.toThrow("Existing index was left unchanged")
+
+ const commands = mockInvoke.mock.calls.map((call) => call[0])
+ expect(commands).toContain("vector_count_chunks")
+ expect(commands).not.toContain("vector_clear_chunks")
+ })
+
it("extracts the title from YAML frontmatter when present", async () => {
listDirectoryMock.mockResolvedValueOnce([
{ name: "rope.md", path: "/proj/wiki/rope.md", is_dir: false },
@@ -1142,6 +1360,19 @@ describe("embedAllPages", () => {
expect(out).toBe(0)
})
+ it("does not clear existing chunks when forced rebuild cannot read the wiki tree", async () => {
+ listDirectoryMock.mockRejectedValueOnce(new Error("ENOENT: no such file"))
+
+ await expect(embedAllPages(
+ "/proj",
+ cfg,
+ undefined,
+ { clearExisting: true },
+ )).rejects.toThrow("Could not read wiki tree")
+
+ expect(mockInvoke.mock.calls.map((call) => call[0])).not.toContain("vector_clear_chunks")
+ })
+
it("continues with remaining files when one file's readFile throws", async () => {
listDirectoryMock.mockResolvedValueOnce([
{ name: "a.md", path: "/proj/wiki/a.md", is_dir: false },
@@ -1154,8 +1385,7 @@ describe("embedAllPages", () => {
mockHttpFetch.mockImplementation(async () => okResponse([0.5]))
const count = await embedAllPages("/proj", cfg)
- // `done` is incremented for every file attempted, even if embed fails.
- expect(count).toBe(2)
+ expect(count).toBe(1)
const upserts = mockInvoke.mock.calls.filter((c) => c[0] === "vector_upsert_chunks")
expect(upserts).toHaveLength(1)
expect((upserts[0][1] as { pageId: string }).pageId).toBe("b")
diff --git a/src/lib/embedding.ts b/src/lib/embedding.ts
index 1b44c832..ca99e87f 100644
--- a/src/lib/embedding.ts
+++ b/src/lib/embedding.ts
@@ -325,6 +325,12 @@ async function vectorCountChunks(projectPath: string): Promise<number> {
})
}
+async function vectorClearChunks(projectPath: string): Promise<void> {
+ await invoke("vector_clear_chunks", {
+ projectPath: normalizePath(projectPath),
+ })
+}
+
export async function legacyVectorRowCount(projectPath: string): Promise<number> {
try {
return await invoke("vector_legacy_row_count", {
@@ -341,6 +347,10 @@ export async function dropLegacyVectorTable(projectPath: string): Promise<void>
})
}
+export async function clearChunkVectorTable(projectPath: string): Promise<void> {
+ await vectorClearChunks(projectPath)
+}
+
// ── Chunk enrichment ─────────────────────────────────────────────────────
/**
@@ -361,30 +371,36 @@ function enrichChunkForEmbedding(
return parts.join("\n\n")
}
-// ── Public API: embedPage / embedAllPages / searchByEmbedding ────────────
+interface PreparedPageEmbedding {
+ pageId: string
+ rows: ChunkUpsertInput[]
+ chunkCount: number
+ failedChunks: number
+}
-/**
- * Embed a wiki page: chunk → per-chunk embed → replace the page's
- * vectors in LanceDB in one batch. Every transient failure leaves the
- * existing v2 rows intact (empty upsert is a no-op Rust-side).
- *
- * Called by ingest.ts after writing a page to disk.
- */
-export async function embedPage(
- projectPath: string,
+type PageEmbeddingPreparation =
+ | { status: "ready"; page: PreparedPageEmbedding }
+ | { status: "empty" }
+ | { status: "failed"; reason: string }
+
+function extractEmbeddingTitle(content: string, fallbackId: string): string {
+ const titleMatch = content.match(/^---\n[\s\S]*?^title:\s*["']?(.+?)["']?\s*$/m)
+ return titleMatch ? titleMatch[1].trim() : fallbackId
+}
+
+async function preparePageEmbeddingRows(
pageId: string,
title: string,
content: string,
cfg: EmbeddingConfig,
-): Promise<void> {
- if (!cfg.enabled || !cfg.model) return
+): Promise<PageEmbeddingPreparation> {
+ if (!cfg.enabled || !cfg.model) return { status: "empty" }
- const t0 = performance.now()
const chunks = chunkMarkdown(content, {
targetChars: cfg.maxChunkChars ?? 1000,
overlapChars: cfg.overlapChunkChars ?? 200,
})
- if (chunks.length === 0) return
+ if (chunks.length === 0) return { status: "empty" }
const rows: ChunkUpsertInput[] = []
let failedChunks = 0
@@ -404,17 +420,56 @@ export async function embedPage(
}
if (rows.length === 0) {
- console.log(
- `[Embedding] Indexed nothing for "${pageId}" — all ${chunks.length} chunks failed. See getLastEmbeddingError().`,
- )
- return
+ return {
+ status: "failed",
+ reason: getLastEmbeddingError() || "all chunks failed to embed",
+ }
+ }
+ return {
+ status: "ready",
+ page: {
+ pageId,
+ rows,
+ chunkCount: chunks.length,
+ failedChunks,
+ },
+ }
+}
+
+// ── Public API: embedPage / embedAllPages / searchByEmbedding ────────────
+
+/**
+ * Embed a wiki page: chunk → per-chunk embed → replace the page's
+ * vectors in LanceDB in one batch. Every transient failure leaves the
+ * existing v2 rows intact (empty upsert is a no-op Rust-side).
+ *
+ * Called by ingest.ts after writing a page to disk.
+ */
+export async function embedPage(
+ projectPath: string,
+ pageId: string,
+ title: string,
+ content: string,
+ cfg: EmbeddingConfig,
+): Promise<boolean> {
+ const t0 = performance.now()
+ const prepared = await preparePageEmbeddingRows(pageId, title, content, cfg)
+
+ if (prepared.status !== "ready") {
+ if (prepared.status === "failed") {
+ console.log(
+ `[Embedding] Indexed nothing for "${pageId}" — no chunks could be embedded. See getLastEmbeddingError().`,
+ )
+ }
+ return false
}
- await vectorUpsertChunks(projectPath, pageId, rows)
+ await vectorUpsertChunks(projectPath, pageId, prepared.page.rows)
const elapsed = Math.round(performance.now() - t0)
console.log(
- `[Embedding] Indexed "${pageId}": ${rows.length}/${chunks.length} chunks (${failedChunks} skipped) in ${elapsed}ms`,
+ `[Embedding] Indexed "${pageId}": ${prepared.page.rows.length}/${prepared.page.chunkCount} chunks (${prepared.page.failedChunks} skipped) in ${elapsed}ms`,
)
+ return true
}
/**
@@ -427,8 +482,10 @@ export async function embedAllPages(
projectPath: string,
cfg: EmbeddingConfig,
onProgress?: (done: number, total: number) => void,
+ options?: { clearExisting?: boolean },
): Promise<number> {
if (!cfg.enabled || !cfg.model) return 0
+ lastEmbeddingError = null
const pp = normalizePath(projectPath)
@@ -436,6 +493,9 @@ export async function embedAllPages(
try {
tree = await listDirectory(`${pp}/wiki`)
} catch {
+ if (options?.clearExisting) {
+ throw new Error("Could not read wiki tree; existing index was left unchanged.")
+ }
return 0
}
@@ -454,13 +514,90 @@ export async function embedAllPages(
}
walk(tree)
+ if (options?.clearExisting) {
+ if (mdFiles.length === 0) {
+ const existingChunks = await vectorCountChunks(pp).catch(() => 0)
+ if (existingChunks > 0) {
+ throw new Error(
+ `Wiki tree returned no content pages, but ${existingChunks} chunks are currently indexed. Existing index was left unchanged.`,
+ )
+ }
+ await clearChunkVectorTable(pp)
+ return 0
+ }
+
+ const preparedPages: PreparedPageEmbedding[] = []
+ const failures: string[] = []
+ let attempted = 0
+ for (const file of mdFiles) {
+ try {
+ const content = await readFile(file.path)
+ const title = extractEmbeddingTitle(content, file.id)
+ const prepared = await preparePageEmbeddingRows(file.id, title, content, cfg)
+ if (prepared.status === "ready") {
+ if (prepared.page.failedChunks > 0) {
+ const reason = getLastEmbeddingError()
+ failures.push(
+ `${file.id}: ${prepared.page.failedChunks} of ${prepared.page.chunkCount} chunks failed to embed${reason ? ` (${reason})` : ""}`,
+ )
+ } else {
+ preparedPages.push(prepared.page)
+ }
+ } else if (prepared.status === "failed") {
+ failures.push(`${file.id}: ${prepared.reason}`)
+ }
+ } catch (err) {
+ failures.push(`${file.id}: ${err instanceof Error ? err.message : String(err)}`)
+ }
+ attempted++
+ if (onProgress) onProgress(attempted, mdFiles.length)
+ }
+
+ if (failures.length > 0) {
+ throw new Error(
+ `${failures.length} of ${mdFiles.length} pages could not be embedded (${failures[0]}). Existing index was left unchanged.`,
+ )
+ }
+
+ if (preparedPages.length === 0) {
+ const existingChunks = await vectorCountChunks(pp).catch(() => 0)
+ if (existingChunks > 0) {
+ throw new Error(
+ `Wiki tree has only empty content pages, but ${existingChunks} chunks are currently indexed. Existing index was left unchanged.`,
+ )
+ }
+ }
+
+ await clearChunkVectorTable(pp)
+
+ let written = 0
+ for (const page of preparedPages) {
+ try {
+ await vectorUpsertChunks(pp, page.pageId, page.rows)
+ } catch (err) {
+ const reason = err instanceof Error ? err.message : String(err)
+ throw new Error(
+ `Rebuild write failed after clearing existing chunks (${page.pageId}: ${reason}). The rebuilt index may be incomplete; run re-index again after fixing the error.`,
+ )
+ }
+ written++
+ console.log(
+ `[Embedding] Rebuilt "${page.pageId}": ${page.rows.length}/${page.chunkCount} chunks (${page.failedChunks} skipped)`,
+ )
+ }
+
+ return written
+ }
+
let done = 0
+ let indexed = 0
for (const file of mdFiles) {
try {
const content = await readFile(file.path)
- const titleMatch = content.match(/^---\n[\s\S]*?^title:\s*["']?(.+?)["']?\s*$/m)
- const title = titleMatch ? titleMatch[1].trim() : file.id
- await embedPage(pp, file.id, title, content, cfg)
+ const title = extractEmbeddingTitle(content, file.id)
+ if (await embedPage(pp, file.id, title, content, cfg)) {
+ indexed++
+ }
} catch {
// skip — individual file failure doesn't halt the batch
}
@@ -468,7 +605,7 @@ export async function embedAllPages(
if (onProgress) onProgress(done, mdFiles.length)
}
- return done
+ return indexed
}
/**