Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

fix: harden page embedding revisions

nash_su9511d4ccdc75
diff --git a/src-tauri/src/commands/page_embedding.rs b/src-tauri/src/commands/page_embedding.rs
index d8048446..6524698b 100644
--- a/src-tauri/src/commands/page_embedding.rs
+++ b/src-tauri/src/commands/page_embedding.rs
@@ -1,7 +1,5 @@
-use std::collections::BTreeMap;
use std::fs;
use std::path::{Component, Path, PathBuf};
-use std::sync::{Mutex, OnceLock};
use std::time::Duration;
use serde::Serialize;
@@ -22,8 +20,7 @@ const MAX_PAGE_BYTES: u64 = 2 * 1024 * 1024;
const MAX_PAGE_CHUNKS: usize = 512;
const EMBEDDING_BATCH_SIZE: usize = 64;
const PROVIDER_PHASE_TIMEOUT: Duration = Duration::from_secs(300);
-const REVISION_FILE: &str = ".llm-wiki/embedding-revisions.json";
-static REVISION_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
+const REVISION_DIR: &str = ".llm-wiki/embedding-revisions";
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
@@ -135,14 +132,10 @@ pub async fn embed_wiki_page(
let revision = format!("sha256:{:x}", Sha256::digest(content.as_bytes()));
let fingerprint = embedding_fingerprint(&revision, &config);
if !force {
- let existing_chunks = vectorstore::vector_page_revision_match(
- project_path,
- &page_id,
- &normalized_path,
- &fingerprint,
- )
- .await
- .map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Storage, err))?;
+ let existing_chunks =
+ vectorstore::vector_page_revision_match(project_path, &page_id, &fingerprint)
+ .await
+ .map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Storage, err))?;
if let Some(existing_chunks) = existing_chunks {
return Ok(PageEmbeddingResult {
path: normalized_path,
@@ -197,16 +190,11 @@ pub async fn embed_wiki_page(
),
)
})??;
+ validate_embedding_rows(&rows)?;
- vectorstore::vector_upsert_chunks_with_revision(
- project_path,
- &page_id,
- rows,
- &normalized_path,
- &fingerprint,
- )
- .await
- .map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Storage, err))?;
+ vectorstore::vector_upsert_chunks_with_revision(project_path, &page_id, rows, &fingerprint)
+ .await
+ .map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Storage, err))?;
Ok(PageEmbeddingResult {
path: normalized_path,
page_id,
@@ -217,6 +205,22 @@ pub async fn embed_wiki_page(
})
}
+fn validate_embedding_rows(rows: &[ChunkUpsertInput]) -> Result<(), PageEmbeddingError> {
+ let Some(expected) = rows.first().map(|row| row.embedding.len()) else {
+ return Err(PageEmbeddingError::new(
+ PageEmbeddingErrorKind::Provider,
+ "Embedding provider returned no vectors",
+ ));
+ };
+ if expected == 0 || rows.iter().any(|row| row.embedding.len() != expected) {
+ return Err(PageEmbeddingError::new(
+ PageEmbeddingErrorKind::Provider,
+ "Embedding provider returned empty or inconsistent vector dimensions",
+ ));
+ }
+ Ok(())
+}
+
async fn prepare_embedding_rows(
title: &str,
chunks: &[MarkdownChunk],
@@ -323,44 +327,33 @@ fn ensure_unique_page_stem(
Ok(())
}
-fn revision_path(project_path: &str) -> PathBuf {
- Path::new(project_path).join(REVISION_FILE)
+fn revision_path(project_path: &str, page_id: &str) -> PathBuf {
+ let key = format!("{:x}", Sha256::digest(page_id.as_bytes()));
+ Path::new(project_path)
+ .join(REVISION_DIR)
+ .join(format!("{key}.revision"))
}
-fn load_revisions(project_path: &str) -> BTreeMap<String, String> {
- fs::read_to_string(revision_path(project_path))
+pub(crate) fn load_revision(project_path: &str, page_id: &str) -> Option<String> {
+ fs::read_to_string(revision_path(project_path, page_id))
.ok()
- .and_then(|raw| serde_json::from_str(&raw).ok())
- .unwrap_or_default()
-}
-
-pub(crate) fn load_revision(project_path: &str, page_path: &str) -> Option<String> {
- let _guard = REVISION_LOCK.get_or_init(|| Mutex::new(())).lock().ok()?;
- load_revisions(project_path).get(page_path).cloned()
+ .map(|value| value.trim().to_string())
+ .filter(|value| !value.is_empty())
}
pub(crate) fn save_revision(
project_path: &str,
- page_path: &str,
+ page_id: &str,
revision: &str,
) -> Result<(), String> {
- let _guard = REVISION_LOCK
- .get_or_init(|| Mutex::new(()))
- .lock()
- .map_err(|_| "Embedding revision lock is poisoned".to_string())?;
- let path = revision_path(project_path);
+ let path = revision_path(project_path, page_id);
let parent = path
.parent()
.ok_or_else(|| "Invalid embedding revision path".to_string())?;
fs::create_dir_all(parent)
.map_err(|err| format!("Failed to create revision directory: {err}"))?;
- let mut revisions = load_revisions(project_path);
- revisions.retain(|relative, _| Path::new(project_path).join(relative).is_file());
- revisions.insert(page_path.to_string(), revision.to_string());
- let serialized = serde_json::to_string_pretty(&revisions)
- .map_err(|err| format!("Failed to serialize embedding revisions: {err}"))?;
let temp = path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
- fs::write(&temp, serialized)
+ fs::write(&temp, revision)
.map_err(|err| format!("Failed to write embedding revision temp file: {err}"))?;
if let Err(err) = fs::rename(&temp, &path) {
// Windows does not replace an existing destination. The metadata is
@@ -387,26 +380,11 @@ pub(crate) fn save_revision(
}
pub(crate) fn invalidate_page_revision(project_path: &str, page_id: &str) -> Result<(), String> {
- let _guard = REVISION_LOCK
- .get_or_init(|| Mutex::new(()))
- .lock()
- .map_err(|_| "Embedding revision lock is poisoned".to_string())?;
- let path = revision_path(project_path);
- let mut revisions = load_revisions(project_path);
- let original_len = revisions.len();
- revisions.retain(|relative, _| {
- Path::new(relative)
- .file_stem()
- .and_then(|value| value.to_str())
- != Some(page_id)
- });
- if revisions.len() == original_len {
- return Ok(());
+ match fs::remove_file(revision_path(project_path, page_id)) {
+ Ok(()) => Ok(()),
+ Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(err) => Err(format!("Failed to invalidate embedding revision: {err}")),
}
- let serialized = serde_json::to_string_pretty(&revisions)
- .map_err(|err| format!("Failed to serialize embedding revisions: {err}"))?;
- fs::write(path, serialized)
- .map_err(|err| format!("Failed to invalidate embedding revision: {err}"))
}
fn resolve_wiki_markdown_path(
@@ -530,6 +508,9 @@ fn find_frontmatter_close(rest: &str) -> Option<(usize, usize)> {
(rest.trim() == "---").then_some((0, rest.len()))
}
+// The regular ingest pipeline uses src/lib/text-chunker.ts. Keep heading,
+// frontmatter, fenced-block, table, overlap, and Unicode behavior aligned when
+// changing either implementation.
fn chunk_markdown(content: &str, target_chars: usize, overlap_chars: usize) -> Vec<MarkdownChunk> {
let normalized = content.replace("\r\n", "\n");
let body = strip_frontmatter(&normalized);
@@ -876,24 +857,36 @@ mod tests {
}
#[test]
- fn saving_revision_prunes_deleted_pages() {
+ fn revision_metadata_is_per_page_and_invalidates_without_rewriting_other_pages() {
let root = project();
- fs::write(root.join("wiki/current.md"), "# Current").unwrap();
- fs::create_dir_all(root.join(".llm-wiki")).unwrap();
- fs::write(
- revision_path(root.to_str().unwrap()),
- r#"{"wiki/deleted.md":"old"}"#,
- )
- .unwrap();
+ save_revision(root.to_str().unwrap(), "current", "sha256:current").unwrap();
+ save_revision(root.to_str().unwrap(), "other", "sha256:other").unwrap();
- save_revision(root.to_str().unwrap(), "wiki/current.md", "sha256:current").unwrap();
- let revisions = load_revisions(root.to_str().unwrap());
-
- assert_eq!(revisions.len(), 1);
assert_eq!(
- revisions.get("wiki/current.md").map(String::as_str),
+ load_revision(root.to_str().unwrap(), "current").as_deref(),
Some("sha256:current")
);
+ invalidate_page_revision(root.to_str().unwrap(), "current").unwrap();
+ assert_eq!(load_revision(root.to_str().unwrap(), "current"), None);
+ assert_eq!(
+ load_revision(root.to_str().unwrap(), "other").as_deref(),
+ Some("sha256:other")
+ );
let _ = fs::remove_dir_all(root);
}
+
+ #[test]
+ fn inconsistent_provider_dimensions_are_rejected_before_storage() {
+ let chunk = MarkdownChunk {
+ text: "content".to_string(),
+ heading_path: String::new(),
+ };
+ let rows = vec![
+ chunk_row(0, &chunk, vec![1.0, 2.0]),
+ chunk_row(1, &chunk, vec![1.0]),
+ ];
+
+ let error = validate_embedding_rows(&rows).unwrap_err();
+ assert_eq!(error.kind, PageEmbeddingErrorKind::Provider);
+ }
}
diff --git a/src-tauri/src/commands/vectorstore.rs b/src-tauri/src/commands/vectorstore.rs
index ab1c7df3..8be032fe 100644
--- a/src-tauri/src/commands/vectorstore.rs
+++ b/src-tauri/src/commands/vectorstore.rs
@@ -474,23 +474,16 @@ pub(crate) async fn vector_upsert_chunks_with_revision(
project_path: &str,
page_id: &str,
chunks: Vec<ChunkUpsertInput>,
- page_path: &str,
fingerprint: &str,
) -> Result<(), String> {
- vector_upsert_chunks_inner(
- project_path,
- page_id,
- chunks,
- Some((page_path, fingerprint)),
- )
- .await
+ vector_upsert_chunks_inner(project_path, page_id, chunks, Some(fingerprint)).await
}
async fn vector_upsert_chunks_inner(
project_path: &str,
page_id: &str,
chunks: Vec<ChunkUpsertInput>,
- revision: Option<(&str, &str)>,
+ revision: Option<&str>,
) -> Result<(), String> {
validate_page_id_for_v2(&page_id)?;
let lock = vectorstore_v2_lock(project_path);
@@ -542,9 +535,8 @@ async fn vector_upsert_chunks_inner(
.map_err(|e| format!("Create table error: {e}"))?;
}
- if let Some((page_path, fingerprint)) = revision {
- if let Err(err) = super::page_embedding::save_revision(project_path, page_path, fingerprint)
- {
+ if let Some(fingerprint) = revision {
+ if let Err(err) = super::page_embedding::save_revision(project_path, page_id, fingerprint) {
let _ = super::page_embedding::invalidate_page_revision(project_path, page_id);
eprintln!("[page-embedding] failed to save revision metadata: {err}");
}
@@ -558,14 +550,12 @@ async fn vector_upsert_chunks_inner(
pub(crate) async fn vector_page_revision_match(
project_path: &str,
page_id: &str,
- page_path: &str,
fingerprint: &str,
) -> Result<Option<usize>, String> {
validate_page_id_for_v2(page_id)?;
let lock = vectorstore_v2_lock(project_path);
let _guard = lock.read().await;
- if super::page_embedding::load_revision(project_path, page_path).as_deref() != Some(fingerprint)
- {
+ if super::page_embedding::load_revision(project_path, page_id).as_deref() != Some(fingerprint) {
return Ok(None);
}
diff --git a/src/lib/text-chunker.ts b/src/lib/text-chunker.ts
index 5165be23..e326a8a6 100644
--- a/src/lib/text-chunker.ts
+++ b/src/lib/text-chunker.ts
@@ -1,6 +1,11 @@
/**
* Markdown-aware recursive text chunker for embedding pipelines.
*
+ * The local API has a separate Rust implementation in
+ * `src-tauri/src/commands/page_embedding.rs`. Keep heading, frontmatter,
+ * fenced-block, table, overlap, and Unicode behavior aligned when changing
+ * either implementation.
+ *
* Design constraints (enforced by tests in text-chunker.test.ts):
*
* 1. Each chunk carries a `headingPath` breadcrumb ("## Intro > ### Usage")