LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::time::Duration;
use serde::Serialize;
use sha2::{Digest, Sha256};
use walkdir::WalkDir;
use super::search::{
fetch_embedding_batch, fetch_embedding_with_retry, supports_embedding_batch,
SearchEmbeddingConfig,
};
use super::vectorstore::{self, ChunkUpsertInput};
const DEFAULT_CHUNK_CHARS: usize = 1_000;
const DEFAULT_OVERLAP_CHARS: usize = 200;
const MIN_CHUNK_CHARS: usize = 64;
const MAX_CHUNK_CHARS: usize = 32_000;
const MAX_PAGE_BYTES: u64 = 2 * 1024 * 1024;
const MAX_PAGE_CHUNKS: usize = 512;
const EMBEDDING_BATCH_SIZE: usize = 64;
const PROVIDER_PHASE_TIMEOUT: Duration = Duration::from_secs(300);
const REVISION_DIR: &str = ".llm-wiki/embedding-revisions";
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct PageEmbeddingResult {
pub path: String,
pub page_id: String,
pub revision: String,
pub chunks: usize,
pub vectors_written: usize,
pub status: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PageEmbeddingErrorKind {
InvalidRequest,
NotFound,
Provider,
Storage,
Timeout,
}
#[derive(Debug, Clone)]
pub struct PageEmbeddingError {
pub kind: PageEmbeddingErrorKind,
pub message: String,
}
impl PageEmbeddingError {
fn new(kind: PageEmbeddingErrorKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct MarkdownChunk {
text: String,
heading_path: String,
}
pub async fn embed_wiki_page(
project_path: &str,
relative_path: &str,
config: SearchEmbeddingConfig,
force: bool,
) -> Result<PageEmbeddingResult, PageEmbeddingError> {
if !config.enabled || config.model.trim().is_empty() {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
"Embedding is not enabled or no embedding model is configured",
));
}
let (page_path, normalized_path) = resolve_wiki_markdown_path(project_path, relative_path)?;
let metadata = fs::metadata(&page_path).map_err(|err| {
PageEmbeddingError::new(
PageEmbeddingErrorKind::NotFound,
format!("Failed to inspect wiki page: {err}"),
)
})?;
if metadata.len() > MAX_PAGE_BYTES {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
format!(
"Wiki page exceeds the {} MiB indexing limit",
MAX_PAGE_BYTES / 1024 / 1024
),
));
}
let content = fs::read_to_string(&page_path).map_err(|err| {
let kind = if err.kind() == std::io::ErrorKind::NotFound {
PageEmbeddingErrorKind::NotFound
} else {
PageEmbeddingErrorKind::InvalidRequest
};
PageEmbeddingError::new(
kind,
format!("Failed to read wiki page as UTF-8 text: {err}"),
)
})?;
let page_id = vector_page_id(&normalized_path).ok_or_else(|| {
PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
"Invalid wiki page name",
)
})?;
vectorstore::validate_page_id_for_v2(&page_id)
.map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::InvalidRequest, err))?;
let page_stem = page_id.rsplit('/').next().unwrap_or(&page_id);
if matches!(
page_stem.to_ascii_lowercase().as_str(),
"index" | "log" | "overview"
) {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
"Aggregate wiki pages index.md, log.md, and overview.md are maintained by the app and are not vector-indexed",
));
}
let revision = format!("sha256:{:x}", Sha256::digest(content.as_bytes()));
let fingerprint = embedding_fingerprint(&revision, &config);
if !force {
let existing_chunks =
vectorstore::vector_page_revision_match(project_path, &page_id, &fingerprint)
.await
.map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Storage, err))?;
if let Some(existing_chunks) = existing_chunks {
return Ok(PageEmbeddingResult {
path: normalized_path,
page_id,
revision,
chunks: existing_chunks,
vectors_written: 0,
status: "unchanged".to_string(),
});
}
}
let title = extract_title(&content, &page_id);
let chunk_chars = config
.max_chunk_chars
.unwrap_or(DEFAULT_CHUNK_CHARS)
.clamp(MIN_CHUNK_CHARS, MAX_CHUNK_CHARS);
let overlap_chars = config
.overlap_chunk_chars
.unwrap_or(DEFAULT_OVERLAP_CHARS)
.min(chunk_chars / 2);
let chunks = chunk_markdown(&content, chunk_chars, overlap_chars);
if chunks.is_empty() {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
"Wiki page has no indexable content",
));
}
if chunks.len() > MAX_PAGE_CHUNKS {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
format!(
"Wiki page produces {} chunks, exceeding the {} chunk limit; increase maxChunkChars or split the page",
chunks.len(), MAX_PAGE_CHUNKS
),
));
}
// Only provider work is cancellable. Once storage replacement starts, let
// it complete so a timeout cannot remove the previous page index midway.
let rows = tokio::time::timeout(
PROVIDER_PHASE_TIMEOUT,
prepare_embedding_rows(&title, &chunks, &config),
)
.await
.map_err(|_| {
PageEmbeddingError::new(
PageEmbeddingErrorKind::Timeout,
format!(
"Embedding provider timed out after {} seconds",
PROVIDER_PHASE_TIMEOUT.as_secs()
),
)
})??;
validate_embedding_rows(&rows)?;
vectorstore::vector_upsert_chunks_with_revision(project_path, &page_id, rows, &fingerprint)
.await
.map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Storage, err))?;
remove_legacy_vector_id_if_safe(project_path, &page_id).await;
Ok(PageEmbeddingResult {
path: normalized_path,
page_id,
revision,
chunks: chunks.len(),
vectors_written: chunks.len(),
status: "indexed".to_string(),
})
}
fn validate_embedding_rows(rows: &[ChunkUpsertInput]) -> Result<(), PageEmbeddingError> {
let Some(expected) = rows.first().map(|row| row.embedding.len()) else {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::Provider,
"Embedding provider returned no vectors",
));
};
if expected == 0 || rows.iter().any(|row| row.embedding.len() != expected) {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::Provider,
"Embedding provider returned empty or inconsistent vector dimensions",
));
}
Ok(())
}
async fn prepare_embedding_rows(
title: &str,
chunks: &[MarkdownChunk],
config: &SearchEmbeddingConfig,
) -> Result<Vec<ChunkUpsertInput>, PageEmbeddingError> {
// Build every vector before replacing the page in LanceDB. A provider
// failure therefore leaves the previous, known-good page index intact.
let mut rows = Vec::with_capacity(chunks.len());
if supports_embedding_batch(&config) {
for batch in chunks.chunks(EMBEDDING_BATCH_SIZE) {
let texts = batch
.iter()
.map(|chunk| enrich_chunk(&title, chunk))
.collect::<Vec<_>>();
let embeddings = fetch_embedding_batch(&texts, &config)
.await
.map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Provider, err))?;
if embeddings.len() != batch.len() {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::Provider,
"Embedding provider returned an incomplete batch",
));
}
for (chunk, embedding) in batch.iter().zip(embeddings) {
rows.push(chunk_row(rows.len(), chunk, embedding));
}
}
} else {
for (index, chunk) in chunks.iter().enumerate() {
let embedding_text = enrich_chunk(&title, chunk);
let embedding = fetch_embedding_with_retry(&embedding_text, &config, 3)
.await
.map_err(|err| PageEmbeddingError::new(PageEmbeddingErrorKind::Provider, err))?;
rows.push(chunk_row(index, chunk, embedding));
}
}
Ok(rows)
}
fn chunk_row(index: usize, chunk: &MarkdownChunk, embedding: Vec<f32>) -> ChunkUpsertInput {
ChunkUpsertInput {
chunk_index: index as u32,
chunk_text: chunk.text.clone(),
heading_path: chunk.heading_path.clone(),
embedding,
}
}
fn embedding_fingerprint(revision: &str, config: &SearchEmbeddingConfig) -> String {
let signature = serde_json::json!({
"revision": revision,
"endpoint": config.endpoint.trim(),
"model": config.model.trim(),
"outputDimensionality": config.output_dimensionality,
"extraHeaders": config.extra_headers,
"maxChunkChars": config.max_chunk_chars.unwrap_or(DEFAULT_CHUNK_CHARS),
"overlapChunkChars": config.overlap_chunk_chars.unwrap_or(DEFAULT_OVERLAP_CHARS),
});
format!(
"sha256:{:x}",
Sha256::digest(signature.to_string().as_bytes())
)
}
fn vector_page_id(normalized_path: &str) -> Option<String> {
let normalized = normalized_path.replace('\\', "/");
let relative = normalized.strip_prefix("wiki/")?;
if relative.len() < 3 || !relative[relative.len() - 3..].eq_ignore_ascii_case(".md") {
return None;
}
let relative = relative[..relative.len() - 3].trim_matches('/').to_string();
(!relative.is_empty()).then_some(relative)
}
async fn remove_legacy_vector_id_if_safe(project_path: &str, page_id: &str) {
let Some(legacy_id) = page_id.rsplit('/').next().filter(|_| page_id.contains('/')) else {
return;
};
let wiki_root = Path::new(project_path).join("wiki");
let owners = WalkDir::new(&wiki_root)
.follow_links(false)
.into_iter()
.filter_map(Result::ok)
.filter(|entry| entry.file_type().is_file())
.filter(|entry| {
entry
.path()
.extension()
.and_then(|value| value.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
&& entry
.path()
.file_stem()
.and_then(|value| value.to_str())
.is_some_and(|stem| stem.eq_ignore_ascii_case(legacy_id))
})
.take(2)
.count();
if owners != 1 {
return;
}
if let Err(error) =
vectorstore::vector_delete_page(project_path.to_string(), legacy_id.to_string()).await
{
eprintln!("[Embedding] failed to remove legacy vector id '{legacy_id}': {error}");
}
}
fn revision_path(project_path: &str, page_id: &str) -> PathBuf {
let key = format!("{:x}", Sha256::digest(page_id.as_bytes()));
Path::new(project_path)
.join(REVISION_DIR)
.join(format!("{key}.revision"))
}
pub(crate) fn load_revision(project_path: &str, page_id: &str) -> Option<String> {
fs::read_to_string(revision_path(project_path, page_id))
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
pub(crate) fn save_revision(
project_path: &str,
page_id: &str,
revision: &str,
) -> Result<(), String> {
let path = revision_path(project_path, page_id);
let parent = path
.parent()
.ok_or_else(|| "Invalid embedding revision path".to_string())?;
fs::create_dir_all(parent)
.map_err(|err| format!("Failed to create revision directory: {err}"))?;
let temp = path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
fs::write(&temp, revision)
.map_err(|err| format!("Failed to write embedding revision temp file: {err}"))?;
if let Err(err) = fs::rename(&temp, &path) {
// Windows does not replace an existing destination. The metadata is
// only an optimization; restore the old valid file if replacement fails.
if path.exists() {
let previous = fs::read(&path).ok();
fs::remove_file(&path).map_err(|remove_err| {
format!("Failed to replace embedding revisions: {remove_err}")
})?;
if let Err(rename_err) = fs::rename(&temp, &path) {
if let Some(previous) = previous {
let _ = fs::write(&path, previous);
}
return Err(format!(
"Failed to replace embedding revisions: {rename_err}"
));
}
} else {
let _ = fs::remove_file(&temp);
return Err(format!("Failed to save embedding revisions: {err}"));
}
}
Ok(())
}
pub(crate) fn invalidate_page_revision(project_path: &str, page_id: &str) -> Result<(), String> {
match fs::remove_file(revision_path(project_path, page_id)) {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(format!("Failed to invalidate embedding revision: {err}")),
}
}
fn resolve_wiki_markdown_path(
project_path: &str,
relative_path: &str,
) -> Result<(PathBuf, String), PageEmbeddingError> {
let raw = relative_path.trim().replace('\\', "/");
if raw.is_empty() {
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
"path is required",
));
}
let relative = Path::new(&raw);
if relative.is_absolute()
|| relative.components().any(|component| {
matches!(
component,
Component::ParentDir | Component::RootDir | Component::Prefix(_)
)
})
|| relative.components().next() != Some(Component::Normal("wiki".as_ref()))
|| relative
.extension()
.and_then(|value| value.to_str())
.map(str::to_ascii_lowercase)
.as_deref()
!= Some("md")
{
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
"path must be a project-relative Markdown file under wiki/",
));
}
let project = fs::canonicalize(project_path).map_err(|err| {
PageEmbeddingError::new(
PageEmbeddingErrorKind::NotFound,
format!("Failed to resolve project path: {err}"),
)
})?;
let wiki = fs::canonicalize(project.join("wiki")).map_err(|err| {
PageEmbeddingError::new(
PageEmbeddingErrorKind::NotFound,
format!("Failed to resolve project wiki directory: {err}"),
)
})?;
let page = fs::canonicalize(project.join(relative)).map_err(|err| {
PageEmbeddingError::new(
PageEmbeddingErrorKind::NotFound,
format!("Wiki page not found: {err}"),
)
})?;
if !page.starts_with(&wiki)
|| !page.is_file()
|| !page
.extension()
.and_then(|value| value.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
{
return Err(PageEmbeddingError::new(
PageEmbeddingErrorKind::InvalidRequest,
"path must resolve to a file inside the project wiki directory",
));
}
let normalized = page
.strip_prefix(&project)
.map_err(|_| {
PageEmbeddingError::new(PageEmbeddingErrorKind::InvalidRequest, "Invalid page path")
})?
.to_string_lossy()
.replace('\\', "/");
Ok((page, normalized))
}
fn extract_title(content: &str, fallback: &str) -> String {
let normalized = content.replace("\r\n", "\n");
if let Some(frontmatter) = leading_frontmatter(&normalized) {
for line in frontmatter.lines() {
if let Some(value) = line.trim().strip_prefix("title:") {
let title = value.trim().trim_matches(['\"', '\'']);
if !title.is_empty() {
return title.to_string();
}
}
}
}
normalized
.lines()
.find_map(|line| line.trim().strip_prefix("# ").map(str::trim))
.filter(|title| !title.is_empty())
.unwrap_or(fallback)
.to_string()
}
fn leading_frontmatter(content: &str) -> Option<&str> {
let rest = content.strip_prefix("---\n")?;
let (end, _) = find_frontmatter_close(rest)?;
Some(&rest[..end])
}
fn strip_frontmatter(content: &str) -> &str {
let Some(rest) = content.strip_prefix("---\n") else {
return content;
};
let Some((end, close_len)) = find_frontmatter_close(rest) else {
return content;
};
let after_fence = &rest[end + close_len..];
after_fence.strip_prefix('\n').unwrap_or(after_fence)
}
fn find_frontmatter_close(rest: &str) -> Option<(usize, usize)> {
let mut offset = 0usize;
for line in rest.split_inclusive('\n') {
if line.trim_end_matches(['\r', '\n']).trim() == "---" {
return Some((offset, line.len()));
}
offset += line.len();
}
(rest.trim() == "---").then_some((0, rest.len()))
}
// The regular ingest pipeline uses src/lib/text-chunker.ts. Keep heading,
// frontmatter, fenced-block, table, overlap, and Unicode behavior aligned when
// changing either implementation.
fn chunk_markdown(content: &str, target_chars: usize, overlap_chars: usize) -> Vec<MarkdownChunk> {
let normalized = content.replace("\r\n", "\n");
let body = strip_frontmatter(&normalized);
let mut chunks = Vec::new();
let mut heading_stack: Vec<(usize, String)> = Vec::new();
let mut heading_path = String::new();
let mut section = String::new();
let flush = |section: &mut String, heading_path: &str, chunks: &mut Vec<MarkdownChunk>| {
let text = section.trim();
if !text.is_empty() {
for piece in split_preserving_atomic_blocks(text, target_chars, overlap_chars) {
chunks.push(MarkdownChunk {
text: piece,
heading_path: heading_path.to_string(),
});
}
}
section.clear();
};
let mut open_fence: Option<(char, usize)> = None;
for line in body.lines() {
let trimmed = line.trim_start();
if let Some((marker, width)) = fence_marker(trimmed) {
match open_fence {
None => open_fence = Some((marker, width)),
Some((open, open_width)) if open == marker && width >= open_width => {
open_fence = None
}
Some(_) => {}
}
}
let heading = if open_fence.is_none() {
parse_heading(line)
} else {
None
};
if let Some((level, title)) = heading {
flush(&mut section, &heading_path, &mut chunks);
heading_stack.retain(|(existing, _)| *existing < level);
heading_stack.push((level, title.to_string()));
heading_path = heading_stack
.iter()
.map(|(level, title)| format!("{} {}", "#".repeat(*level), title))
.collect::<Vec<_>>()
.join(" > ");
}
if !section.is_empty() {
section.push('\n');
}
section.push_str(line);
}
flush(&mut section, &heading_path, &mut chunks);
chunks
}
fn split_preserving_atomic_blocks(
text: &str,
target_chars: usize,
overlap_chars: usize,
) -> Vec<String> {
let lines = text.lines().collect::<Vec<_>>();
let mut chunks = Vec::new();
let mut normal = Vec::new();
let mut index = 0usize;
let flush_normal = |normal: &mut Vec<&str>, chunks: &mut Vec<String>| {
if normal.is_empty() {
return;
}
chunks.extend(split_with_overlap(
&normal.join("\n"),
target_chars,
overlap_chars,
));
normal.clear();
};
while index < lines.len() {
if let Some((marker, width)) = fence_marker(lines[index]) {
flush_normal(&mut normal, &mut chunks);
let start = index;
index += 1;
while index < lines.len() {
let is_close =
fence_marker(lines[index]).is_some_and(|(candidate, candidate_width)| {
candidate == marker && candidate_width >= width
});
index += 1;
if is_close {
break;
}
}
push_atomic_chunk(&mut chunks, lines[start..index].join("\n"));
continue;
}
if lines[index].trim_start().starts_with('|') {
flush_normal(&mut normal, &mut chunks);
let start = index;
while index < lines.len() && lines[index].trim_start().starts_with('|') {
index += 1;
}
push_atomic_chunk(&mut chunks, lines[start..index].join("\n"));
continue;
}
normal.push(lines[index]);
index += 1;
}
flush_normal(&mut normal, &mut chunks);
chunks
.into_iter()
.filter(|chunk| !chunk.trim().is_empty())
.collect()
}
fn push_atomic_chunk(chunks: &mut Vec<String>, chunk: String) {
if chunk.chars().count() <= MAX_CHUNK_CHARS {
chunks.push(chunk);
} else {
// Preserve ordinary code/table blocks as a unit, but never send an
// unbounded payload to an embedding provider.
chunks.extend(split_with_overlap(
&chunk,
MAX_CHUNK_CHARS,
DEFAULT_OVERLAP_CHARS,
));
}
}
fn fence_marker(line: &str) -> Option<(char, usize)> {
let trimmed = line.trim_start();
let marker = trimmed.chars().next()?;
if marker != '`' && marker != '~' {
return None;
}
let width = trimmed.chars().take_while(|ch| *ch == marker).count();
(width >= 3).then_some((marker, width))
}
fn parse_heading(line: &str) -> Option<(usize, &str)> {
let hashes = line.chars().take_while(|ch| *ch == '#').count();
if !(1..=6).contains(&hashes) || line.chars().nth(hashes) != Some(' ') {
return None;
}
let title = line[hashes + 1..].trim();
(!title.is_empty()).then_some((hashes, title))
}
fn split_with_overlap(text: &str, target_chars: usize, overlap_chars: usize) -> Vec<String> {
let chars: Vec<char> = text.chars().collect();
if chars.len() <= target_chars {
return vec![text.to_string()];
}
let mut out = Vec::new();
let mut start = 0usize;
while start < chars.len() {
let hard_end = (start + target_chars).min(chars.len());
let end = if hard_end < chars.len() {
(start + target_chars / 2..hard_end)
.rev()
.find(|index| {
chars[*index].is_whitespace()
|| matches!(
chars[*index],
'。' | '!' | '?' | '.' | '!' | '?' | ';' | ';'
)
})
.map(|index| index + 1)
.unwrap_or(hard_end)
} else {
hard_end
};
let piece: String = chars[start..end].iter().collect();
if !piece.trim().is_empty() {
out.push(piece.trim().to_string());
}
if end == chars.len() {
break;
}
start = end.saturating_sub(overlap_chars).max(start + 1);
}
out
}
fn enrich_chunk(title: &str, chunk: &MarkdownChunk) -> String {
[title.trim(), chunk.heading_path.trim(), chunk.text.trim()]
.into_iter()
.filter(|part| !part.is_empty())
.collect::<Vec<_>>()
.join("\n\n")
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeMap;
use uuid::Uuid;
fn project() -> PathBuf {
let root = std::env::temp_dir().join(format!("llm-wiki-page-embed-{}", Uuid::new_v4()));
fs::create_dir_all(root.join("wiki/nested")).unwrap();
root
}
fn embedding_config(model: &str) -> SearchEmbeddingConfig {
SearchEmbeddingConfig {
enabled: true,
endpoint: "https://example.com/v1/embeddings".to_string(),
api_key: "secret".to_string(),
model: model.to_string(),
output_dimensionality: Some(768.0),
extra_headers: Some(BTreeMap::from([("X-Route".to_string(), "a".to_string())])),
max_chunk_chars: Some(1_000),
overlap_chunk_chars: Some(200),
}
}
#[test]
fn path_guard_accepts_only_existing_markdown_inside_wiki() {
let root = project();
fs::write(root.join("wiki/nested/page.md"), "# Page").unwrap();
fs::write(root.join("outside.md"), "# Outside").unwrap();
let (_, relative) =
resolve_wiki_markdown_path(root.to_str().unwrap(), "wiki\\nested/page.md").unwrap();
assert_eq!(relative, "wiki/nested/page.md");
assert!(resolve_wiki_markdown_path(root.to_str().unwrap(), "../outside.md").is_err());
assert!(resolve_wiki_markdown_path(root.to_str().unwrap(), "outside.md").is_err());
assert!(resolve_wiki_markdown_path(root.to_str().unwrap(), "wiki/missing.md").is_err());
let _ = fs::remove_dir_all(root);
}
#[test]
fn chunker_strips_crlf_frontmatter_and_preserves_heading_context() {
let input =
"---\r\ntitle: 测试页面\r\n---\r\n# 第一章\r\n\r\n内容一。\r\n## 第二节\r\n内容二。";
let chunks = chunk_markdown(input, 64, 8);
assert_eq!(extract_title(input, "fallback"), "测试页面");
assert!(chunks.iter().all(|chunk| !chunk.text.contains("title:")));
assert!(chunks
.iter()
.any(|chunk| chunk.heading_path.contains("# 第一章")));
assert!(chunks
.iter()
.any(|chunk| chunk.heading_path.contains("## 第二节")));
}
#[test]
fn cjk_hard_split_uses_character_boundaries_and_overlap() {
let text = "中文内容用于验证字符边界不会发生截断异常。".repeat(20);
let chunks = split_with_overlap(&text, 64, 8);
assert!(chunks.len() > 1);
assert!(chunks.iter().all(|chunk| chunk.chars().count() <= 64));
for pair in chunks.windows(2) {
let suffix: String = pair[0]
.chars()
.rev()
.take(8)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
assert!(pair[1].starts_with(&suffix));
}
}
#[test]
fn fingerprint_changes_with_embedding_semantics_but_not_api_key_rotation() {
let base = embedding_config("model-a");
let mut changed_model = base.clone();
changed_model.model = "model-b".to_string();
let mut rotated_key = base.clone();
rotated_key.api_key = "rotated".to_string();
assert_ne!(
embedding_fingerprint("sha256:content", &base),
embedding_fingerprint("sha256:content", &changed_model)
);
assert_eq!(
embedding_fingerprint("sha256:content", &base),
embedding_fingerprint("sha256:content", &rotated_key)
);
}
#[test]
fn frontmatter_close_requires_a_standalone_fence() {
let input = "---\ntitle: Kept\nnote: ---not-a-fence\n---\n# Body";
assert_eq!(extract_title(input, "fallback"), "Kept");
assert_eq!(strip_frontmatter(input), "# Body");
}
#[test]
fn vector_page_ids_include_the_wiki_relative_directory() {
assert_eq!(
vector_page_id("wiki/nested/page.md").as_deref(),
Some("nested/page")
);
assert_eq!(
vector_page_id("wiki/other/page.md").as_deref(),
Some("other/page")
);
assert_eq!(
vector_page_id("wiki/notes/Foo.MD").as_deref(),
Some("notes/Foo")
);
assert_eq!(vector_page_id("outside/page.md"), None);
}
#[test]
fn chunker_keeps_fenced_code_and_tables_atomic() {
let code = format!("```rust\n{}\n```", "let value = 1;\n".repeat(12));
let table = format!(
"| Name | Value |\n| --- | --- |\n{}",
"| A | B |\n".repeat(12)
);
let input = format!("# Page\n\nBefore text.\n\n{code}\n\n{table}\n\nAfter text.");
let chunks = chunk_markdown(&input, 64, 8);
assert!(chunks.iter().any(|chunk| chunk.text.trim() == code.trim()));
assert!(chunks.iter().any(|chunk| chunk.text.trim() == table.trim()));
assert_eq!(
chunks
.iter()
.filter(|chunk| chunk.text.contains("```rust"))
.count(),
1
);
assert_eq!(
chunks
.iter()
.filter(|chunk| chunk.text.contains("| Name | Value |"))
.count(),
1
);
}
#[test]
fn oversized_atomic_blocks_respect_the_hard_provider_limit() {
let input = format!("```text\n{}\n", "x".repeat(MAX_CHUNK_CHARS + 500));
let chunks = chunk_markdown(&input, 1_000, 200);
assert!(chunks.len() > 1);
assert!(chunks
.iter()
.all(|chunk| chunk.text.chars().count() <= MAX_CHUNK_CHARS));
}
#[test]
fn revision_metadata_is_per_page_and_invalidates_without_rewriting_other_pages() {
let root = project();
save_revision(root.to_str().unwrap(), "current", "sha256:current").unwrap();
save_revision(root.to_str().unwrap(), "other", "sha256:other").unwrap();
assert_eq!(
load_revision(root.to_str().unwrap(), "current").as_deref(),
Some("sha256:current")
);
invalidate_page_revision(root.to_str().unwrap(), "current").unwrap();
assert_eq!(load_revision(root.to_str().unwrap(), "current"), None);
assert_eq!(
load_revision(root.to_str().unwrap(), "other").as_deref(),
Some("sha256:other")
);
let _ = fs::remove_dir_all(root);
}
#[test]
fn inconsistent_provider_dimensions_are_rejected_before_storage() {
let chunk = MarkdownChunk {
text: "content".to_string(),
heading_path: String::new(),
};
let rows = vec![
chunk_row(0, &chunk, vec![1.0, 2.0]),
chunk_row(1, &chunk, vec![1.0]),
];
let error = validate_embedding_rows(&rows).unwrap_err();
assert_eq!(error.kind, PageEmbeddingErrorKind::Provider);
}
}